From 551326863c11c5729b049254f3692eb55079e8e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 18:35:03 -0800 Subject: [PATCH 001/115] =?UTF-8?q?=F0=9F=94=A7=20build:=20add=20project?= =?UTF-8?q?=20tooling=20and=20policies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/CODEOWNERS | 1 + .github/FUNDING.yaml | 2 + .github/SECURITY.md | 45 + .github/dependabot.yaml | 30 + .github/release.yaml | 5 + .mdformat.toml | 2 + .pre-commit-config.yaml | 54 + CHANGELOG.md | 3 + CONTRIBUTING.md | 48 + changelog.d/.gitkeep | 0 pylock.toml | 2972 +++++++++++++++++++++++++++++++++++++++ pyproject.toml | 210 +++ tox.toml | 72 + 13 files changed, 3444 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/FUNDING.yaml create mode 100644 .github/SECURITY.md create mode 100644 .github/dependabot.yaml create mode 100644 .github/release.yaml create mode 100644 .mdformat.toml create mode 100644 .pre-commit-config.yaml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 changelog.d/.gitkeep create mode 100644 pylock.toml create mode 100644 pyproject.toml create mode 100644 tox.toml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..0f9e29b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @gaborbernat diff --git a/.github/FUNDING.yaml b/.github/FUNDING.yaml new file mode 100644 index 0000000..220f93d --- /dev/null +++ b/.github/FUNDING.yaml @@ -0,0 +1,2 @@ +github: gaborbernat +thanks_dev: u/gh/gaborbernat diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000..5447e64 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,45 @@ +# Security policy + +hatch-pep770 is a hatchling build hook that writes a CycloneDX SBOM into wheels, plus a command line tool that renders +that SBOM as SPDX and validates it. This policy says how to report a security problem and how we decide whether a report +is one. + +## Reporting a vulnerability + +**Do not open a public issue for a security problem.** Use +[GitHub's private vulnerability reporting](https://github.com/tox-dev/hatch-pep770/security/advisories/new). If you do +not have a GitHub account, email [gaborjbernat@gmail.com](mailto:gaborjbernat@gmail.com). + +Include what the attacker controls and what they gain, the steps or a proof of concept, and the versions and platform +you tested. Write the report yourself; unreviewed tool or model output takes longer to triage. + +## What to expect + +Volunteers maintain hatch-pep770, with no on-call rotation. We aim to acknowledge a report within **5 working days** and +assess it within **15 working days**. Fixes are developed in a private fork attached to the draft advisory, ship in a +new release, and the advisory, with a CVE requested through GitHub, goes out with that release. The default embargo is +**90 days** from the report; we agree any change with the reporter. We credit reporters in the advisory under the name +they choose, or not at all. There is no bug bounty. + +## Supported versions + +Only the latest release receives fixes. Confirm the problem reproduces on it before reporting. + +## Vulnerability or bug + +The build hook runs inside a project's own build, with the trust that build already has: the project source, its +`pyproject.toml`, its build requirements and the build environment are trusted. A report that needs control of one of +those is a bug: we fix it in the open and publish no advisory. + +**In scope** + +- A wheel bundled as a data file, or an SBOM given to `hatch-pep770 spdx` or `validate`, that makes the tool write + outside its output file or the wheel's `.dist-info/sboms/` directory, or run code. +- An SBOM that misstates what the wheel contains in a way the project's own inputs cannot explain, such as a wrong hash. + +**Out of scope** + +- Anything hatchling's own build trusts. +- Vulnerabilities in the packages the SBOM lists; report those to their projects. + +If you cannot tell which side a finding falls on, report it privately and we will decide. diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..3c1c011 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,30 @@ +version: 2 +multi-ecosystem-groups: + dependencies: + schedule: + interval: weekly + day: tuesday + labels: + - dependencies + - skip news +updates: + - package-ecosystem: github-actions + directory: / + patterns: + - '*' + multi-ecosystem-group: dependencies + labels: + - dependencies + - skip news + cooldown: + default-days: 7 + - package-ecosystem: pre-commit + directory: / + patterns: + - '*' + multi-ecosystem-group: dependencies + labels: + - dependencies + - skip news + cooldown: + default-days: 7 diff --git a/.github/release.yaml b/.github/release.yaml new file mode 100644 index 0000000..5f89818 --- /dev/null +++ b/.github/release.yaml @@ -0,0 +1,5 @@ +changelog: + exclude: + authors: + - dependabot[bot] + - pre-commit-ci[bot] diff --git a/.mdformat.toml b/.mdformat.toml new file mode 100644 index 0000000..a571de0 --- /dev/null +++ b/.mdformat.toml @@ -0,0 +1,2 @@ +wrap = 120 +number = true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2a029eb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,54 @@ +ci: + autoupdate_schedule: quarterly +default_language_version: + python: python3.15 +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: a76b92851fc9eede9f9bc4073eb55202f34eb0df # frozen: 0.39.0 + hooks: + - id: check-github-workflows + args: ["--verbose"] + - repo: https://github.com/rhysd/actionlint + rev: 914e7df21a07ef503a81201c76d2b11c789d3fca # frozen: v1.7.12 + hooks: + - id: actionlint + args: ["-ignore", 'label "ubuntu-26[.]04" is unknown'] # GitHub offers this runner before actionlint lists it. + - repo: https://github.com/codespell-project/codespell + rev: 57b21406f092110c18776e39b0bda50d37c945c8 # frozen: v2.4.3 + hooks: + - id: codespell + - repo: https://github.com/tox-dev/pyproject-fmt + rev: "50e0f3aee9dc5747a5f173f79a7417403c18dda6" # frozen: v2.30.3 + hooks: + - id: pyproject-fmt + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "25bcf95026b96e2dd3c0aec3090c3d6ef6ca2e6c" # frozen: v0.17.3 + hooks: + - id: ruff-format + - id: ruff-check + args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] + - repo: https://github.com/hukkin/mdformat + rev: "82912cdaea4fb830f751504486a7879c70526547" # frozen: 1.0.0 + hooks: + - id: mdformat + additional_dependencies: + - mdformat-config==0.2.1 + - mdformat-gfm==1.0.0 + - mdformat-toc==0.5.0 + - repo: https://github.com/google/yamlfmt + rev: "21ca5323a9c87ee37a434e0ca908efc0a89daa07" # frozen: v0.21.0 + hooks: + - id: yamlfmt + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: c92afdf2c6ca336aaa8f39e2bd8d48f62dbd8e1f # frozen: v1.31.1 + hooks: + - id: zizmor + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b132379 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# Changelog + + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a1095ef --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,48 @@ +# Contributing + +Add a `changelog.d/+short-name..md` fragment for each user-visible change. Use a descriptive name instead of a +pull request number. After a squash merge, the release workflow links the fragment to the pull request. A pull request +with two separate user-visible changes needs two fragments. + +Do not write credits in fragments. The release credits come from the commit authors and `Co-authored-by` trailers, and +the prepare workflow runs `release-creator contributors` to add first-time contributors to the changelog. + +Use these categories: + +| Fragment suffix | Changelog heading | Use for | +| --------------- | ----------------- | --------------------------------------------------------- | +| `feature` | Added | New capabilities and Python support additions or removals | +| `change` | Changes | Changes to existing behavior | +| `bugfix` | Fixed | User-visible fixes | +| `doc` | Documentation | User documentation changes | +| `breaking` | Breaking changes | Incompatible changes to the public interface | +| `security` | Security | Security fixes users should know about | +| `performance` | Performance | User-visible speed or resource improvements | +| `deprecation` | Deprecated | Public features scheduled for removal | +| `removal` | Removed | Removed public features other than Python support | +| `packaging` | Packaging | Distribution or installation changes | + +For example, `changelog.d/+vendored-licenses.feature.md` could contain: + +```markdown +Record the licenses of packages that bundled wheels vendor. +``` + +Internal maintenance needs no fragment. Apply the `skip news` label to a human-authored maintenance pull request; the +bot labeling workflow handles supported automation. A bot-supplied fragment is kept. + +To preview pending news locally, install the locked release tools and let the fragments select the next version: + +```bash +uv venv --python 3.15 .tox/release-tools +uv pip sync --preview-features pylock --python .tox/release-tools/bin/python --group release-news pylock.toml +version=$(git tag --list | .tox/release-tools/bin/release-creator tags --next auto --directory changelog.d) +.tox/release-tools/bin/towncrier build --draft --version "$version" +``` + +The draft shows named fragments before merge. The final changelog gains pull request links after the squash commit is on +`main`, and the prepare workflow adds the credits. + +## Tests + +Add a test for every new feature and bug fix. A change without a test needs a reason in the pull request description. diff --git a/changelog.d/.gitkeep b/changelog.d/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pylock.toml b/pylock.toml new file mode 100644 index 0000000..64ef8ba --- /dev/null +++ b/pylock.toml @@ -0,0 +1,2972 @@ +lock-version = "1.0" +environments = [ + "python_version == \"3.15\" and sys_platform == \"linux\" and platform_machine == \"x86_64\" and implementation_name == \"cpython\" and platform_python_implementation == \"CPython\" and platform_system == \"Linux\"", + "python_version == \"3.15\" and sys_platform == \"darwin\" and platform_machine == \"arm64\" and platform_python_implementation == \"CPython\" and platform_system == \"Darwin\"", + "python_version == \"3.15\" and sys_platform == \"win32\" and platform_machine == \"AMD64\" and platform_python_implementation == \"CPython\" and platform_system == \"Windows\"", +] +requires-python = ">=3.9" +extras = [ + "validate", +] +dependency-groups = [ + "build", + "pkg-meta", + "project-runtime", + "release", + "release-news", + "release-notes", + "release-prepare", + "test", + "type", +] +default-groups = [ + "project-runtime", +] +created-by = "nab" + +[[packages]] +name = "annotated-types" +version = "0.8.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "annotated_types-0.8.0.tar.gz" +upload-time = 2026-07-23 20:16:13.995706+00:00 +url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz" +size = 15893 + +[packages.sdist.hashes] +sha256 = "13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7" + +[[packages.wheels]] +name = "annotated_types-0.8.0-py3-none-any.whl" +upload-time = 2026-07-23 20:16:12.938651+00:00 +url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl" +size = 13427 + +[packages.wheels.hashes] +sha256 = "f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0" + +[[packages]] +name = "arrow" +version = "1.4.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.8" +dependencies = [ + { name = "python-dateutil" }, + { name = "tzdata" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "arrow-1.4.0.tar.gz" +upload-time = 2025-10-18 17:46:46.761994+00:00 +url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz" +size = 152931 + +[packages.sdist.hashes] +sha256 = "ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7" + +[[packages.wheels]] +name = "arrow-1.4.0-py3-none-any.whl" +upload-time = 2025-10-18 17:46:45.663184+00:00 +url = "https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl" +size = 68797 + +[packages.wheels.hashes] +sha256 = "749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205" + +[[packages]] +name = "attrs" +version = "26.1.0" +marker = "\"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "attrs-26.1.0.tar.gz" +upload-time = 2026-03-19 14:22:25.026315+00:00 +url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz" +size = 952055 + +[packages.sdist.hashes] +sha256 = "d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32" + +[[packages.wheels]] +name = "attrs-26.1.0-py3-none-any.whl" +upload-time = 2026-03-19 14:22:23.645947+00:00 +url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl" +size = 67548 + +[packages.wheels.hashes] +sha256 = "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + +[[packages]] +name = "beartype" +version = "0.23.1" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "beartype-0.23.1.tar.gz" +upload-time = 2026-10-11 05:36:55.251880+00:00 +url = "https://files.pythonhosted.org/packages/79/34/c0cfdaee9c796c8381439fed0fcab62810753b8b0adb1ec88fa7f41eaa5d/beartype-0.23.1.tar.gz" +size = 1993714 + +[packages.sdist.hashes] +sha256 = "8b805f246b32931c74f2b321ea0939fe5bb2ff43c12a3eff19cc4c854b832e1d" + +[[packages.wheels]] +name = "beartype-0.23.1-py3-none-any.whl" +upload-time = 2026-10-11 05:36:52.431648+00:00 +url = "https://files.pythonhosted.org/packages/1c/cb/2f6aa982e2860c1156830db3da7f823b71784ed7d533ca289ea9b6c6347a/beartype-0.23.1-py3-none-any.whl" +size = 1582101 + +[packages.wheels.hashes] +sha256 = "4461b4dc57e3fdd6c8a8464b22cf118002eea107c33e9fef651c69068ab3cce3" + +[[packages]] +name = "boolean-py" +version = "5.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "boolean_py-5.0.tar.gz" +upload-time = 2025-04-03 10:39:49.734090+00:00 +url = "https://files.pythonhosted.org/packages/c4/cf/85379f13b76f3a69bca86b60237978af17d6aa0bc5998978c3b8cf05abb2/boolean_py-5.0.tar.gz" +size = 37047 + +[packages.sdist.hashes] +sha256 = "60cbc4bad079753721d32649545505362c754e121570ada4658b852a3a318d95" + +[[packages.wheels]] +name = "boolean_py-5.0-py3-none-any.whl" +upload-time = 2025-04-03 10:39:48.449265+00:00 +url = "https://files.pythonhosted.org/packages/e5/ca/78d423b324b8d77900030fa59c4aa9054261ef0925631cd2501dd015b7b7/boolean_py-5.0-py3-none-any.whl" +size = 26577 + +[packages.wheels.hashes] +sha256 = "ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9" + +[[packages]] +name = "certifi" +version = "2026.10.23" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "certifi-2026.10.23.tar.gz" +upload-time = 2026-10-23 11:36:16.478386+00:00 +url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz" +size = 136225 + +[packages.sdist.hashes] +sha256 = "a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd" + +[[packages.wheels]] +name = "certifi-2026.10.23-py3-none-any.whl" +upload-time = 2026-10-23 11:36:14.763067+00:00 +url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl" +size = 135082 + +[packages.wheels.hashes] +sha256 = "e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1" + +[[packages]] +name = "cffi" +version = "2.1.1" +marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +requires-python = ">=3.10" +dependencies = [ + { name = "pycparser" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "cffi-2.1.1.tar.gz" +upload-time = 2026-08-03 21:21:18.939988+00:00 +url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz" +size = 530807 + +[packages.sdist.hashes] +sha256 = "dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be" + +[[packages.wheels]] +name = "cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +upload-time = 2026-08-03 21:20:57.336082+00:00 +url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +size = 221630 + +[packages.wheels.hashes] +sha256 = "aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399" + +[[packages]] +name = "charset-normalizer" +version = "3.5.2" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "charset_normalizer-3.5.2.tar.gz" +upload-time = 2026-09-30 04:39:23.398219+00:00 +url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz" +size = 172659 + +[packages.sdist.hashes] +sha256 = "39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef" + +[[packages.wheels]] +name = "charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl" +upload-time = 2026-09-30 04:37:28.329851+00:00 +url = "https://files.pythonhosted.org/packages/ca/e9/34e597dee616d0b8ee4b34d29399e85c2204ade174157a48505d42baa4ff/charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl" +size = 367676 + +[packages.wheels.hashes] +sha256 = "87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26" + +[[packages.wheels]] +name = "charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" +upload-time = 2026-09-30 04:37:37.397915+00:00 +url = "https://files.pythonhosted.org/packages/a0/a1/4fbf5d0f0f1b2a080474c1cf9a2f12c4c6531bb0e8ba591055e846d2b4e9/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" +size = 257336 + +[packages.wheels.hashes] +sha256 = "3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229" + +[[packages.wheels]] +name = "charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl" +upload-time = 2026-09-30 04:37:53.427053+00:00 +url = "https://files.pythonhosted.org/packages/98/e8/55d5fd3935b4bce6da4fe0df61898e8c82653e317e677bd58aceb9c60f13/charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl" +size = 210858 + +[packages.wheels.hashes] +sha256 = "89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8" + +[[packages.wheels]] +name = "charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl" +upload-time = 2026-09-30 04:38:26.216534+00:00 +url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl" +size = 331815 + +[packages.wheels.hashes] +sha256 = "4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd" + +[[packages.wheels]] +name = "charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl" +upload-time = 2026-09-30 04:38:28.032254+00:00 +url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl" +size = 253276 + +[packages.wheels.hashes] +sha256 = "1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7" + +[[packages.wheels]] +name = "charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl" +upload-time = 2026-09-30 04:38:49.707190+00:00 +url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl" +size = 200092 + +[packages.wheels.hashes] +sha256 = "9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036" + +[[packages.wheels]] +name = "charset_normalizer-3.5.2-py3-none-any.whl" +upload-time = 2026-09-30 04:39:21.828049+00:00 +url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl" +size = 68872 + +[packages.wheels.hashes] +sha256 = "b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685" + +[[packages]] +name = "check-wheel-contents" +version = "0.6.3" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "attrs" }, + { name = "click" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "wheel-filename" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "check_wheel_contents-0.6.3.tar.gz" +upload-time = 2025-08-02 14:01:45.546952+00:00 +url = "https://files.pythonhosted.org/packages/95/37/2b9e0d38c2f668791ffe8b97711185c364d91c027e47f189c371c2348d18/check_wheel_contents-0.6.3.tar.gz" +size = 586023 + +[packages.sdist.hashes] +sha256 = "10e6939e2fe4e6ce1edf2ff6ec6157808677e80782e78021ae139dd88473a442" + +[[packages.wheels]] +name = "check_wheel_contents-0.6.3-py3-none-any.whl" +upload-time = 2025-08-02 14:01:43.968164+00:00 +url = "https://files.pythonhosted.org/packages/be/05/f39fde9f31ef80b285ef5822fad4ddabf73fec62a1f02c5beb4b2f328972/check_wheel_contents-0.6.3-py3-none-any.whl" +size = 27541 + +[packages.wheels.hashes] +sha256 = "5ae39c8c434b972f0740d04610759168590713175aab584b012b1b84f6771874" + +[[packages]] +name = "click" +version = "8.5.0" +marker = "\"pkg-meta\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "click-8.5.0.tar.gz" +upload-time = 2026-08-26 13:33:14.560531+00:00 +url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz" +size = 382235 + +[packages.sdist.hashes] +sha256 = "ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34" + +[[packages.wheels]] +name = "click-8.5.0-py3-none-any.whl" +upload-time = 2026-08-26 13:33:12.928043+00:00 +url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl" +size = 125251 + +[packages.wheels.hashes] +sha256 = "255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360" + +[[packages]] +name = "colorama" +version = "0.4.6" +marker = "(\"test\" in dependency_groups or \"type\" in dependency_groups) and sys_platform == \"win32\"" +requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "colorama-0.4.6.tar.gz" +upload-time = 2022-10-25 02:36:22.414799+00:00 +url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz" +size = 27697 + +[packages.sdist.hashes] +sha256 = "08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44" + +[[packages.wheels]] +name = "colorama-0.4.6-py2.py3-none-any.whl" +upload-time = 2022-10-25 02:36:20.889702+00:00 +url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl" +size = 25335 + +[packages.wheels.hashes] +sha256 = "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + +[[packages]] +name = "covdefaults" +version = "2.3.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.7" +dependencies = [ + { name = "coverage" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "covdefaults-2.3.0.tar.gz" +upload-time = 2023-03-05 16:43:34.779146+00:00 +url = "https://files.pythonhosted.org/packages/44/ee/9a6f2611f72e4c5657ae5542a510cf4164d2c673687c0ea73bb1cbd85b4d/covdefaults-2.3.0.tar.gz" +size = 4835 + +[packages.sdist.hashes] +sha256 = "4e99f679f12d792bc62e5510fa3eb59546ed47bd569e36e4fddc4081c9c3ebf7" + +[[packages.wheels]] +name = "covdefaults-2.3.0-py2.py3-none-any.whl" +upload-time = 2023-03-05 16:43:33.245581+00:00 +url = "https://files.pythonhosted.org/packages/76/4c/823bc951445aa97e5a1b7e337690db3abf85212c8d138e170922e7916ac8/covdefaults-2.3.0-py2.py3-none-any.whl" +size = 5144 + +[packages.wheels.hashes] +sha256 = "2832961f6ffcfe4b57c338bc3418a3526f495c26fb9c54565409c5532f7c41be" + +[[packages]] +name = "coverage" +version = "7.17.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.11" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "coverage-7.17.0.tar.gz" +upload-time = 2026-10-22 16:06:50.009670+00:00 +url = "https://files.pythonhosted.org/packages/d2/79/e78bd8f75a9b8e32233ce92d4607f0a21ce88a9156cace2884e7175a82f5/coverage-7.17.0.tar.gz" +size = 991021 + +[packages.sdist.hashes] +sha256 = "cc010a0a7fbdd9bb8bdfd0d3f04a5151275860213cbd7e2b876b29778489c712" + +[[packages.wheels]] +name = "coverage-7.17.0-cp315-cp315-macosx_11_0_arm64.whl" +upload-time = 2026-10-22 16:06:06.167385+00:00 +url = "https://files.pythonhosted.org/packages/87/db/94697b9ccf4ec94cdad07bf94e2a21b9702247cb212d2d5f18d2b9c3d0de/coverage-7.17.0-cp315-cp315-macosx_11_0_arm64.whl" +size = 226193 + +[packages.wheels.hashes] +sha256 = "66efc0fb99cee34626fc17d3adac9c3cd816bfdad563bb1e983c3ef432f566d4" + +[[packages.wheels]] +name = "coverage-7.17.0-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl" +upload-time = 2026-10-22 16:06:08.861264+00:00 +url = "https://files.pythonhosted.org/packages/95/ee/84ed138990ed752abd73db4adffc1842d6152017d8e21809cc109a413fda/coverage-7.17.0-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl" +size = 261735 + +[packages.wheels.hashes] +sha256 = "aed06129a9f530a948928260d7d2d81c47261355b0127e3c782251b38958463c" + +[[packages.wheels]] +name = "coverage-7.17.0-cp315-cp315-win_amd64.whl" +upload-time = 2026-10-22 16:06:23.756207+00:00 +url = "https://files.pythonhosted.org/packages/43/14/283120142ac281834808584249b89957de997c28573ddd65ae106b622121/coverage-7.17.0-cp315-cp315-win_amd64.whl" +size = 228532 + +[packages.wheels.hashes] +sha256 = "79cf71e5f9a48877ccfed5baa2c72f45af4feeaa596249e23eb15b473134830e" + +[[packages.wheels]] +name = "coverage-7.17.0-py3-none-any.whl" +upload-time = 2026-10-22 16:06:48.471847+00:00 +url = "https://files.pythonhosted.org/packages/c9/17/48032919343afa2906112d458aceda2043afdb24a2ee8eb4c60904d8ddf1/coverage-7.17.0-py3-none-any.whl" +size = 217383 + +[packages.wheels.hashes] +sha256 = "bac487dce0d2aff28ddf1a533a916f771416eee0e493960f611e49037b878e9d" + +[[packages]] +name = "cryptography" +version = "50.0.2" +marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +requires-python = "!=3.9.0,!=3.9.1,>=3.9" +dependencies = [ + { name = "cffi" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "cryptography-50.0.2.tar.gz" +upload-time = 2026-09-30 15:30:04.884601+00:00 +url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz" +size = 880623 + +[packages.sdist.hashes] +sha256 = "7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5" + +[[packages.wheels]] +name = "cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +upload-time = 2026-09-30 14:43:49.010540+00:00 +url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +size = 4719841 + +[packages.wheels.hashes] +sha256 = "630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2" + +[[packages.wheels]] +name = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl" +upload-time = 2026-09-30 14:43:55.272150+00:00 +url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl" +size = 4753050 + +[packages.wheels.hashes] +sha256 = "4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856" + +[[packages.wheels]] +name = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl" +upload-time = 2026-09-30 14:44:04.545137+00:00 +url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl" +size = 4752576 + +[packages.wheels.hashes] +sha256 = "9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079" + +[[packages.wheels]] +name = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +upload-time = 2026-09-30 14:44:45.769143+00:00 +url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +size = 4966465 + +[packages.wheels.hashes] +sha256 = "f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67" + +[[packages.wheels]] +name = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl" +upload-time = 2026-09-30 14:44:53.379605+00:00 +url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl" +size = 5001199 + +[packages.wheels.hashes] +sha256 = "58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7" + +[[packages.wheels]] +name = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl" +upload-time = 2026-09-30 14:45:05.009031+00:00 +url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl" +size = 5000859 + +[packages.wheels.hashes] +sha256 = "e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c" + +[[packages.wheels]] +name = "cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +upload-time = 2026-09-30 15:29:26.807001+00:00 +url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +size = 4749145 + +[packages.wheels.hashes] +sha256 = "ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767" + +[[packages.wheels]] +name = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl" +upload-time = 2026-09-30 15:29:32.605292+00:00 +url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl" +size = 4781387 + +[packages.wheels.hashes] +sha256 = "f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5" + +[[packages.wheels]] +name = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl" +upload-time = 2026-09-30 15:29:41.251208+00:00 +url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl" +size = 4780973 + +[packages.wheels.hashes] +sha256 = "1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c" + +[[packages]] +name = "cyclonedx-python-lib" +version = "11.12.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = "<4.0,>=3.9" +dependencies = [ + { name = "jsonschema" }, + { name = "license-expression" }, + { name = "packageurl-python" }, + { name = "py-serializable" }, + { name = "referencing" }, + { name = "sortedcontainers" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "cyclonedx_python_lib-11.12.0.tar.gz" +upload-time = 2026-08-13 07:52:18.675291+00:00 +url = "https://files.pythonhosted.org/packages/17/40/6509e6cfd7f2f3255501690f46375fdd224949e21fd1e96f4f4c8a9041b1/cyclonedx_python_lib-11.12.0.tar.gz" +size = 1445378 + +[packages.sdist.hashes] +sha256 = "16767c4039de90c04e9f03348f8f0ed4b8ff842eaa7eefcad3a95685f970dacf" + +[[packages.wheels]] +name = "cyclonedx_python_lib-11.12.0-py3-none-any.whl" +upload-time = 2026-08-13 07:52:16.836824+00:00 +url = "https://files.pythonhosted.org/packages/cf/f0/b2cb999244f2f4194df63ecff6939228178fa63a33be809c412684ca8db7/cyclonedx_python_lib-11.12.0-py3-none-any.whl" +size = 529453 + +[packages.wheels.hashes] +sha256 = "0e807521a921a5c3cb8ce1153f8a61d29eedfe76a46aac2796b7c6b573391a54" + +[[packages]] +name = "defusedxml" +version = "0.7.1" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "defusedxml-0.7.1.tar.gz" +upload-time = 2021-03-08 10:59:26.269532+00:00 +url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz" +size = 75520 + +[packages.sdist.hashes] +sha256 = "1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69" + +[[packages.wheels]] +name = "defusedxml-0.7.1-py2.py3-none-any.whl" +upload-time = 2021-03-08 10:59:24.450195+00:00 +url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl" +size = 25604 + +[packages.wheels.hashes] +sha256 = "a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" + +[[packages]] +name = "docutils" +version = "0.23" +marker = "\"pkg-meta\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "docutils-0.23.tar.gz" +upload-time = 2026-05-27 17:41:06.934109+00:00 +url = "https://files.pythonhosted.org/packages/39/a4/5180d9afc57e8fca05601dd652bdff19604c218814037fe90ffc7625a50a/docutils-0.23.tar.gz" +size = 2303823 + +[packages.sdist.hashes] +sha256 = "746f5060322511280a1e50eb76846ed6bf2342984b2ac04dc42caa1a8d78799e" + +[[packages.wheels]] +name = "docutils-0.23-py3-none-any.whl" +upload-time = 2026-05-27 17:40:58.442528+00:00 +url = "https://files.pythonhosted.org/packages/32/91/30151a39f7570f448ed84529390628a651d7f27c87d73c9b887f8189695e/docutils-0.23-py3-none-any.whl" +size = 634701 + +[packages.wheels.hashes] +sha256 = "25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea" + +[[packages]] +name = "fqdn" +version = "1.6.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<4,>=2.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "fqdn-1.6.0.tar.gz" +upload-time = 2026-09-28 05:43:35.482270+00:00 +url = "https://files.pythonhosted.org/packages/02/01/248d44fbda9e78fa1a7b5a475194bef4c4a3e285db286419f3bb42f2efdf/fqdn-1.6.0.tar.gz" +size = 130737 + +[packages.sdist.hashes] +sha256 = "e39bf62e2a9481aa2cb780554fdfd944dd8ee5f230eb831684d539cbafed8de2" + +[[packages.wheels]] +name = "fqdn-1.6.0-py3-none-any.whl" +upload-time = 2026-09-28 05:43:33.951821+00:00 +url = "https://files.pythonhosted.org/packages/15/d5/a8f9149b97cced9e1f4bff92ff0b6206120e2c853ae774d7020d6ecd8fa8/fqdn-1.6.0-py3-none-any.whl" +size = 12569 + +[packages.wheels.hashes] +sha256 = "6c793c312ccfa29981e1135b3ef5e1277ebe639731a6b0f7350d90e96e63f0b4" + +[[packages]] +name = "hatch-vcs" +version = "0.5.0" +marker = "\"build\" in dependency_groups" +requires-python = ">=3.9" +dependencies = [ + { name = "hatchling" }, + { name = "setuptools-scm" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "hatch_vcs-0.5.0.tar.gz" +upload-time = 2025-05-27 05:16:04.490926+00:00 +url = "https://files.pythonhosted.org/packages/6b/b0/4cc743d38adbee9d57d786fa496ed1daadb17e48589b6da8fa55717a0746/hatch_vcs-0.5.0.tar.gz" +size = 11424 + +[packages.sdist.hashes] +sha256 = "0395fa126940340215090c344a2bf4e2a77bcbe7daab16f41b37b98c95809ff9" + +[[packages.wheels]] +name = "hatch_vcs-0.5.0-py3-none-any.whl" +upload-time = 2025-05-27 05:16:03.184909+00:00 +url = "https://files.pythonhosted.org/packages/5f/48/1f85cee4b7b4f40b9b814b1febbc661bda6ced9649e410a0b74f6e415dd0/hatch_vcs-0.5.0-py3-none-any.whl" +size = 8507 + +[packages.wheels.hashes] +sha256 = "b49677dbdc597460cc22d01b27ab3696f5e16a21ecf2700fb01bc28e1f2a04a7" + +[[packages]] +name = "hatchling" +version = "1.33.0" +marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "packaging" }, + { name = "pathspec" }, + { name = "pluggy" }, + { name = "tomlkit" }, + { name = "trove-classifiers" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "hatchling-1.33.0.tar.gz" +upload-time = 2026-10-28 02:27:30.694770+00:00 +url = "https://files.pythonhosted.org/packages/7a/f1/e47b937226ae3b093bbd0132eb0b7ee5bfad12c2b5324afd10ae49946b05/hatchling-1.33.0.tar.gz" +size = 58079 + +[packages.sdist.hashes] +sha256 = "4c422bd4d554fcff5de5b73e9596f85e41e9f97f7e553d8f7595ecb31a627bef" + +[[packages.wheels]] +name = "hatchling-1.33.0-py3-none-any.whl" +upload-time = 2026-10-28 02:27:29.033037+00:00 +url = "https://files.pythonhosted.org/packages/32/64/4afe5dd77b2939a93e8c9513f8082dbd049ecba9c1484a379a0f6ae29d6b/hatchling-1.33.0-py3-none-any.whl" +size = 78750 + +[packages.wheels.hashes] +sha256 = "415290b84f464d2884303fca8974d38e7b85579069fab02a15db0828fbb1446d" + +[[packages]] +name = "id" +version = "1.6.1" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.9" +dependencies = [ + { name = "urllib3" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "id-1.6.1.tar.gz" +upload-time = 2026-02-04 16:19:41.260133+00:00 +url = "https://files.pythonhosted.org/packages/6d/04/c2156091427636080787aac190019dc64096e56a23b7364d3c1764ee3a06/id-1.6.1.tar.gz" +size = 18088 + +[packages.sdist.hashes] +sha256 = "d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069" + +[[packages.wheels]] +name = "id-1.6.1-py3-none-any.whl" +upload-time = 2026-02-04 16:19:40.051745+00:00 +url = "https://files.pythonhosted.org/packages/42/77/de194443bf38daed9452139e960c632b0ef9f9a5dd9ce605fdf18ca9f1b1/id-1.6.1-py3-none-any.whl" +size = 14689 + +[packages.wheels.hashes] +sha256 = "f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca" + +[[packages]] +name = "idna" +version = "3.20" +marker = "\"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "idna-3.20.tar.gz" +upload-time = 2026-09-17 14:11:04.752193+00:00 +url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz" +size = 216463 + +[packages.sdist.hashes] +sha256 = "a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44" + +[[packages.wheels]] +name = "idna-3.20-py3-none-any.whl" +upload-time = 2026-09-17 14:11:03.168153+00:00 +url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl" +size = 69583 + +[packages.wheels.hashes] +sha256 = "ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c" + +[[packages]] +name = "iniconfig" +version = "2.3.1" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "iniconfig-2.3.1.tar.gz" +upload-time = 2026-10-06 22:48:38.076390+00:00 +url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz" +size = 21209 + +[packages.sdist.hashes] +sha256 = "67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960" + +[[packages.wheels]] +name = "iniconfig-2.3.1-py3-none-any.whl" +upload-time = 2026-10-06 22:48:36.959685+00:00 +url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl" +size = 7552 + +[packages.wheels.hashes] +sha256 = "9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7" + +[[packages]] +name = "isoduration" +version = "20.11.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.7" +dependencies = [ + { name = "arrow" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "isoduration-20.11.0.tar.gz" +upload-time = 2020-11-01 11:00:00.312591+00:00 +url = "https://files.pythonhosted.org/packages/7c/1a/3c8edc664e06e6bd06cce40c6b22da5f1429aa4224d0c590f3be21c91ead/isoduration-20.11.0.tar.gz" +size = 11649 + +[packages.sdist.hashes] +sha256 = "ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9" + +[[packages.wheels]] +name = "isoduration-20.11.0-py3-none-any.whl" +upload-time = 2020-11-01 10:59:58.020369+00:00 +url = "https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl" +size = 11321 + +[packages.wheels.hashes] +sha256 = "b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042" + +[[packages]] +name = "jaraco-classes" +version = "3.4.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.8" +dependencies = [ + { name = "more-itertools" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "jaraco.classes-3.4.0.tar.gz" +upload-time = 2024-03-31 07:27:36.643708+00:00 +url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz" +size = 11780 + +[packages.sdist.hashes] +sha256 = "47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd" + +[[packages.wheels]] +name = "jaraco.classes-3.4.0-py3-none-any.whl" +upload-time = 2024-03-31 07:27:34.792832+00:00 +url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl" +size = 6777 + +[packages.wheels.hashes] +sha256 = "f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790" + +[[packages]] +name = "jaraco-context" +version = "6.1.2" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "jaraco_context-6.1.2.tar.gz" +upload-time = 2026-03-20 22:13:33.922545+00:00 +url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz" +size = 16801 + +[packages.sdist.hashes] +sha256 = "f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3" + +[[packages.wheels]] +name = "jaraco_context-6.1.2-py3-none-any.whl" +upload-time = 2026-03-20 22:13:32.808832+00:00 +url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl" +size = 7871 + +[packages.wheels.hashes] +sha256 = "bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535" + +[[packages]] +name = "jaraco-functools" +version = "4.6.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "more-itertools" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "jaraco_functools-4.6.0.tar.gz" +upload-time = 2026-07-14 01:28:02.544761+00:00 +url = "https://files.pythonhosted.org/packages/6c/1f/c23395957d41ccf27c4e535c3d334c4051e5395b3752057ba4cbaec35c56/jaraco_functools-4.6.0.tar.gz" +size = 20837 + +[packages.sdist.hashes] +sha256 = "880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280" + +[[packages.wheels]] +name = "jaraco_functools-4.6.0-py3-none-any.whl" +upload-time = 2026-07-14 01:28:01.590470+00:00 +url = "https://files.pythonhosted.org/packages/02/36/ecc85bc96c273dc8a11273ed4782272975e6338d4a3e9228621175edf0e3/jaraco_functools-4.6.0-py3-none-any.whl" +size = 11677 + +[packages.wheels.hashes] +sha256 = "99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30" + +[[packages]] +name = "jeepney" +version = "0.9.0" +marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +requires-python = ">=3.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "jeepney-0.9.0.tar.gz" +upload-time = 2025-02-27 18:51:01.684959+00:00 +url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz" +size = 106758 + +[packages.sdist.hashes] +sha256 = "cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732" + +[[packages.wheels]] +name = "jeepney-0.9.0-py3-none-any.whl" +upload-time = 2025-02-27 18:51:00.104279+00:00 +url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl" +size = 49010 + +[packages.wheels.hashes] +sha256 = "97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683" + +[[packages]] +name = "jinja2" +version = "3.1.6" +marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.7" +dependencies = [ + { name = "markupsafe" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "jinja2-3.1.6.tar.gz" +upload-time = 2025-03-05 20:05:02.478302+00:00 +url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz" +size = 245115 + +[packages.sdist.hashes] +sha256 = "0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d" + +[[packages.wheels]] +name = "jinja2-3.1.6-py3-none-any.whl" +upload-time = 2025-03-05 20:05:00.369721+00:00 +url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl" +size = 134899 + +[packages.wheels.hashes] +sha256 = "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" + +[[packages]] +name = "jsonpointer" +version = "3.6" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "jsonpointer-3.6.tar.gz" +upload-time = 2026-10-26 20:59:26.307086+00:00 +url = "https://files.pythonhosted.org/packages/54/04/e35b34f4e441473e62d82d6ed9a8f748d93d895f4c05d4e645aa0fde51d7/jsonpointer-3.6.tar.gz" +size = 14352 + +[packages.sdist.hashes] +sha256 = "82e839a3ba3a9e2b54394fda8b3d276c2459d80408d06ef9a6fbc369d7544ffc" + +[[packages.wheels]] +name = "jsonpointer-3.6-py3-none-any.whl" +upload-time = 2026-10-26 20:59:24.947043+00:00 +url = "https://files.pythonhosted.org/packages/6e/34/cee746fb30473bfaa92f19267fcc102a1118eaec83e7e2b5652a91ead01c/jsonpointer-3.6-py3-none-any.whl" +size = 9333 + +[packages.wheels.hashes] +sha256 = "a45c858f8a9e171f6b3790ed9446c9382782f6c4cb4589176813c5a553f4cfcd" + +[[packages]] +name = "jsonschema" +version = "4.26.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.10" +dependencies = [ + { name = "attrs" }, + { name = "fqdn" }, + { name = "idna" }, + { name = "isoduration" }, + { name = "jsonpointer" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rfc3339-validator" }, + { name = "rfc3986-validator" }, + { name = "rfc3987-syntax" }, + { name = "rpds-py" }, + { name = "uri-template" }, + { name = "webcolors" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "jsonschema-4.26.0.tar.gz" +upload-time = 2026-01-07 13:41:07.246891+00:00 +url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz" +size = 366583 + +[packages.sdist.hashes] +sha256 = "0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326" + +[[packages.wheels]] +name = "jsonschema-4.26.0-py3-none-any.whl" +upload-time = 2026-01-07 13:41:05.306692+00:00 +url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl" +size = 90630 + +[packages.wheels.hashes] +sha256 = "d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce" + +[[packages]] +name = "jsonschema-specifications" +version = "2025.9.1" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.9" +dependencies = [ + { name = "referencing" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "jsonschema_specifications-2025.9.1.tar.gz" +upload-time = 2025-09-08 01:34:59.186960+00:00 +url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz" +size = 32855 + +[packages.sdist.hashes] +sha256 = "b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d" + +[[packages.wheels]] +name = "jsonschema_specifications-2025.9.1-py3-none-any.whl" +upload-time = 2025-09-08 01:34:57.871954+00:00 +url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl" +size = 18437 + +[packages.wheels.hashes] +sha256 = "98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe" + +[[packages]] +name = "keyring" +version = "25.7.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.9" +dependencies = [ + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney" }, + { name = "pywin32-ctypes" }, + { name = "secretstorage" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "keyring-25.7.0.tar.gz" +upload-time = 2025-11-16 16:26:09.482176+00:00 +url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz" +size = 63516 + +[packages.sdist.hashes] +sha256 = "fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b" + +[[packages.wheels]] +name = "keyring-25.7.0-py3-none-any.whl" +upload-time = 2025-11-16 16:26:08.402146+00:00 +url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl" +size = 39160 + +[packages.wheels.hashes] +sha256 = "be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f" + +[[packages]] +name = "lark" +version = "1.3.1" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.8" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "lark-1.3.1.tar.gz" +upload-time = 2025-10-27 18:25:56.653447+00:00 +url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz" +size = 382732 + +[packages.sdist.hashes] +sha256 = "b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905" + +[[packages.wheels]] +name = "lark-1.3.1-py3-none-any.whl" +upload-time = 2025-10-27 18:25:54.882451+00:00 +url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl" +size = 113151 + +[packages.wheels.hashes] +sha256 = "c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12" + +[[packages]] +name = "license-expression" +version = "30.4.4" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.9" +dependencies = [ + { name = "boolean-py" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "license_expression-30.4.4.tar.gz" +upload-time = 2025-07-22 11:13:32.170576+00:00 +url = "https://files.pythonhosted.org/packages/40/71/d89bb0e71b1415453980fd32315f2a037aad9f7f70f695c7cec7035feb13/license_expression-30.4.4.tar.gz" +size = 186402 + +[packages.sdist.hashes] +sha256 = "73448f0aacd8d0808895bdc4b2c8e01a8d67646e4188f887375398c761f340fd" + +[[packages.wheels]] +name = "license_expression-30.4.4-py3-none-any.whl" +upload-time = 2025-07-22 11:13:31.217173+00:00 +url = "https://files.pythonhosted.org/packages/af/40/791891d4c0c4dab4c5e187c17261cedc26285fd41541577f900470a45a4d/license_expression-30.4.4-py3-none-any.whl" +size = 120615 + +[packages.wheels.hashes] +sha256 = "421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4" + +[[packages]] +name = "markdown-it-py" +version = "4.2.0" +marker = "\"pkg-meta\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "mdurl" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "markdown_it_py-4.2.0.tar.gz" +upload-time = 2026-05-07 12:08:28.360612+00:00 +url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz" +size = 82454 + +[packages.sdist.hashes] +sha256 = "04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49" + +[[packages.wheels]] +name = "markdown_it_py-4.2.0-py3-none-any.whl" +upload-time = 2026-05-07 12:08:27.182789+00:00 +url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl" +size = 91687 + +[packages.wheels.hashes] +sha256 = "9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a" + +[[packages]] +name = "markupsafe" +version = "3.0.4" +marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "markupsafe-3.0.4.tar.gz" +upload-time = 2026-10-02 23:07:22.290270+00:00 +url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz" +size = 153777 + +[packages.sdist.hashes] +sha256 = "2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6" + +[[packages.wheels]] +name = "markupsafe-3.0.4-cp315-cp315-macosx_11_0_arm64.whl" +upload-time = 2026-10-02 23:06:17.855823+00:00 +url = "https://files.pythonhosted.org/packages/96/24/3ec292b44064c16229e064d770b2625bd8ea941aa61f44905a9fa44942c0/markupsafe-3.0.4-cp315-cp315-macosx_11_0_arm64.whl" +size = 12075 + +[packages.wheels.hashes] +sha256 = "ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97" + +[[packages.wheels]] +name = "markupsafe-3.0.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" +upload-time = 2026-10-02 23:06:22.524416+00:00 +url = "https://files.pythonhosted.org/packages/82/96/ef49135ce260db4ca4a12b119ed468449cd248db6b1468e2112b546d7a2e/markupsafe-3.0.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" +size = 23111 + +[packages.wheels.hashes] +sha256 = "925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369" + +[[packages.wheels]] +name = "markupsafe-3.0.4-cp315-cp315-win_amd64.whl" +upload-time = 2026-10-02 23:06:33.258900+00:00 +url = "https://files.pythonhosted.org/packages/5b/ec/6000fd82e8791e58fcd0456ec20f098957e2b03d5ed02eb73241a577c0ba/markupsafe-3.0.4-cp315-cp315-win_amd64.whl" +size = 14496 + +[packages.wheels.hashes] +sha256 = "d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214" + +[[packages]] +name = "mdformat" +version = "1.0.0" +marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "markdown-it-py" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "mdformat-1.0.0.tar.gz" +upload-time = 2025-10-16 12:05:03.695033+00:00 +url = "https://files.pythonhosted.org/packages/3f/05/32b5e14b192b0a8a309f32232c580aefedd9d06017cb8fe8fce34bec654c/mdformat-1.0.0.tar.gz" +size = 56953 + +[packages.sdist.hashes] +sha256 = "4954045fcae797c29f86d4ad879e43bb151fa55dbaf74ac6eaeacf1d45bb3928" + +[[packages.wheels]] +name = "mdformat-1.0.0-py3-none-any.whl" +upload-time = 2025-10-16 12:05:02.607432+00:00 +url = "https://files.pythonhosted.org/packages/54/9a/8fe71b95985ca7a4001effbcc58e5a07a1f2a2884203f74dcf48a3b08315/mdformat-1.0.0-py3-none-any.whl" +size = 53288 + +[packages.wheels.hashes] +sha256 = "bca015d65a1d063a02e885a91daee303057bc7829c2cd37b2075a50dbb65944b" + +[[packages]] +name = "mdformat-gfm" +version = "1.0.0" +marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "markdown-it-py" }, + { name = "mdformat" }, + { name = "mdit-py-plugins" }, + { name = "wcwidth" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "mdformat_gfm-1.0.0.tar.gz" +upload-time = 2025-10-16 09:12:22.402429+00:00 +url = "https://files.pythonhosted.org/packages/56/6f/a626ebb142a290474401b67e2d61e73ce096bf7798ee22dfe6270f924b3f/mdformat_gfm-1.0.0.tar.gz" +size = 10112 + +[packages.sdist.hashes] +sha256 = "d1d49a409a6acb774ce7635c72d69178df7dce1dc8cdd10e19f78e8e57b72623" + +[[packages.wheels]] +name = "mdformat_gfm-1.0.0-py3-none-any.whl" +upload-time = 2025-10-16 09:12:21.276157+00:00 +url = "https://files.pythonhosted.org/packages/e6/18/6bc2189b744dd383cad03764f41f30352b1278d2205096f77a29c0b327ad/mdformat_gfm-1.0.0-py3-none-any.whl" +size = 10970 + +[packages.wheels.hashes] +sha256 = "7305a50efd2a140d7c83505b58e3ac5df2b09e293f9bbe72f6c7bee8c678b005" + +[[packages]] +name = "mdit-py-plugins" +version = "0.6.1" +marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "markdown-it-py" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "mdit_py_plugins-0.6.1.tar.gz" +upload-time = 2026-05-13 09:03:38.910230+00:00 +url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz" +size = 56114 + +[packages.sdist.hashes] +sha256 = "a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0" + +[[packages.wheels]] +name = "mdit_py_plugins-0.6.1-py3-none-any.whl" +upload-time = 2026-05-13 09:03:37.760000+00:00 +url = "https://files.pythonhosted.org/packages/a5/69/6da5581c6a7fede7dc261bf4e67d6adca4196f176b43288b55b3db395b6e/mdit_py_plugins-0.6.1-py3-none-any.whl" +size = 66663 + +[packages.wheels.hashes] +sha256 = "214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d" + +[[packages]] +name = "mdurl" +version = "0.1.2" +marker = "\"pkg-meta\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "mdurl-0.1.2.tar.gz" +upload-time = 2022-08-14 12:40:10.846281+00:00 +url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz" +size = 8729 + +[packages.sdist.hashes] +sha256 = "bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" + +[[packages.wheels]] +name = "mdurl-0.1.2-py3-none-any.whl" +upload-time = 2022-08-14 12:40:09.779676+00:00 +url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl" +size = 9979 + +[packages.wheels.hashes] +sha256 = "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" + +[[packages]] +name = "more-itertools" +version = "11.2.1" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.11" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "more_itertools-11.2.1.tar.gz" +upload-time = 2026-10-14 18:50:30.407709+00:00 +url = "https://files.pythonhosted.org/packages/52/b8/c63995bebbb9ac9c059350e0fb8c82c2420c85aeca7d34e40aecf3df9281/more_itertools-11.2.1.tar.gz" +size = 154133 + +[packages.sdist.hashes] +sha256 = "cbf08fd0af284dc69718b9b76a8e6203df624d70209ea511e4219d350c856f63" + +[[packages.wheels]] +name = "more_itertools-11.2.1-py3-none-any.whl" +upload-time = 2026-10-14 18:50:29.116732+00:00 +url = "https://files.pythonhosted.org/packages/5d/57/3c4b1c6f2e0cc29ba023802458ea3d2a996f87085559e4a5f2192d549532/more_itertools-11.2.1-py3-none-any.whl" +size = 74405 + +[packages.wheels.hashes] +sha256 = "35a7377edd1dd6608dcb2cdf534ded55ea32d49448875ad042cd3f879fb1ded0" + +[[packages]] +name = "nh3" +version = "0.3.7" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.8" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "nh3-0.3.7.tar.gz" +upload-time = 2026-08-23 14:26:30.728660+00:00 +url = "https://files.pythonhosted.org/packages/18/2f/022b27146d52d24b1b353b003359134788ecbcd6fcdf6283adbd57c0fbc8/nh3-0.3.7.tar.gz" +size = 25662 + +[packages.sdist.hashes] +sha256 = "71860d01c16f4d8c72e334e0674beb2b0899dbd0bf760de18932ef4390303848" + +[[packages.wheels]] +name = "nh3-0.3.7-cp38-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl" +upload-time = 2026-08-23 14:26:09.025294+00:00 +url = "https://files.pythonhosted.org/packages/94/0d/c257754bf57f829f307aa226bbe136d3a1356b5a0d08324c7b6bd2a8aacd/nh3-0.3.7-cp38-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl" +size = 1493959 + +[packages.wheels.hashes] +sha256 = "6c3aa50eb26e9228238271db9f983cbc3b006dfbfeca2d4dc34c33ddc6ac5ea5" + +[[packages.wheels]] +name = "nh3-0.3.7-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +upload-time = 2026-08-23 14:26:17.272038+00:00 +url = "https://files.pythonhosted.org/packages/a6/ed/c5510c615dce55b6fcc364aa1838142f938beed64f5e4927490dfcaf4405/nh3-0.3.7-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +size = 832161 + +[packages.wheels.hashes] +sha256 = "70f5ac8626e899a4bab0ef74ca2f5bd602f49c7b739e6e5026b4afc6d63dac42" + +[[packages.wheels]] +name = "nh3-0.3.7-cp38-abi3-win_amd64.whl" +upload-time = 2026-08-23 14:26:28.294566+00:00 +url = "https://files.pythonhosted.org/packages/b4/b9/34433ccb1f0fe6968dabbb7d4bf5721c6221878ef07832748c06655a6a80/nh3-0.3.7-cp38-abi3-win_amd64.whl" +size = 644462 + +[packages.wheels.hashes] +sha256 = "618e3059caf41ccdf5dcccb3fa9df4cf6e4efe23d1382a8bbfca272a8a4f8bfc" + +[[packages]] +name = "packageurl-python" +version = "0.17.6" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.8" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "packageurl_python-0.17.6.tar.gz" +upload-time = 2025-11-24 15:20:17.998933+00:00 +url = "https://files.pythonhosted.org/packages/f5/d6/3b5a4e3cfaef7a53869a26ceb034d1ff5e5c27c814ce77260a96d50ab7bb/packageurl_python-0.17.6.tar.gz" +size = 50618 + +[packages.sdist.hashes] +sha256 = "1252ce3a102372ca6f86eb968e16f9014c4ba511c5c37d95a7f023e2ca6e5c25" + +[[packages.wheels]] +name = "packageurl_python-0.17.6-py3-none-any.whl" +upload-time = 2025-11-24 15:20:16.962118+00:00 +url = "https://files.pythonhosted.org/packages/b1/2f/c7277b7615a93f51b5fbc1eacfc1b75e8103370e786fd8ce2abf6e5c04ab/packageurl_python-0.17.6-py3-none-any.whl" +size = 36776 + +[packages.wheels.hashes] +sha256 = "31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9" + +[[packages]] +name = "packaging" +version = "26.3" +marker = "\"build\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"project-runtime\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "packaging-26.3.tar.gz" +upload-time = 2026-08-04 18:15:28.737071+00:00 +url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz" +size = 313412 + +[packages.sdist.hashes] +sha256 = "94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79" + +[[packages.wheels]] +name = "packaging-26.3-py3-none-any.whl" +upload-time = 2026-08-04 18:15:27.159957+00:00 +url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl" +size = 129956 + +[packages.wheels.hashes] +sha256 = "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c" + +[[packages]] +name = "pathspec" +version = "1.1.1" +marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pathspec-1.1.1.tar.gz" +upload-time = 2026-04-27 01:46:08.907631+00:00 +url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz" +size = 135180 + +[packages.sdist.hashes] +sha256 = "17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a" + +[[packages.wheels]] +name = "pathspec-1.1.1-py3-none-any.whl" +upload-time = 2026-04-27 01:46:07.060850+00:00 +url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl" +size = 57328 + +[packages.wheels.hashes] +sha256 = "a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189" + +[[packages]] +name = "pluggy" +version = "1.7.0" +marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pluggy-1.7.0.tar.gz" +upload-time = 2026-10-15 09:50:58.343484+00:00 +url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz" +size = 123304 + +[packages.sdist.hashes] +sha256 = "d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8" + +[[packages.wheels]] +name = "pluggy-1.7.0-py3-none-any.whl" +upload-time = 2026-10-15 09:50:56.808462+00:00 +url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl" +size = 27082 + +[packages.wheels.hashes] +sha256 = "7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec" + +[[packages]] +name = "ply" +version = "3.11" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "ply-3.11.tar.gz" +upload-time = 2018-02-15 19:01:31.097641+00:00 +url = "https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz" +size = 159130 + +[packages.sdist.hashes] +sha256 = "00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3" + +[[packages.wheels]] +name = "ply-3.11-py2.py3-none-any.whl" +upload-time = 2018-02-15 19:01:27.172483+00:00 +url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl" +size = 49567 + +[packages.wheels.hashes] +sha256 = "096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce" + +[[packages]] +name = "prek" +version = "0.5.9" +marker = "\"release\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.8" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "prek-0.5.9.tar.gz" +upload-time = 2026-10-30 10:32:01.272337+00:00 +url = "https://files.pythonhosted.org/packages/f2/c6/89312171d4b87b7d2419a077f84e638bee69d1b7943598c72685655f86a7/prek-0.5.9.tar.gz" +size = 603716 + +[packages.sdist.hashes] +sha256 = "47bc660a7e959860a44afb5ff3dc167d96f04d929b405020a0f2c7683d3d00d8" + +[[packages.wheels]] +name = "prek-0.5.9-py3-none-macosx_11_0_arm64.whl" +upload-time = 2026-10-30 10:31:46.060256+00:00 +url = "https://files.pythonhosted.org/packages/5d/17/cef855f5a9c40d588557bf7a703bce0420d9592fa8407d2320df9a48135a/prek-0.5.9-py3-none-macosx_11_0_arm64.whl" +size = 5511607 + +[packages.wheels.hashes] +sha256 = "db03274472abdc033fce78ff28d6abe221eebd2388f0c6d293376b1b09df0978" + +[[packages.wheels]] +name = "prek-0.5.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +upload-time = 2026-10-30 10:31:52.132595+00:00 +url = "https://files.pythonhosted.org/packages/7c/de/9f538470c56f0524215d5ede21fdb092b5a5a55ee8b4ccc88a35487d57e0/prek-0.5.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +size = 5942047 + +[packages.wheels.hashes] +sha256 = "1ef04fd7b4bae3a31e1180bc87e877a3a5584063fe79320a125267afee47090d" + +[[packages.wheels]] +name = "prek-0.5.9-py3-none-win_amd64.whl" +upload-time = 2026-10-30 10:31:57.930703+00:00 +url = "https://files.pythonhosted.org/packages/cf/1e/2c35ff3b059b6fbb43273c3ebe996b6d9d37c1889607c5ba6b9eea66530c/prek-0.5.9-py3-none-win_amd64.whl" +size = 5501419 + +[packages.wheels.hashes] +sha256 = "b8fe1a01273203932b46318ebf304304838b6b73ff99b89531f42c86b63b86d5" + +[[packages]] +name = "py-serializable" +version = "2.1.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = "<4.0,>=3.8" +dependencies = [ + { name = "defusedxml" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "py_serializable-2.1.0.tar.gz" +upload-time = 2025-07-21 09:56:48.070686+00:00 +url = "https://files.pythonhosted.org/packages/73/21/d250cfca8ff30c2e5a7447bc13861541126ce9bd4426cd5d0c9f08b5547d/py_serializable-2.1.0.tar.gz" +size = 52368 + +[packages.sdist.hashes] +sha256 = "9d5db56154a867a9b897c0163b33a793c804c80cee984116d02d49e4578fc103" + +[[packages.wheels]] +name = "py_serializable-2.1.0-py3-none-any.whl" +upload-time = 2025-07-21 09:56:46.848289+00:00 +url = "https://files.pythonhosted.org/packages/9b/bf/7595e817906a29453ba4d99394e781b6fabe55d21f3c15d240f85dd06bb1/py_serializable-2.1.0-py3-none-any.whl" +size = 23045 + +[packages.wheels.hashes] +sha256 = "b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304" + +[[packages]] +name = "pycparser" +version = "3.12" +marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +requires-python = ">=3.11" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pycparser-3.12.tar.gz" +upload-time = 2026-10-22 13:40:03.560254+00:00 +url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz" +size = 158785 + +[packages.sdist.hashes] +sha256 = "f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6" + +[[packages.wheels]] +name = "pycparser-3.12-py3-none-any.whl" +upload-time = 2026-10-22 13:40:02.021539+00:00 +url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl" +size = 62600 + +[packages.wheels.hashes] +sha256 = "a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e" + +[[packages]] +name = "pydantic" +version = "2.14.2" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pydantic-2.14.2.tar.gz" +upload-time = 2026-10-29 16:37:12.476407+00:00 +url = "https://files.pythonhosted.org/packages/9f/d5/b4867fa7c7f04c34997d7d8aad744f64d210bb99d67cf18bf3fb1eea1aed/pydantic-2.14.2.tar.gz" +size = 911596 + +[packages.sdist.hashes] +sha256 = "fb7b2db6cefe8414dd612edeede49c4c433acb0432b0f92b95024e45f66c505d" + +[[packages.wheels]] +name = "pydantic-2.14.2-py3-none-any.whl" +upload-time = 2026-10-29 16:37:10.181777+00:00 +url = "https://files.pythonhosted.org/packages/97/01/c71a71258b58659c72504e5fd65b451f6e36e34dc931d0b237ded213c814/pydantic-2.14.2-py3-none-any.whl" +size = 490403 + +[packages.wheels.hashes] +sha256 = "7efbf7507b937931cd32a5755acb00ea1cf5ccb1d4e8067a5850d5069e382118" + +[[packages]] +name = "pydantic-core" +version = "2.50.2" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "typing-extensions" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pydantic_core-2.50.2.tar.gz" +upload-time = 2026-10-29 16:34:57.465908+00:00 +url = "https://files.pythonhosted.org/packages/af/8e/23ec1021834e35041fbb2f10946141e6acdfd39379c45cc4d65a4e8a1d98/pydantic_core-2.50.2.tar.gz" +size = 518711 + +[packages.sdist.hashes] +sha256 = "1b0e8dc3932cf4e9ed5727570325cacc716b9409ffc794a221c0ccc72e8eb775" + +[[packages.wheels]] +name = "pydantic_core-2.50.2-cp315-cp315-macosx_11_0_arm64.whl" +upload-time = 2026-10-29 16:33:44.946711+00:00 +url = "https://files.pythonhosted.org/packages/ae/ff/af99a09eddc348c1d0744c0df28dfcd4839ca497067f0fa2168208b41b1b/pydantic_core-2.50.2-cp315-cp315-macosx_11_0_arm64.whl" +size = 2100409 + +[packages.wheels.hashes] +sha256 = "f73930dacad5e611fe019e18df6f53c30629b1843d038d303b8dec86a32493bd" + +[[packages.wheels]] +name = "pydantic_core-2.50.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +upload-time = 2026-10-29 16:33:53.333286+00:00 +url = "https://files.pythonhosted.org/packages/e6/d0/dd0e355694094f0045922f4da60ac9bc7a314babc494e46089fb00640fd3/pydantic_core-2.50.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +size = 2249999 + +[packages.wheels.hashes] +sha256 = "5299296ff5cafcb69851fd652de8f9268a0bb036bcbedbfcaa94cf743baf3986" + +[[packages.wheels]] +name = "pydantic_core-2.50.2-cp315-cp315-win_amd64.whl" +upload-time = 2026-10-29 16:34:01.773818+00:00 +url = "https://files.pythonhosted.org/packages/24/6b/702adc91486fb96ba55675d58ee15ddc38fb48553fb35e26fd005e303206/pydantic_core-2.50.2-cp315-cp315-win_amd64.whl" +size = 2261064 + +[packages.wheels.hashes] +sha256 = "04cf672960b8c4fc28bb159c8cdd101fbd311c520237a5c0360448b851adfb41" + +[[packages]] +name = "pygments" +version = "2.21.0" +marker = "\"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pygments-2.21.0.tar.gz" +upload-time = 2026-08-17 08:02:48.824391+00:00 +url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz" +size = 5005329 + +[packages.sdist.hashes] +sha256 = "610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c" + +[[packages.wheels]] +name = "pygments-2.21.0-py3-none-any.whl" +upload-time = 2026-08-17 08:02:44.912148+00:00 +url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl" +size = 1250147 + +[packages.wheels.hashes] +sha256 = "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9" + +[[packages]] +name = "pyparsing" +version = "3.3.3" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pyparsing-3.3.3.tar.gz" +upload-time = 2026-09-20 20:59:05.609622+00:00 +url = "https://files.pythonhosted.org/packages/e4/11/b213bebff182584360cb8d17c72c1677fec5c5c228de439e63bcf8ab1c8f/pyparsing-3.3.3.tar.gz" +size = 1050487 + +[packages.sdist.hashes] +sha256 = "928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36" + +[[packages.wheels]] +name = "pyparsing-3.3.3-py3-none-any.whl" +upload-time = 2026-09-20 20:59:04.025856+00:00 +url = "https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl" +size = 126420 + +[packages.wheels.hashes] +sha256 = "ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4" + +[[packages]] +name = "pytest" +version = "9.2.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "colorama" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pytest-9.2.0.tar.gz" +upload-time = 2026-10-27 21:30:20.728655+00:00 +url = "https://files.pythonhosted.org/packages/79/42/bb4641652e58dedeac2ec51abc1f8cae4fee4ec63899a0999016229a0f2a/pytest-9.2.0.tar.gz" +size = 1797555 + +[packages.sdist.hashes] +sha256 = "97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b" + +[[packages.wheels]] +name = "pytest-9.2.0-py3-none-any.whl" +upload-time = 2026-10-27 21:30:19.131148+00:00 +url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl" +size = 411776 + +[packages.wheels.hashes] +sha256 = "07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588" + +[[packages]] +name = "pytest-cov" +version = "7.1.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pytest_cov-7.1.0.tar.gz" +upload-time = 2026-03-21 20:11:16.284275+00:00 +url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz" +size = 55592 + +[packages.sdist.hashes] +sha256 = "30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2" + +[[packages.wheels]] +name = "pytest_cov-7.1.0-py3-none-any.whl" +upload-time = 2026-03-21 20:11:14.438324+00:00 +url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl" +size = 22876 + +[packages.wheels.hashes] +sha256 = "a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678" + +[[packages]] +name = "pytest-mock" +version = "3.17.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "pytest" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pytest_mock-3.17.0.tar.gz" +upload-time = 2026-10-17 16:21:42.435701+00:00 +url = "https://files.pythonhosted.org/packages/97/c2/28b9f5b955ff2bcd4727a6f768e2ca9b586cd7c37b95ea5f7b9b873ac7b3/pytest_mock-3.17.0.tar.gz" +size = 43064 + +[packages.sdist.hashes] +sha256 = "e36576e0985e01027e999891e7a966c67822d40c89ef1b4c8b6e22737e3590b4" + +[[packages.wheels]] +name = "pytest_mock-3.17.0-py3-none-any.whl" +upload-time = 2026-10-17 16:21:41.265482+00:00 +url = "https://files.pythonhosted.org/packages/e8/ea/47fbdc8f37410956fd445720e927589b1e2a47772c5f28109f0ce2fa26d9/pytest_mock-3.17.0-py3-none-any.whl" +size = 11248 + +[packages.wheels.hashes] +sha256 = "1b82aa4b37137248bd687055e109723debeb07b489de1565297dc9589c5f2d6b" + +[[packages]] +name = "python-dateutil" +version = "3.0.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +dependencies = [ + { name = "six" }, + { name = "tzdata" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "python_dateutil-3.0.0.tar.gz" +upload-time = 2026-10-21 23:41:45.075347+00:00 +url = "https://files.pythonhosted.org/packages/0e/df/f15f80383f152452d8985b36d95cb8751f901a3e0ff4cd27f7e118f168e2/python_dateutil-3.0.0.tar.gz" +size = 271066 + +[packages.sdist.hashes] +sha256 = "7ebcb3c29a92f3ceafe79a7a48a5c45677e9e004bbeec5742ac0bbb00560b7a2" + +[[packages.wheels]] +name = "python_dateutil-3.0.0-py2.py3-none-any.whl" +upload-time = 2026-10-21 23:41:43.431814+00:00 +url = "https://files.pythonhosted.org/packages/5a/17/cebee01d1c0c1441174cf0456ddc91b280d79acb65340c05e41d77866ee8/python_dateutil-3.0.0-py2.py3-none-any.whl" +size = 94659 + +[packages.wheels.hashes] +sha256 = "3c79630202e4152c6add678446984236efe4f57e84d8e892aab0637bab82e9fd" + +[[packages]] +name = "pywin32-ctypes" +version = "0.2.3" +marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"win32\"" +requires-python = ">=3.6" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pywin32-ctypes-0.2.3.tar.gz" +upload-time = 2024-08-14 10:15:34.626878+00:00 +url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz" +size = 29471 + +[packages.sdist.hashes] +sha256 = "d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755" + +[[packages.wheels]] +name = "pywin32_ctypes-0.2.3-py3-none-any.whl" +upload-time = 2024-08-14 10:15:33.187242+00:00 +url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl" +size = 30756 + +[packages.wheels.hashes] +sha256 = "8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8" + +[[packages]] +name = "pyyaml" +version = "6.0.3" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.8" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pyyaml-6.0.3.tar.gz" +upload-time = 2025-09-25 21:33:16.546886+00:00 +url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz" +size = 130960 + +[packages.sdist.hashes] +sha256 = "d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f" + +[[packages]] +name = "rdflib" +version = "7.6.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.8.1" +dependencies = [ + { name = "pyparsing" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "rdflib-7.6.0.tar.gz" +upload-time = 2026-02-13 07:15:55.938955+00:00 +url = "https://files.pythonhosted.org/packages/98/f5/18bb77b7af9526add0c727a3b2048959847dc5fb030913e2918bf384fec3/rdflib-7.6.0.tar.gz" +size = 4943826 + +[packages.sdist.hashes] +sha256 = "6c831288d5e4a5a7ece85d0ccde9877d512a3d0f02d7c06455d00d6d0ea379df" + +[[packages.wheels]] +name = "rdflib-7.6.0-py3-none-any.whl" +upload-time = 2026-02-13 07:15:46.487762+00:00 +url = "https://files.pythonhosted.org/packages/10/c2/6604a71269e0c1bd75656d5a001432d16f2cc5b8c057140ec797155c295e/rdflib-7.6.0-py3-none-any.whl" +size = 615416 + +[packages.wheels.hashes] +sha256 = "30c0a3ebf4c0e09215f066be7246794b6492e054e782d7ac2a34c9f70a15e0dd" + +[[packages]] +name = "readme-renderer" +version = "46.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "docutils" }, + { name = "nh3" }, + { name = "pygments" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "readme_renderer-46.0.tar.gz" +upload-time = 2026-08-28 15:18:32.490778+00:00 +url = "https://files.pythonhosted.org/packages/25/d7/9309494fad74ee831d4546f69325b5519f37c6dfb2d9ba495db8c6d4f4ca/readme_renderer-46.0.tar.gz" +size = 38382 + +[packages.sdist.hashes] +sha256 = "af3e964914f6310a33ff67b72a4bdd940bed8d7c3bdecd2d14f40edf284bfe90" + +[[packages.wheels]] +name = "readme_renderer-46.0-py3-none-any.whl" +upload-time = 2026-08-28 15:18:31.132970+00:00 +url = "https://files.pythonhosted.org/packages/4b/72/ac5ca81fe9121fcaa9d828d21017cba00a16a98e4ea5fb60c878f93dda4f/readme_renderer-46.0-py3-none-any.whl" +size = 14239 + +[packages.wheels.hashes] +sha256 = "d0dae1f74bb273b534770cb4cccb6bb78735540afdb03c2146f4e19dcd412560" + +[[packages]] +name = "referencing" +version = "0.37.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.10" +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "referencing-0.37.0.tar.gz" +upload-time = 2025-10-13 15:30:48.871335+00:00 +url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz" +size = 78036 + +[packages.sdist.hashes] +sha256 = "44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8" + +[[packages.wheels]] +name = "referencing-0.37.0-py3-none-any.whl" +upload-time = 2025-10-13 15:30:47.625937+00:00 +url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl" +size = 26766 + +[packages.wheels.hashes] +sha256 = "381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231" + +[[packages]] +name = "release-creator" +version = "4.4.0" +marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.15" +dependencies = [ + { name = "docutils" }, + { name = "markdown-it-py" }, + { name = "mdformat" }, + { name = "mdformat-gfm" }, + { name = "packaging" }, + { name = "towncrier" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "release_creator-4.4.0.tar.gz" +upload-time = 2026-11-02 21:44:48.475063+00:00 +url = "https://files.pythonhosted.org/packages/d8/cf/8f376cc925c26fa5434d327ba03f4ed709b80d2077edda865fc3b2fe1940/release_creator-4.4.0.tar.gz" +size = 171450 + +[packages.sdist.hashes] +sha256 = "263d518ddee6371fa13d25bd07f7b5af0fa7a6e421b16f1e8143b0a9443bc84d" + +[[packages.wheels]] +name = "release_creator-4.4.0-py3-none-any.whl" +upload-time = 2026-11-02 21:44:47.195366+00:00 +url = "https://files.pythonhosted.org/packages/7b/29/6f2a3158b082b3bf8a94eb93662d80b4db4212abea0fc1fae07ac9a043da/release_creator-4.4.0-py3-none-any.whl" +size = 54784 + +[packages.wheels.hashes] +sha256 = "560ed208cca0dc53bc227878e5596e3e26e44fa277658c5d0dc662901d95f8cb" + +[[packages]] +name = "requests" +version = "2.34.2" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "requests-2.34.2.tar.gz" +upload-time = 2026-05-14 19:25:27.735762+00:00 +url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz" +size = 142856 + +[packages.sdist.hashes] +sha256 = "f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed" + +[[packages.wheels]] +name = "requests-2.34.2-py3-none-any.whl" +upload-time = 2026-05-14 19:25:26.443000+00:00 +url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl" +size = 73075 + +[packages.wheels.hashes] +sha256 = "2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0" + +[[packages]] +name = "requests-toolbelt" +version = "1.0.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +dependencies = [ + { name = "requests" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "requests-toolbelt-1.0.0.tar.gz" +upload-time = 2023-05-01 04:11:33.229998+00:00 +url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz" +size = 206888 + +[packages.sdist.hashes] +sha256 = "7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6" + +[[packages.wheels]] +name = "requests_toolbelt-1.0.0-py2.py3-none-any.whl" +upload-time = 2023-05-01 04:11:28.427086+00:00 +url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl" +size = 54481 + +[packages.wheels.hashes] +sha256 = "cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" + +[[packages]] +name = "rfc3339-validator" +version = "0.1.4" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +dependencies = [ + { name = "six" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "rfc3339_validator-0.1.4.tar.gz" +upload-time = 2021-05-12 16:37:54.178122+00:00 +url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz" +size = 5513 + +[packages.sdist.hashes] +sha256 = "138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b" + +[[packages.wheels]] +name = "rfc3339_validator-0.1.4-py2.py3-none-any.whl" +upload-time = 2021-05-12 16:37:52.536885+00:00 +url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl" +size = 3490 + +[packages.wheels.hashes] +sha256 = "24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa" + +[[packages]] +name = "rfc3986" +version = "2.0.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "rfc3986-2.0.0.tar.gz" +upload-time = 2022-01-10 00:52:30.832978+00:00 +url = "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz" +size = 49026 + +[packages.sdist.hashes] +sha256 = "97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c" + +[[packages.wheels]] +name = "rfc3986-2.0.0-py2.py3-none-any.whl" +upload-time = 2022-01-10 00:52:29.594625+00:00 +url = "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl" +size = 31326 + +[packages.wheels.hashes] +sha256 = "50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd" + +[[packages]] +name = "rfc3986-validator" +version = "0.1.1" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "rfc3986_validator-0.1.1.tar.gz" +upload-time = 2019-10-28 16:00:19.144483+00:00 +url = "https://files.pythonhosted.org/packages/da/88/f270de456dd7d11dcc808abfa291ecdd3f45ff44e3b549ffa01b126464d0/rfc3986_validator-0.1.1.tar.gz" +size = 6760 + +[packages.sdist.hashes] +sha256 = "3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055" + +[[packages.wheels]] +name = "rfc3986_validator-0.1.1-py2.py3-none-any.whl" +upload-time = 2019-10-28 16:00:13.976195+00:00 +url = "https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl" +size = 4242 + +[packages.wheels.hashes] +sha256 = "2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9" + +[[packages]] +name = "rfc3987-syntax" +version = "1.1.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.9" +dependencies = [ + { name = "lark" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "rfc3987_syntax-1.1.0.tar.gz" +upload-time = 2025-07-18 01:05:05.015375+00:00 +url = "https://files.pythonhosted.org/packages/2c/06/37c1a5557acf449e8e406a830a05bf885ac47d33270aec454ef78675008d/rfc3987_syntax-1.1.0.tar.gz" +size = 14239 + +[packages.sdist.hashes] +sha256 = "717a62cbf33cffdd16dfa3a497d81ce48a660ea691b1ddd7be710c22f00b4a0d" + +[[packages.wheels]] +name = "rfc3987_syntax-1.1.0-py3-none-any.whl" +upload-time = 2025-07-18 01:05:03.843605+00:00 +url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl" +size = 8046 + +[packages.wheels.hashes] +sha256 = "6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f" + +[[packages]] +name = "rich" +version = "15.0.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.9.0" +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "rich-15.0.0.tar.gz" +upload-time = 2026-04-12 08:24:00.750018+00:00 +url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz" +size = 230680 + +[packages.sdist.hashes] +sha256 = "edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36" + +[[packages.wheels]] +name = "rich-15.0.0-py3-none-any.whl" +upload-time = 2026-04-12 08:24:02.830783+00:00 +url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl" +size = 310654 + +[packages.wheels.hashes] +sha256 = "33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb" + +[[packages]] +name = "rpds-py" +version = "2026.9.1" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.11" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "rpds_py-2026.9.1.tar.gz" +upload-time = 2026-10-04 16:32:36.469775+00:00 +url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz" +size = 63948 + +[packages.sdist.hashes] +sha256 = "4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12" + +[[packages.wheels]] +name = "rpds_py-2026.9.1-cp315-cp315-macosx_11_0_arm64.whl" +upload-time = 2026-10-04 16:30:49.872953+00:00 +url = "https://files.pythonhosted.org/packages/ae/3f/ad45d03df4f84ebae5439577ee81f3999d182711e82235c8037b6528890e/rpds_py-2026.9.1-cp315-cp315-macosx_11_0_arm64.whl" +size = 342051 + +[packages.wheels.hashes] +sha256 = "00ba2d8c7dd4ee537978ddf4b3fbd712bef2d8751603f7f3146b3f4287768e25" + +[[packages.wheels]] +name = "rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +upload-time = 2026-10-04 16:30:58.295504+00:00 +url = "https://files.pythonhosted.org/packages/57/ca/e2e9a0a46a74ed51a0498ba1fe10f4ea6b2d9a155f372a2f91e51f18cf10/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +size = 372022 + +[packages.wheels.hashes] +sha256 = "2c16ab111bc27c646ba8aa005d0527754edc538ebb636f0b1bf8e244b48d1945" + +[[packages.wheels]] +name = "rpds_py-2026.9.1-cp315-cp315-win_amd64.whl" +upload-time = 2026-10-04 16:31:10.840084+00:00 +url = "https://files.pythonhosted.org/packages/6d/a5/71b5cd00e0521e3b6b81828baea368c62b6b700ebdd9554cd7d41ddf12fa/rpds_py-2026.9.1-cp315-cp315-win_amd64.whl" +size = 223121 + +[packages.wheels.hashes] +sha256 = "75c38c50ab9aca840225d9a9a3810bf11d04bd5c1f186cabbb8aee56db3e9b15" + +[[packages]] +name = "secretstorage" +version = "3.5.0" +marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +requires-python = ">=3.10" +dependencies = [ + { name = "cryptography" }, + { name = "jeepney" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "secretstorage-3.5.0.tar.gz" +upload-time = 2025-11-23 19:02:53.191898+00:00 +url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz" +size = 19884 + +[packages.sdist.hashes] +sha256 = "f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be" + +[[packages.wheels]] +name = "secretstorage-3.5.0-py3-none-any.whl" +upload-time = 2025-11-23 19:02:51.545472+00:00 +url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl" +size = 15554 + +[packages.wheels.hashes] +sha256 = "0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137" + +[[packages]] +name = "semantic-version" +version = "2.10.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=2.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "semantic_version-2.10.0.tar.gz" +upload-time = 2022-05-26 13:35:23.454764+00:00 +url = "https://files.pythonhosted.org/packages/7d/31/f2289ce78b9b473d582568c234e104d2a342fd658cc288a7553d83bb8595/semantic_version-2.10.0.tar.gz" +size = 52289 + +[packages.sdist.hashes] +sha256 = "bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c" + +[[packages.wheels]] +name = "semantic_version-2.10.0-py2.py3-none-any.whl" +upload-time = 2022-05-26 13:35:21.206388+00:00 +url = "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl" +size = 15552 + +[packages.wheels.hashes] +sha256 = "de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177" + +[[packages]] +name = "setuptools" +version = "84.2.1" +marker = "\"build\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "setuptools-84.2.1.tar.gz" +upload-time = 2026-10-20 19:01:37.742699+00:00 +url = "https://files.pythonhosted.org/packages/70/b3/c19426f783614d625699a4b247faf628ea9ab1db228674491767b417f9e1/setuptools-84.2.1.tar.gz" +size = 1230289 + +[packages.sdist.hashes] +sha256 = "3ff8b3b5492a0aa66960f4419365a86b84e013070d4313e4d4a6894f5d64d6bc" + +[[packages.wheels]] +name = "setuptools-84.2.1-py3-none-any.whl" +upload-time = 2026-10-20 19:01:35.678013+00:00 +url = "https://files.pythonhosted.org/packages/ef/55/6dc6d6bb953bdfc52519a0059592c52b6e3b8ad51cf8256dffceb55b0610/setuptools-84.2.1-py3-none-any.whl" +size = 876263 + +[packages.wheels.hashes] +sha256 = "ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2" + +[[packages]] +name = "setuptools-scm" +version = "10.3.4" +marker = "\"build\" in dependency_groups" +requires-python = ">=3.8" +dependencies = [ + { name = "packaging" }, + { name = "setuptools" }, + { name = "vcs-versioning" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "setuptools_scm-10.3.4.tar.gz" +upload-time = 2026-09-23 19:21:42.950372+00:00 +url = "https://files.pythonhosted.org/packages/85/d8/fc143f88819ccf10ba2388ba86732ee2de193e578234e25a783f6cc14bf7/setuptools_scm-10.3.4.tar.gz" +size = 78410 + +[packages.sdist.hashes] +sha256 = "a69f28bfc245608781205e912faae437c2b2165773afa4e7b979d77447a69dd2" + +[[packages.wheels]] +name = "setuptools_scm-10.3.4-py3-none-any.whl" +upload-time = 2026-09-23 19:21:41.607429+00:00 +url = "https://files.pythonhosted.org/packages/da/f5/54538a1f17ea753c42b0928dc2e62986d463f8c393340b9a1e602f2d4dd9/setuptools_scm-10.3.4-py3-none-any.whl" +size = 33366 + +[packages.wheels.hashes] +sha256 = "82f34c3e3084fc2b57d397200637cc13f2338004052d2fee47baa5f0e902464e" + +[[packages]] +name = "six" +version = "1.17.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "six-1.17.0.tar.gz" +upload-time = 2024-12-04 17:35:28.174320+00:00 +url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz" +size = 34031 + +[packages.sdist.hashes] +sha256 = "ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + +[[packages.wheels]] +name = "six-1.17.0-py2.py3-none-any.whl" +upload-time = 2024-12-04 17:35:26.475808+00:00 +url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl" +size = 11050 + +[packages.wheels.hashes] +sha256 = "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274" + +[[packages]] +name = "sortedcontainers" +version = "2.4.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sortedcontainers-2.4.0.tar.gz" +upload-time = 2021-05-16 22:03:42.897871+00:00 +url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz" +size = 30594 + +[packages.sdist.hashes] +sha256 = "25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88" + +[[packages.wheels]] +name = "sortedcontainers-2.4.0-py2.py3-none-any.whl" +upload-time = 2021-05-16 22:03:41.177807+00:00 +url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl" +size = 29575 + +[packages.wheels.hashes] +sha256 = "a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" + +[[packages]] +name = "spdx-tools" +version = "0.8.5" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.10" +dependencies = [ + { name = "beartype" }, + { name = "click" }, + { name = "license-expression" }, + { name = "ply" }, + { name = "pyyaml" }, + { name = "rdflib" }, + { name = "semantic-version" }, + { name = "uritools" }, + { name = "xmltodict" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "spdx_tools-0.8.5.tar.gz" +upload-time = 2026-03-13 09:29:23.155398+00:00 +url = "https://files.pythonhosted.org/packages/2f/99/33383f587b59cbd191cc1c0fd5408e550b9275e0090d32561cbddc973fdc/spdx_tools-0.8.5.tar.gz" +size = 696323 + +[packages.sdist.hashes] +sha256 = "be600beb2f762f0116025e05490d399e724f668bef84025a7c421bb266688bdb" + +[[packages.wheels]] +name = "spdx_tools-0.8.5-py3-none-any.whl" +upload-time = 2026-03-13 09:29:20.907842+00:00 +url = "https://files.pythonhosted.org/packages/88/f5/1ac402e1f8d253fa60ac167c4a1fb7eeb0f29a04942788e2d43c984cd749/spdx_tools-0.8.5-py3-none-any.whl" +size = 286573 + +[packages.wheels.hashes] +sha256 = "7c2d5865941be9d2e898f5b084e8d5422dd298dc5a29320ddb198fec304f59c4" + +[[packages]] +name = "tomlkit" +version = "0.15.2" +marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "tomlkit-0.15.2.tar.gz" +upload-time = 2026-10-16 04:07:08.166827+00:00 +url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz" +size = 190522 + +[packages.sdist.hashes] +sha256 = "83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6" + +[[packages.wheels]] +name = "tomlkit-0.15.2-py3-none-any.whl" +upload-time = 2026-10-16 04:07:09.216014+00:00 +url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl" +size = 52225 + +[packages.wheels.hashes] +sha256 = "05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437" + +[[packages]] +name = "towncrier" +version = "26.9.0" +marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.9" +dependencies = [ + { name = "click" }, + { name = "jinja2" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "towncrier-26.9.0.tar.gz" +upload-time = 2026-09-04 12:57:25.341701+00:00 +url = "https://files.pythonhosted.org/packages/92/d4/e85ce614bf45b1a7fe13b82cc687819c2eaac5a1dd94b84834322095fd98/towncrier-26.9.0.tar.gz" +size = 79338 + +[packages.sdist.hashes] +sha256 = "ace9031631c718cc0709107b8755e6e31350eb8acdc877af43df3177addb9a55" + +[[packages.wheels]] +name = "towncrier-26.9.0-py3-none-any.whl" +upload-time = 2026-09-04 12:57:23.986160+00:00 +url = "https://files.pythonhosted.org/packages/82/88/f9340b545dafa64de053798e760515a2736fbb60092cf9aa6b146ccf3ede/towncrier-26.9.0-py3-none-any.whl" +size = 67030 + +[packages.wheels.hashes] +sha256 = "ae4d223e6aadaff98d29b7f09e58d9c9ccf4cd3b455a2d2e0486d432cd8bf660" + +[[packages]] +name = "trove-classifiers" +version = "2026.9.21.13" +marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "trove_classifiers-2026.9.21.13.tar.gz" +upload-time = 2026-09-21 13:29:07.301512+00:00 +url = "https://files.pythonhosted.org/packages/bf/93/af436dfaa845cab5d96f0adbc1e4f3730532d37fa249e4eb796fb1d7fc82/trove_classifiers-2026.9.21.13.tar.gz" +size = 17085 + +[packages.sdist.hashes] +sha256 = "0a9ebc8d4e2f3e8a22848c5258033035bec17a3012ac3fea16dbaa764489eb71" + +[[packages.wheels]] +name = "trove_classifiers-2026.9.21.13-py3-none-any.whl" +upload-time = 2026-09-21 13:29:06.123578+00:00 +url = "https://files.pythonhosted.org/packages/30/81/0da8afb52a71d0a4f2bd3152357b1a441e393b286374802b9d3addab4ab5/trove_classifiers-2026.9.21.13-py3-none-any.whl" +size = 14232 + +[packages.wheels.hashes] +sha256 = "8b1ff4f9c191b1040b71c37f1e445ab99732911e3cd91de52838453a854d7a17" + +[[packages]] +name = "twine" +version = "7.0.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "id" }, + { name = "keyring" }, + { name = "packaging" }, + { name = "readme-renderer" }, + { name = "requests" }, + { name = "requests-toolbelt" }, + { name = "rfc3986" }, + { name = "rich" }, + { name = "urllib3" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "twine-7.0.0.tar.gz" +upload-time = 2026-07-27 15:59:00.825945+00:00 +url = "https://files.pythonhosted.org/packages/92/3c/58f808a359700f39a967dffede33efeac809262c03303fa3eec6afff8f49/twine-7.0.0.tar.gz" +size = 215032 + +[packages.sdist.hashes] +sha256 = "85cdb29c518efef867360ae4acd4b0dfd61c8654a22fca08e6f8539f05022177" + +[[packages.wheels]] +name = "twine-7.0.0-py3-none-any.whl" +upload-time = 2026-07-27 15:58:59.260418+00:00 +url = "https://files.pythonhosted.org/packages/96/08/ddcdc06225eaad6de0e48e1002b06d919dbde20582d0662c7af51308e5d6/twine-7.0.0-py3-none-any.whl" +size = 43204 + +[packages.wheels.hashes] +sha256 = "b854164df26db268af05f49aa5c0344b10e27a494343ff05b1e0bad3b135f5a7" + +[[packages]] +name = "ty" +version = "0.0.95" +marker = "\"type\" in dependency_groups" +requires-python = ">=3.8" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "ty-0.0.95.tar.gz" +upload-time = 2026-10-29 19:21:51.955948+00:00 +url = "https://files.pythonhosted.org/packages/39/08/d63954788578b9189dbf85081d8cbdba9a32a9a6a29729129a0da09cd3a0/ty-0.0.95.tar.gz" +size = 8115525 + +[packages.sdist.hashes] +sha256 = "a480853a74d43074b2d4b28025d406b9d7223f7fe9f3a6c2adec3eacdbf14ca0" + +[[packages.wheels]] +name = "ty-0.0.95-py3-none-macosx_11_0_arm64.whl" +upload-time = 2026-10-29 19:21:16.734602+00:00 +url = "https://files.pythonhosted.org/packages/dc/6b/2a084a72c8069f8496fcc0f576ec053f648b1baeed07532b3eb84c4982e4/ty-0.0.95-py3-none-macosx_11_0_arm64.whl" +size = 13144133 + +[packages.wheels.hashes] +sha256 = "3e7902477410e897a60d2cf29ca89c28ef2a2abdb81698d8ae4be2e33b934d4b" + +[[packages.wheels]] +name = "ty-0.0.95-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +upload-time = 2026-10-29 19:21:30.801674+00:00 +url = "https://files.pythonhosted.org/packages/90/64/636308717989afa2fe765604ae2a76189f470d566e5e58da4482b625d64e/ty-0.0.95-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +size = 16312046 + +[packages.wheels.hashes] +sha256 = "6d47be2db5db80cd70c582b240a71dd8e9be215ce9288298c863c7c3611ef02b" + +[[packages.wheels]] +name = "ty-0.0.95-py3-none-win_amd64.whl" +upload-time = 2026-10-29 19:21:47.462829+00:00 +url = "https://files.pythonhosted.org/packages/b2/df/46fdcc84de81159e1d40ca103d24ac08b2450a2a51a1572333e8e796b8fc/ty-0.0.95-py3-none-win_amd64.whl" +size = 14097073 + +[packages.wheels.hashes] +sha256 = "d07bd09f8a525759a595e04051e923aa54f8dc483f90e6a0b003d77a3fae0f04" + +[[packages]] +name = "typing-extensions" +version = "4.16.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "typing_extensions-4.16.0.tar.gz" +upload-time = 2026-07-02 08:40:05.920538+00:00 +url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz" +size = 113555 + +[packages.sdist.hashes] +sha256 = "dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5" + +[[packages.wheels]] +name = "typing_extensions-4.16.0-py3-none-any.whl" +upload-time = 2026-07-02 08:40:04.659120+00:00 +url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl" +size = 45571 + +[packages.wheels.hashes] +sha256 = "481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8" + +[[packages]] +name = "typing-inspection" +version = "0.4.4" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "typing-extensions" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "typing_inspection-0.4.4.tar.gz" +upload-time = 2026-08-12 12:37:25.997329+00:00 +url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz" +size = 76928 + +[packages.sdist.hashes] +sha256 = "547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47" + +[[packages.wheels]] +name = "typing_inspection-0.4.4-py3-none-any.whl" +upload-time = 2026-08-12 12:37:24.648475+00:00 +url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl" +size = 14750 + +[packages.wheels.hashes] +sha256 = "65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147" + +[[packages]] +name = "tzdata" +version = "2026.6" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=2" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "tzdata-2026.6.tar.gz" +upload-time = 2026-10-28 09:57:09.137703+00:00 +url = "https://files.pythonhosted.org/packages/7a/29/6a1391d31cd00ff9e927215a224a880e1e84e826b64ba80a499e51c074d8/tzdata-2026.6.tar.gz" +size = 201017 + +[packages.sdist.hashes] +sha256 = "85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d" + +[[packages.wheels]] +name = "tzdata-2026.6-py2.py3-none-any.whl" +upload-time = 2026-10-28 09:57:07.276552+00:00 +url = "https://files.pythonhosted.org/packages/0a/66/9a06c2227fedfd4919f0eaaa8b94377d9730882a7e4f95823c9a331ab5e1/tzdata-2026.6-py2.py3-none-any.whl" +size = 348210 + +[packages.wheels.hashes] +sha256 = "0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec" + +[[packages]] +name = "uri-template" +version = "1.3.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "uri-template-1.3.0.tar.gz" +upload-time = 2023-06-21 01:49:05.374755+00:00 +url = "https://files.pythonhosted.org/packages/31/c7/0336f2bd0bcbada6ccef7aaa25e443c118a704f828a0620c6fa0207c1b64/uri-template-1.3.0.tar.gz" +size = 21678 + +[packages.sdist.hashes] +sha256 = "0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7" + +[[packages.wheels]] +name = "uri_template-1.3.0-py3-none-any.whl" +upload-time = 2023-06-21 01:49:03.467606+00:00 +url = "https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl" +size = 11140 + +[packages.wheels.hashes] +sha256 = "a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363" + +[[packages]] +name = "uritools" +version = "6.1.3" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "uritools-6.1.3.tar.gz" +upload-time = 2026-07-23 23:17:44.612642+00:00 +url = "https://files.pythonhosted.org/packages/a3/0d/20d02264b6682f07e92cbf7ee43e5e803670d101a03ef204ba18368c321f/uritools-6.1.3.tar.gz" +size = 23909 + +[packages.sdist.hashes] +sha256 = "3a498e7e85ef3249343d5710618d641a414da0fbae6d23053ada7976ee83ea5f" + +[[packages.wheels]] +name = "uritools-6.1.3-py3-none-any.whl" +upload-time = 2026-07-23 23:17:43.275613+00:00 +url = "https://files.pythonhosted.org/packages/ac/76/e9bb35862bcc5b82218d2c017b1eacdcfeb3ba46ce0c0ca8ffc4db80ddbe/uritools-6.1.3-py3-none-any.whl" +size = 11981 + +[packages.wheels.hashes] +sha256 = "136f113e76e53f85bf2d9cce0c3d40ceec775d0409e7d6de9b28f046a7d42838" + +[[packages]] +name = "urllib3" +version = "2.8.0" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "urllib3-2.8.0.tar.gz" +upload-time = 2026-09-15 19:29:36.253420+00:00 +url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz" +size = 458972 + +[packages.sdist.hashes] +sha256 = "63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63" + +[[packages.wheels]] +name = "urllib3-2.8.0-py3-none-any.whl" +upload-time = 2026-09-15 19:29:34.577402+00:00 +url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl" +size = 135717 + +[packages.wheels.hashes] +sha256 = "0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3" + +[[packages]] +name = "uv" +version = "0.13.6" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.8" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "uv-0.13.6.tar.gz" +upload-time = 2026-10-27 00:45:57.605614+00:00 +url = "https://files.pythonhosted.org/packages/08/d5/88d43310d425f6e3a43997ad771658ded3237d0ed78c16a89b2353eddf56/uv-0.13.6.tar.gz" +size = 7545242 + +[packages.sdist.hashes] +sha256 = "d762215478854f6cf20c3faed8f1eb30c8101d568e7bc32a3abafc063929d5c5" + +[[packages.wheels]] +name = "uv-0.13.6-py3-none-macosx_11_0_arm64.whl" +upload-time = 2026-10-27 00:45:09.125609+00:00 +url = "https://files.pythonhosted.org/packages/f4/14/77230cc4f6a73a31578cb8070a5f67f58a76946ead643ae8a4155620d6da/uv-0.13.6-py3-none-macosx_11_0_arm64.whl" +size = 14344205 + +[packages.wheels.hashes] +sha256 = "51361029b7cf91185d0092c4b356d110b4ad629e9cb11593f1e006ac72a72ca9" + +[[packages.wheels]] +name = "uv-0.13.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +upload-time = 2026-10-27 00:45:31.291019+00:00 +url = "https://files.pythonhosted.org/packages/5c/43/0688ff5142934238f2812e348f4a24e09b57d48bfb38dd8f850325a0d9de/uv-0.13.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +size = 17333810 + +[packages.wheels.hashes] +sha256 = "22759b8895e1103578de6fb4cd27e94b4399d021023378d29f01333717751fba" + +[[packages.wheels]] +name = "uv-0.13.6-py3-none-win_amd64.whl" +upload-time = 2026-10-27 00:45:52.086199+00:00 +url = "https://files.pythonhosted.org/packages/7c/9d/714523188aea3054644d8e69ae8089aac70325a3cee2d3d05509d4f23398/uv-0.13.6-py3-none-win_amd64.whl" +size = 15334118 + +[packages.wheels.hashes] +sha256 = "7c9bc35ccb65932d8dc10a0669f9c805eaa3c9827ce18ecbecb1bd67944b2cdf" + +[[packages]] +name = "vcs-versioning" +version = "2.6.0" +marker = "\"build\" in dependency_groups" +requires-python = ">=3.8" +dependencies = [ + { name = "packaging" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "vcs_versioning-2.6.0.tar.gz" +upload-time = 2026-10-06 22:21:55.024903+00:00 +url = "https://files.pythonhosted.org/packages/ad/e7/2a691db7d076b75d099fdba1a14cd966931f2aeb591e77f6d22f15097326/vcs_versioning-2.6.0.tar.gz" +size = 166800 + +[packages.sdist.hashes] +sha256 = "22e9159288e2d8bca2fa6f6c31c34c9c66aec157f60e9e526f5d160d2516d60f" + +[[packages.wheels]] +name = "vcs_versioning-2.6.0-py3-none-any.whl" +upload-time = 2026-10-06 22:21:53.267565+00:00 +url = "https://files.pythonhosted.org/packages/73/1d/ad4064bf12f4503bf61e65d74c93c523672776d25ee0a06aaf42ae09ef3f/vcs_versioning-2.6.0-py3-none-any.whl" +size = 123154 + +[packages.wheels.hashes] +sha256 = "f8bebd1302ebc12e67ceb30fc7dd698ec969414b6fa43b30c3f14ac9b1e72c41" + +[[packages]] +name = "wcwidth" +version = "0.9.4" +marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "wcwidth-0.9.4.tar.gz" +upload-time = 2026-10-24 18:11:44.954051+00:00 +url = "https://files.pythonhosted.org/packages/ea/01/e5ac23733628a6bbb91913a765fcf55ee8070d8d991bc88fdaaf805052c7/wcwidth-0.9.4.tar.gz" +size = 970051 + +[packages.sdist.hashes] +sha256 = "fd8d57d3cdfe570e3265c8fab1885c24e34ef40889c0759c0c304a19e9835954" + +[[packages.wheels]] +name = "wcwidth-0.9.4-cp310-abi3-macosx_11_0_arm64.whl" +upload-time = 2026-10-24 18:11:02.008720+00:00 +url = "https://files.pythonhosted.org/packages/04/e9/dfdca30ca36c472de3e260406dee29fc108499ec590c63b3fe279887ee9d/wcwidth-0.9.4-cp310-abi3-macosx_11_0_arm64.whl" +size = 618904 + +[packages.wheels.hashes] +sha256 = "f279f32c8e206a9092fa350e53ca0c2bd8072e45b1c9fd31bb6d476b59b591c4" + +[[packages.wheels]] +name = "wcwidth-0.9.4-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +upload-time = 2026-10-24 18:11:18.748328+00:00 +url = "https://files.pythonhosted.org/packages/fa/e8/0485850748dfdba9443f4151c7b2442cfa4987ba6cd871038eb5d6b8eca9/wcwidth-0.9.4-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" +size = 785946 + +[packages.wheels.hashes] +sha256 = "b185c8b927b23af2721be1783499d1cfa5c028b098b0858951b4ae224477f1c2" + +[[packages.wheels]] +name = "wcwidth-0.9.4-cp310-abi3-win_amd64.whl" +upload-time = 2026-10-24 18:11:36.952282+00:00 +url = "https://files.pythonhosted.org/packages/d6/f1/c123f902bc2ff9e9c2ab66241b8872204df87cbb18ecdf035e120860fedc/wcwidth-0.9.4-cp310-abi3-win_amd64.whl" +size = 597780 + +[packages.wheels.hashes] +sha256 = "f28d77153303450ee13ebbe888297e75cf532c754555b6cabafc9bdda16c421c" + +[[packages.wheels]] +name = "wcwidth-0.9.4-py3-none-any.whl" +upload-time = 2026-10-24 18:10:52.651030+00:00 +url = "https://files.pythonhosted.org/packages/6f/0c/309b1813b230d1dbbd6f2c2ab2bb21ef68992358977605d9ef289bbfedcc/wcwidth-0.9.4-py3-none-any.whl" +size = 301806 + +[packages.wheels.hashes] +sha256 = "901eace6d2dd25ebb917d0ce5db18e5c537e1504c905cf32f0d006c12098b05d" + +[[packages]] +name = "webcolors" +version = "25.10.0" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "webcolors-25.10.0.tar.gz" +upload-time = 2025-10-31 07:51:03.977878+00:00 +url = "https://files.pythonhosted.org/packages/1d/7a/eb316761ec35664ea5174709a68bbd3389de60d4a1ebab8808bfc264ed67/webcolors-25.10.0.tar.gz" +size = 53491 + +[packages.sdist.hashes] +sha256 = "62abae86504f66d0f6364c2a8520de4a0c47b80c03fc3a5f1815fedbef7c19bf" + +[[packages.wheels]] +name = "webcolors-25.10.0-py3-none-any.whl" +upload-time = 2025-10-31 07:51:01.778455+00:00 +url = "https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl" +size = 14905 + +[packages.wheels.hashes] +sha256 = "032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d" + +[[packages]] +name = "wheel-filename" +version = "1.4.2" +marker = "\"pkg-meta\" in dependency_groups" +requires-python = ">=3.8" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "wheel_filename-1.4.2.tar.gz" +upload-time = 2024-12-01 13:03:16.012739+00:00 +url = "https://files.pythonhosted.org/packages/38/be/726dab762b770d0417e505c58e26d661aac1ec0c831e483cda4817ca2417/wheel_filename-1.4.2.tar.gz" +size = 7911 + +[packages.sdist.hashes] +sha256 = "87891c465dcbb40b40394a906f01a93214bdd51aa5d25e3a9a59cae62bc298fd" + +[[packages.wheels]] +name = "wheel_filename-1.4.2-py3-none-any.whl" +upload-time = 2024-12-01 13:03:00.536117+00:00 +url = "https://files.pythonhosted.org/packages/b4/0f/6e97a3bc38cdde32e3ec49f8c0903fe3559ec9ec9db181782f0bb4417717/wheel_filename-1.4.2-py3-none-any.whl" +size = 6195 + +[packages.wheels.hashes] +sha256 = "3fa599046443d4ca830d06e3d180cd0a675d5871af0a68daa5623318bb4d17e3" + +[[packages]] +name = "xmltodict" +version = "1.0.4" +marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "xmltodict-1.0.4.tar.gz" +upload-time = 2026-02-22 02:21:22.074263+00:00 +url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz" +size = 26124 + +[packages.sdist.hashes] +sha256 = "6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61" + +[[packages.wheels]] +name = "xmltodict-1.0.4-py3-none-any.whl" +upload-time = 2026-02-22 02:21:21.039836+00:00 +url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl" +size = 13580 + +[packages.wheels.hashes] +sha256 = "a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a" + +[tool.nab] +nab-version = "0.0.23" +created-at = 2026-11-07 02:33:11.469337+00:00 +command-line = [ + "nab", + "lock", + "--all-groups", + "--all-extras", + "--no-cache", + "--upgrade", +] +input-path = "pyproject.toml" +mode = "universal" +python-specifier = ">=3.15,<3.16" +platforms = [ + "linux_x86_64", + "macos_arm64", + "windows_amd64", +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ead9044 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,210 @@ +[build-system] +build-backend = "hatchling.build" +requires = [ + "hatch-vcs>=0.5", + "hatchling>=1.32.4", +] + +[project] +name = "hatch-pep770" +import-names = [ "hatch_pep770" ] +description = "Hatchling build hook that embeds a PEP 770 CycloneDX SBOM in the wheel." +readme = "README.md" +keywords = [ + "cyclonedx", + "hatch", + "pep770", + "sbom", + "spdx", +] +license = "MIT" +maintainers = [ + { name = "Bernรกt Gรกbor", email = "gaborjbernat@gmail.com" }, +] +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 4 - Beta", + "Framework :: Hatch", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Topic :: Security", + "Topic :: Software Development :: Build Tools", +] +dynamic = [ + "version", +] +dependencies = [ + "hatchling>=1.27", + "packaging>=24.2", +] +optional-dependencies.validate = [ + "cyclonedx-python-lib[json-validation]>=11", + "spdx-tools>=0.8.3", +] +urls.Changelog = "https://github.com/tox-dev/hatch-pep770/blob/main/CHANGELOG.md" +urls.Documentation = "https://github.com/tox-dev/hatch-pep770#hatch-pep770" +urls.Homepage = "https://github.com/tox-dev/hatch-pep770" +urls.Source = "https://github.com/tox-dev/hatch-pep770" +urls.Tracker = "https://github.com/tox-dev/hatch-pep770/issues" +scripts.hatch-pep770 = "hatch_pep770.__main__:main" +entry-points.hatch.pep770 = "hatch_pep770.hooks" + +[dependency-groups] +test = [ + "covdefaults>=2.3", + "cyclonedx-python-lib[json-validation]>=11", + "pytest>=8.4.2", + "pytest-cov>=7.1", + "pytest-mock>=3.15.1", + "spdx-tools>=0.8.3", +] +type = [ + "ty>=0.0.84", + { include-group = "test" }, +] +build = [ + "hatch-vcs>=0.5", + "hatchling>=1.32.4", +] +pkg-meta = [ + "check-wheel-contents>=0.6.3", + "twine>=7", + "uv>=0.12.22", +] +release = [ + "prek>=0.5", + "release-creator>=4.4", + "towncrier>=26.9", +] +release-news = [ + "release-creator>=4.4", + "towncrier>=26.9", +] +release-notes = [ + "release-creator>=4.4", +] +release-prepare = [ + "prek>=0.5", + "release-creator>=4.4", + "towncrier>=26.9", +] + +[tool.hatch] +version.source = "vcs" +build.hooks.vcs.version-file = "src/hatch_pep770/version.py" +build.targets.sdist.include = [ + "/src", + "/tests", +] + +[tool.uv] +dependency-groups.release = { requires-python = ">=3.15" } +dependency-groups.release-news = { requires-python = ">=3.15" } +dependency-groups.release-notes = { requires-python = ">=3.15" } +dependency-groups.release-prepare = { requires-python = ">=3.15" } + +[tool.ruff] +line-length = 120 +format.preview = true +format.docstring-code-line-length = 100 +format.docstring-code-format = true +lint.select = [ + "ALL", +] +lint.ignore = [ + "CPY", # No copyright statements + "D1", # docstrings only on the public API, enforced in review + "DOC", # not supported + "incorrect-blank-line-before-class", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible + "missing-trailing-comma", # Conflict with formatter + "multi-line-summary-first-line", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible + "single-line-implicit-string-concatenation", # Conflict with formatter + "subprocess-without-shell-equals-true", # subprocess calls are fine + "suspicious-subprocess-import", # Using subprocess is alright +] +lint.per-file-ignores."tests/**/*.py" = [ + "assert", # asserts allowed in tests + "D", # don't care about documentation in tests + "FBT", # don't care about booleans as positional arguments in tests + "implicit-namespace-package", # no implicit namespace + "magic-value-comparison", # Magic value used in comparison, consider replacing with a constant variable + "pytest-fixture-autouse", # shared setup fixtures are meant to apply to every test + "start-process-with-partial-path", # partial executable path is fine in tests + "subprocess-without-shell-equals-true", # `subprocess` call: check for execution of untrusted input + "too-many-arguments", # fixtures arrive as arguments + "too-many-positional-arguments", # fixtures arrive as arguments +] +lint.isort = { known-first-party = [ + "hatch_pep770", + "tests", +], required-imports = [ + "from __future__ import annotations", +] } +lint.preview = true + +[tool.codespell] +builtin = "clear,usage,en-GB_to_en-US" +ignore-words-list = "acknowledgement" +write-changes = true +count = true + +[tool.pyproject-fmt] +max_supported_python = "3.15" + +[tool.ty] +environment.python-version = "3.15" + +[tool.coverage] +run.parallel = true +run.plugins = [ + "covdefaults", +] +paths.source = [ + "src", + ".tox*/*/lib/python*/site-packages", + ".tox*/pypy*/site-packages", + ".tox*\\*\\Lib\\site-packages", + "*/src", + "*\\src", +] +html.show_contexts = true +html.skip_covered = false + +[tool.towncrier] +name = "hatch-pep770" +directory = "changelog.d" +filename = "CHANGELOG.md" +template = "release_creator.fragments:template.jinja2" +title_format = false +type = [ + { directory = "feature", name = "Added", showcontent = true }, + { directory = "change", name = "Changes", showcontent = true }, + { directory = "bugfix", name = "Fixed", showcontent = true }, + { directory = "doc", name = "Documentation", showcontent = true }, + { directory = "breaking", name = "Breaking changes", showcontent = true }, + { directory = "security", name = "Security", showcontent = true }, + { directory = "performance", name = "Performance", showcontent = true }, + { directory = "deprecation", name = "Deprecated", showcontent = true }, + { directory = "removal", name = "Removed", showcontent = true }, + { directory = "packaging", name = "Packaging", showcontent = true }, +] + +[tool.nab] +mode = "universal" +base-group = "project-runtime" +uploaded-prior-to = "P7D" +decision-order = "stable" +matrix.python = ">=3.15,<3.16" +matrix.platforms = [ "linux_x86_64", "macos_arm64", "windows_amd64" ] +# release-creator 4.4.0 needs a seven-day cutoff exception. +# expires 2026-11-09 +packages.release-creator.uploaded-prior-to = 2026-11-02T21:45:00Z diff --git a/tox.toml b/tox.toml new file mode 100644 index 0000000..bcb5f79 --- /dev/null +++ b/tox.toml @@ -0,0 +1,72 @@ +requires = [ "tox>=4.64.7", "tox-uv>=1.36.1" ] +env_list = [ "3.15", "3.14", "3.13", "3.12", "3.11", "3.10", "3.9", "3.15t", "fix", "pkg_meta", "type" ] +skip_missing_interpreters = true + +[env_run_base] +description = "run the tests with pytest under {env_name}" +default_base_python = [ "3.15" ] +package = "wheel" +wheel_build_env = ".pkg" +dependency_groups = [ "test" ] +pass_env = [ "PYTEST_*" ] +set_env.COVERAGE_FILE = { replace = "env", name = "COVERAGE_FILE", default = "{work_dir}{/}.coverage.{env_name}" } +commands = [ + [ + "pytest", + { replace = "posargs", default = [ + "--durations", + "5", + "--junitxml", + "{work_dir}{/}junit.{env_name}.xml", + "--no-cov-on-fail", + "--cov", + "{env_site_packages_dir}{/}hatch_pep770", + "--cov", + "{tox_root}{/}tests", + "--cov-config", + "{tox_root}{/}pyproject.toml", + "--cov-context", + "test", + "--cov-report", + "term-missing:skip-covered", + "--cov-report", + "html:{env_tmp_dir}{/}htmlcov", + "--cov-report", + "xml:{work_dir}{/}coverage.{env_name}.xml", + "--cov-fail-under=100", + "tests", + ], extend = true }, + ], +] + +[env_pkg_base] +default_base_python = [ "3.15" ] + +[env.fix] +description = "format the code base to adhere to our styles, and complain about what we cannot do automatically" +skip_install = true +deps = [ "pre-commit-uv>=4.3" ] +pass_env = [ { replace = "ref", of = [ "env_run_base", "pass_env" ], extend = true }, "PROGRAMDATA" ] +commands = [ [ "pre-commit", "run", "--all-files", "--show-diff-on-failure", { replace = "posargs", extend = true } ] ] + +[env.pkg_meta] +description = "check that the long description is valid" +skip_install = true +dependency_groups = [ "pkg-meta" ] +commands = [ + [ "uv", "build", "--sdist", "--wheel", "--out-dir", "{env_tmp_dir}", "." ], + [ "twine", "check", "{env_tmp_dir}{/}*" ], + [ "check-wheel-contents", "--no-config", "{env_tmp_dir}" ], +] + +[env.type] +description = "run type check on code base" +dependency_groups = [ "type" ] +commands = [ [ "ty", "check", "--output-format", "concise", "--error-on-warning", "." ] ] + +[env.dev] +description = "dev environment with all deps at {envdir}" +package = "editable" +deps = [ { replace = "ref", of = [ "requires" ], extend = true } ] +dependency_groups = [ "release", "type" ] +commands = [ [ "uv", "pip", "tree" ], [ "python", "-c", 'print(r"{env_python}")' ] ] From 5c0a18b8bd161db0ad7d1e44777d66ca4c7466e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 18:35:03 -0800 Subject: [PATCH 002/115] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20the=20PEP=207?= =?UTF-8?q?70=20SBOM=20build=20hook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the CycloneDX SBOM generator from pypa/virtualenv's hatch_build.py into a reusable hatchling plugin, with the project-specific parts (supplier, repository, bundled wheels) moved into hook options and a subclass method for per-wheel properties. The SPDX rendering and the validator ship as the hatch-pep770 spdx and validate commands. --- README.md | 117 +++- changelog.d/+pep770-hook.feature.md | 2 + src/hatch_pep770/__init__.py | 1 + src/hatch_pep770/__main__.py | 62 +++ src/hatch_pep770/cyclonedx.py | 487 ++++++++++++++++ src/hatch_pep770/hooks.py | 17 + src/hatch_pep770/plugin.py | 453 +++++++++++++++ src/hatch_pep770/spdx.py | 213 +++++++ src/hatch_pep770/validate.py | 238 ++++++++ tests/conftest.py | 137 +++++ tests/test___main__.py | 62 +++ tests/test_plugin.py | 829 ++++++++++++++++++++++++++++ tests/test_spdx.py | 205 +++++++ tests/test_validate.py | 223 ++++++++ 14 files changed, 3044 insertions(+), 2 deletions(-) create mode 100644 changelog.d/+pep770-hook.feature.md create mode 100644 src/hatch_pep770/__init__.py create mode 100644 src/hatch_pep770/__main__.py create mode 100644 src/hatch_pep770/cyclonedx.py create mode 100644 src/hatch_pep770/hooks.py create mode 100644 src/hatch_pep770/plugin.py create mode 100644 src/hatch_pep770/spdx.py create mode 100644 src/hatch_pep770/validate.py create mode 100644 tests/conftest.py create mode 100644 tests/test___main__.py create mode 100644 tests/test_plugin.py create mode 100644 tests/test_spdx.py create mode 100644 tests/test_validate.py diff --git a/README.md b/README.md index ed8faf6..13319d8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,117 @@ # hatch-pep770 -A [hatchling](https://hatch.pypa.io/latest/) build hook that writes a [CycloneDX](https://cyclonedx.org/) SBOM into -every wheel, where [PEP 770](https://peps.python.org/pep-0770/) says installers and scanners look for it. +[![PyPI](https://img.shields.io/pypi/v/hatch-pep770?style=flat-square)](https://pypi.org/project/hatch-pep770/) +[![Supported Python versions](https://img.shields.io/pypi/pyversions/hatch-pep770.svg)](https://pypi.org/project/hatch-pep770/) +[![check](https://github.com/tox-dev/hatch-pep770/actions/workflows/check.yaml/badge.svg)](https://github.com/tox-dev/hatch-pep770/actions/workflows/check.yaml) + +A [hatchling](https://hatch.pypa.io/latest/) build hook that writes a [CycloneDX](https://cyclonedx.org/) 1.6 SBOM into +every wheel at `-.dist-info/sboms/.cdx.json`, where [PEP 770](https://peps.python.org/pep-0770/) +says installers and scanners look for it. + +General-purpose SBOM scanners read installed environments or declared dependencies. They cannot see a wheel your project +bundles as a data file, and they say nothing about how a wheel was built. The SBOM this hook writes records: + +- the project, from its core metadata: version, license, maintainers, project URLs, classifiers, and the source revision + when the build runs in a git checkout; +- the runtime dependencies the wheel declares, unversioned, since the installer resolves them; +- every wheel the project bundles as a data file, with its SHA-256, license, files, and the packages it vendors; +- the build toolchain: the Python version, and every distribution in the build environment with its dependencies and, + for pure-Python ones, the hash of each file. + +It records nothing about the build machine and takes its timestamp from `SOURCE_DATE_EPOCH`, so rebuilding the same +source with the same toolchain and epoch reproduces the wheel byte for byte on any operating system or architecture. + +## Use it + +Add the hook to the build requirements and enable it: + +```toml +[build-system] +build-backend = "hatchling.build" +requires = ["hatch-pep770>=1", "hatchling>=1.28"] + +[tool.hatch.build.hooks.pep770] +``` + +hatchling 1.28 added SBOM support to wheels. With an older hatchling the hook prints a warning and the wheel carries no +SBOM, so a project that still builds on Python 3.9 (where hatchling stops at 1.27) can keep the hook enabled. + +The hook only acts on the wheel target. Put it after any hook that sets the wheel tag, since the SBOM names the wheel +file in its attestation link. + +## Options + +All options are optional. + +| Option | Default | Meaning | +| ---------------- | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `supplier` | none | `{ name = "...", url = ["..."] }`, the organization that supplies the project; it becomes the SBOM's supplier and manufacturer and the project's supplier. | +| `repository` | the first `Source`, `Repository` or `GitHub` project URL | Repository URL for the source revision, license and security policy links. | +| `default-branch` | `main` | Branch used in links to files in the repository. | +| `bundled-wheels` | `[]` | Glob patterns, relative to the project root, of wheels the project ships as data files. | + +For example, a project that ships the wheels it installs into new environments: + +```toml +[tool.hatch.build.hooks.pep770] +supplier = { name = "Python Packaging Authority", url = [ + "https://www.pypa.io", +] } +bundled-wheels = ["src/virtualenv/seed/wheels/embed/*.whl"] +``` + +A bundled wheel whose project URLs point to the same forge account as `repository` gets the same `supplier`. Its +vendored packages come from `_vendor/vendor.txt` (pip's manifest, whose entries must pin one exact version) and from any +`.dist-info/METADATA` under `_vendor/`. + +Property names the hook writes start with the project name, for example `virtualenv:bundled-wheel`. + +## Add project-specific detail + +To say more about each bundled wheel, subclass the hook in a +[custom build hook](https://hatch.pypa.io/latest/plugins/build-hook/custom/) and override `bundled_wheel_properties`. +Its options then live under `[tool.hatch.build.hooks.custom]`, and the SBOM lists the script among the build tools: + +```python +# hatch_build.py +from __future__ import annotations + +from pathlib import Path + +from hatch_pep770 import cyclonedx, plugin + + +class SbomBuildHook(plugin.Pep770BuildHook): + def bundled_wheel_properties(self, wheel: Path) -> list[cyclonedx.Property]: + return [{"name": "example:installed-for-python", "value": "3.14"}] + + +def get_build_hook() -> type[SbomBuildHook]: + return SbomBuildHook +``` + +```toml +[tool.hatch.build.hooks.custom] +path = "hatch_build.py" +bundled-wheels = ["src/example/wheels/*.whl"] +``` + +The `get_build_hook` function tells hatchling which class to load, since the module also sees the base class. +`hatch_pep770.cyclonedx` holds the pieces the hook is built from (`build_tools`, `component_from_metadata`, `document`) +for describing other artifacts, such as a zipapp, the same way. + +## Command line + +```shell +hatch-pep770 spdx dist/example.cdx.json example.spdx.json +hatch-pep770 validate dist/ --allow-license MIT --allow-license Apache-2.0 +``` + +`spdx` renders an SBOM as [SPDX](https://spdx.dev/) 2.3 JSON for tools that read only SPDX. It keeps packages, hashes, +licenses, purls and the relationships, and leaves out per-file hashes (SPDX 2.3 wants a SHA-1 per file, wheel `RECORD` +files hold SHA-256) and the build formulation, which SPDX 2.3 has no field for. + +`validate` needs the `validate` extra (`pip install hatch-pep770[validate]`). For each wheel it checks that an SBOM +exists, conforms to the CycloneDX 1.6 schema, carries the `urn:uuid` serial number `actions/attest` needs to recognize +it, has unique and resolvable references, and renders as valid SPDX. With `--allow-license`, every bundled component +must also carry a license the given SPDX ids allow. diff --git a/changelog.d/+pep770-hook.feature.md b/changelog.d/+pep770-hook.feature.md new file mode 100644 index 0000000..708aa1e --- /dev/null +++ b/changelog.d/+pep770-hook.feature.md @@ -0,0 +1,2 @@ +Add the `pep770` hatchling build hook, which writes a CycloneDX 1.6 SBOM into the wheel at +`.dist-info/sboms/.cdx.json`, and the `hatch-pep770 spdx` and `hatch-pep770 validate` commands. diff --git a/src/hatch_pep770/__init__.py b/src/hatch_pep770/__init__.py new file mode 100644 index 0000000..1aa6c39 --- /dev/null +++ b/src/hatch_pep770/__init__.py @@ -0,0 +1 @@ +"""Embed a PEP 770 CycloneDX SBOM in wheels built with hatchling.""" diff --git a/src/hatch_pep770/__main__.py b/src/hatch_pep770/__main__.py new file mode 100644 index 0000000..e57defc --- /dev/null +++ b/src/hatch_pep770/__main__.py @@ -0,0 +1,62 @@ +"""Command line: render a CycloneDX SBOM as SPDX, or validate the SBOMs in built wheels.""" + +from __future__ import annotations + +import json +import sys +from argparse import ArgumentParser, Namespace +from pathlib import Path + +from .spdx import to_spdx + + +class _Options(Namespace): + command: str + source: Path + target: Path + paths: list[Path] + allow_license: list[str] | None + + +def main(argv: list[str] | None = None) -> int: + parser = ArgumentParser(prog="hatch-pep770", description="Work with the PEP 770 SBOMs hatch-pep770 writes.") + commands = parser.add_subparsers(dest="command", required=True) + spdx = commands.add_parser("spdx", help="render a CycloneDX SBOM as SPDX 2.3 JSON") + spdx.add_argument("source", type=Path, help="the CycloneDX JSON file to read") + spdx.add_argument("target", type=Path, help="the SPDX JSON file to write") + validate = commands.add_parser( + "validate", help="validate the SBOMs in wheels, needs the validate extra: hatch-pep770[validate]" + ) + validate.add_argument("paths", type=Path, nargs="+", help="wheel files, or directories holding them") + validate.add_argument( + "--allow-license", + action="append", + metavar="SPDX-ID", + help="SPDX license id bundled components may use; repeat it per id, leave it out to skip the license policy", + ) + options = parser.parse_args(argv, namespace=_Options()) + if options.command == "spdx": + bom = json.loads(options.source.read_text(encoding="utf-8")) + options.target.write_bytes(f"{json.dumps(to_spdx(bom), indent=2)}\n".encode()) + return 0 + # validate pulls in the optional extra, so the spdx command works without it + from .validate import wheel_problems # ruff: ignore[import-outside-top-level] + + allowed = None if options.allow_license is None else frozenset(options.allow_license) + wheels = sorted(wheel for path in options.paths for wheel in (path.glob("*.whl") if path.is_dir() else [path])) + if not wheels: + sys.stderr.write(f"no wheel found in {', '.join(map(str, options.paths))}\n") + return 1 + failed = False + for wheel in wheels: + for problem in (problems := wheel_problems(wheel, allowed)): + sys.stderr.write(f"SBOM invalid: {wheel.name}: {problem}\n") + if problems: + failed = True + else: + sys.stdout.write(f"SBOM in {wheel.name} is valid\n") + return int(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/hatch_pep770/cyclonedx.py b/src/hatch_pep770/cyclonedx.py new file mode 100644 index 0000000..171b825 --- /dev/null +++ b/src/hatch_pep770/cyclonedx.py @@ -0,0 +1,487 @@ +"""CycloneDX 1.6 building blocks shared by the build hook and by projects that describe other artifacts.""" + +from __future__ import annotations + +import base64 +import json +import platform +import re +import sys +import uuid +from datetime import datetime, timezone +from email.parser import Parser +from email.utils import getaddresses +from importlib.metadata import distributions +from typing import TYPE_CHECKING, Final, TypedDict + +from hatchling.builders.utils import get_reproducible_timestamp +from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression +from packaging.requirements import Requirement + +if TYPE_CHECKING: + from email.message import Message + from importlib.metadata import Distribution, PackageMetadata + + +class OrganizationalEntity(TypedDict): + name: str + url: list[str] + + +class Contact(TypedDict, total=False): + name: str + email: str + + +class Hash(TypedDict): + alg: str + content: str + + +class Property(TypedDict): + name: str + value: str + + +class _ExternalReferenceKeys(TypedDict): + type: str + url: str + + +class ExternalReference(_ExternalReferenceKeys, total=False): + comment: str + + +class _LicenseIdentity(TypedDict, total=False): + id: str + name: str + + +class LicenseInfo(_LicenseIdentity): + acknowledgement: str + + +class License(TypedDict, total=False): + # a CycloneDX license choice holds either a `license` or an `expression` with its `acknowledgement` + license: LicenseInfo + expression: str + acknowledgement: str + + +class _Method(TypedDict): + technique: str + confidence: int + value: str + + +class _Identity(TypedDict): + field: str + confidence: int + methods: list[_Method] + + +class Evidence(TypedDict): + identity: list[_Identity] + + +_ComponentKeys = TypedDict("_ComponentKeys", {"type": str, "bom-ref": str, "name": str}) + + +class _ComponentFields(TypedDict, total=False): + version: str + purl: str + supplier: OrganizationalEntity + authors: list[Contact] + description: str + licenses: list[License] + copyright: str + hashes: list[Hash] + externalReferences: list[ExternalReference] + properties: list[Property] + evidence: Evidence + components: list[Component] + + +class Component(_ComponentKeys, _ComponentFields): + pass + + +class Dependency(TypedDict): + ref: str + dependsOn: list[str] + + +class _Ref(TypedDict): + ref: str + + +class _Output(TypedDict): + type: str + resource: _Ref + + +class _Input(TypedDict): + environmentVars: list[Property] + + +Workflow = TypedDict( + "Workflow", + { + "bom-ref": str, + "uid": str, + "name": str, + "taskTypes": list[str], + "resourceReferences": list[_Ref], + "inputs": list[_Input], + "outputs": list[_Output], + }, +) +Formula = TypedDict("Formula", {"bom-ref": str, "workflows": list[Workflow]}) + + +class _Lifecycle(TypedDict): + phase: str + + +class _Tools(TypedDict): + components: list[Component] + + +class _MetadataKeys(TypedDict): + timestamp: str + lifecycles: list[_Lifecycle] + tools: _Tools + component: Component + + +class Metadata(_MetadataKeys, total=False): + manufacturer: OrganizationalEntity + supplier: OrganizationalEntity + authors: list[Contact] + licenses: list[License] + properties: list[Property] + + +class Composition(TypedDict): + aggregate: str + dependencies: list[str] + + +class _BodyKeys(TypedDict): + metadata: Metadata + components: list[Component] + dependencies: list[Dependency] + formulation: list[Formula] + + +class Body(_BodyKeys, total=False): + compositions: list[Composition] + + +_Header = TypedDict( + "_Header", {"$schema": str, "bomFormat": str, "specVersion": str, "serialNumber": str, "version": int} +) + + +class Document(_Header, Body): + pass + + +# the label normalization and mapping cyclonedx-py applies to Project-URL entries, plus "source", so the same metadata +# yields the same reference types whichever generator produced the document +_URL_LABEL_TO_REFERENCE_TYPE: Final[dict[str, str]] = { + "bugtracker": "issue-tracker", + "issuetracker": "issue-tracker", + "issues": "issue-tracker", + "bugreports": "issue-tracker", + "tracker": "issue-tracker", + "home": "website", + "homepage": "website", + "download": "distribution", + "documentation": "documentation", + "docs": "documentation", + "changelog": "release-notes", + "changes": "release-notes", + "source": "vcs", + "repository": "vcs", + "github": "vcs", + "chat": "chat", +} +# core metadata headers copied verbatim onto a component as properties +_METADATA_PROPERTIES: Final[tuple[str, ...]] = ( + "Requires-Python", + "Requires-Dist", + "Provides-Extra", + "Classifier", + "Keywords", +) +# only the license classifiers that name exactly one SPDX license; "BSD License" or "Apache Software License" leave the +# version or variant open +_CLASSIFIER_LICENSES: Final[dict[str, str]] = { + "ISC License (ISCL)": "ISC", + "MIT License": "MIT", + "MIT No Attribution License (MIT-0)": "MIT-0", + "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", + "Python Software Foundation License": "PSF-2.0", + "The Unlicense (Unlicense)": "Unlicense", +} +_RECORD_HASH_ALGORITHMS: Final[dict[str, str]] = { + "md5": "MD5", + "sha1": "SHA-1", + "sha256": "SHA-256", + "sha384": "SHA-384", + "sha512": "SHA-512", +} + + +def document(body: Body, namespace: uuid.UUID) -> Document: + """ + Wrap a document body in the CycloneDX 1.6 header. + + :param body: the metadata, components, dependencies and formulation of the document + :param namespace: UUID namespace for the serial number, one per project + :returns: the complete document, whose serial number derives from the body so identical bodies share it + + """ + return { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": f"urn:uuid:{uuid.uuid5(namespace, json.dumps(body, sort_keys=True))}", + "version": 1, + **body, + } + + +def timestamp() -> str: + """ + Format the build time for the document. + + :returns: the ISO 8601 time from ``SOURCE_DATE_EPOCH``, or hatchling's fixed default when unset + + """ + return datetime.fromtimestamp(get_reproducible_timestamp(), tz=timezone.utc).isoformat() + + +def purl(name: str, version: str | None = None) -> str: + """ + Build the package URL of a Python distribution. + + :param name: distribution name, normalized here + :param version: version, or ``None`` for an unversioned purl + :returns: the ``pkg:pypi`` package URL + + """ + normalized = re.sub(r"[-_.]+", "-", name).lower() + return f"pkg:pypi/{normalized}@{version}" if version else f"pkg:pypi/{normalized}" + + +def external_reference(label: str, url: str) -> ExternalReference: + """ + Turn a ``Project-URL`` entry into an external reference. + + :param label: the entry's label, which picks the reference type + :param url: the entry's URL + :returns: the reference, typed ``other`` when the label is not a known one + + """ + reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") + return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} + + +def contacts(names: list[str], addresses: list[str]) -> list[Contact]: + """ + Turn core metadata author or maintainer fields into CycloneDX contacts. + + :param names: bare names, from ``Author`` or ``Maintainer`` + :param addresses: RFC 5322 addresses, from ``Author-email`` or ``Maintainer-email`` + :returns: one contact per name, then one per address + + """ + found: list[Contact] = [{"name": name} for name in names] + for name, email in getaddresses(addresses): + contact: Contact = {} + if name: + contact["name"] = name + if email: + contact["email"] = email + found.append(contact) + return found + + +def component_from_metadata(metadata: PackageMetadata | Message, component_type: str) -> Component: + """ + Describe a distribution from its core metadata. + + :param metadata: the distribution's ``METADATA`` + :param component_type: the CycloneDX component type, such as ``library`` + :returns: the component, with its purl as ``bom-ref``, licenses, Project-URL references and metadata properties + + """ + name, version = metadata["Name"], metadata["Version"] + component: Final[Component] = { + "type": component_type, + "bom-ref": (package := purl(name, version)), + "name": name, + "version": version, + "purl": package, + } + if summary := metadata.get("Summary"): + component["description"] = summary + if authors := contacts( + metadata.get_all("Author", []) + metadata.get_all("Maintainer", []), + metadata.get_all("Author-email", []) + metadata.get_all("Maintainer-email", []), + ): + component["authors"] = authors + if licenses := _licenses(metadata): + component["licenses"] = licenses + component["externalReferences"] = [ + external_reference(label.strip(), url.strip()) + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) + ] + if home_page := metadata.get("Home-page"): + component["externalReferences"].insert(0, {"type": "website", "url": home_page}) + component["properties"] = [ + {"name": f"python:{header.lower()}", "value": value} + for header in _METADATA_PROPERTIES + for value in metadata.get_all(header, []) + ] + return component + + +def _licenses(metadata: PackageMetadata | Message) -> list[License]: + if expression := metadata.get("License-Expression"): + return [{"expression": expression, "acknowledgement": "declared"}] + names = [ + entry.rsplit(" :: ", 1)[-1] for entry in metadata.get_all("Classifier", []) if entry.startswith("License :: ") + ] + if (declared := metadata.get("License")) and "\n" not in declared: + names.append(declared) + licenses: Final[list[License]] = [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] + # a license policy judges SPDX ids only, so a component keeps its bare names when any one of them does not resolve, + # and the policy reports it instead of judging a partial set + identifiers: Final[list[str]] = [identifier for name in names if (identifier := _spdx_license(name)) is not None] + if len(identifiers) == len(names): + licenses.extend( + {"license": {"id": identifier, "acknowledgement": "concluded"}} for identifier in dict.fromkeys(identifiers) + ) + return licenses + + +def _spdx_license(name: str) -> str | None: + if identifier := _CLASSIFIER_LICENSES.get(name): + return identifier + try: + canonical = canonicalize_license_expression(name) + except InvalidLicenseExpression: + return None + # a CycloneDX license id holds one entry of the SPDX license list, never an expression or a LicenseRef + return canonical if re.fullmatch(r"[A-Za-z0-9.-]+", canonical) and not canonical.startswith("LicenseRef-") else None + + +def file_component(parent_ref: str, path: str, digest: str, size: str) -> Component: + """ + Describe one file from a wheel ``RECORD`` row. + + :param parent_ref: ``bom-ref`` of the component that holds the file + :param path: the file's path inside the wheel + :param digest: the ``RECORD`` digest, ``=`` + :param size: the file's size in bytes + :returns: a ``file`` component with the digest as a lowercase hex hash + + """ + algorithm, _, encoded = digest.partition("=") + raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + return { + "type": "file", + "bom-ref": f"{parent_ref}#{path}", + "name": path, + "hashes": [{"alg": _RECORD_HASH_ALGORITHMS[algorithm], "content": raw.hex()}], + "properties": [{"name": "size", "value": size}], + } + + +def build_tools() -> tuple[list[Component], list[Dependency]]: + """ + Describe the running build environment: the interpreter and every installed distribution. + + Nothing about the build machine goes in, so the same toolchain yields the same record on any operating system or + architecture. A pure-Python distribution lists its files with their hashes; a platform one does not, because its + files differ per machine. + + :returns: the tool components, interpreter first, and the dependency graph between the distributions + + """ + interpreter = f"pkg:generic/{sys.implementation.name}@{platform.python_version()}" + tools: Final[list[Component]] = [ + { + "type": "platform", + "bom-ref": f"tool:{interpreter}", + "name": sys.implementation.name, + "version": platform.python_version(), + "purl": interpreter, + "properties": [{"name": "python:implementation", "value": platform.python_implementation()}], + }, + ] + # bom-refs are prefixed because the same distribution can be both a build tool and a component of the artifact + installed = {purl(distribution.metadata["Name"]): distribution for distribution in distributions()} + tool_dependencies: Final[list[Dependency]] = [] + for distribution in (installed[key] for key in sorted(installed)): + component = component_from_metadata(distribution.metadata, "library") + component["bom-ref"] = f"tool:{component['purl']}" + if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] == "true": + component["components"] = [ + file_component( + component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) + ) + for file in distribution.files or [] + # console-script launchers live outside site-packages and embed the build env's interpreter path in + # their shebang, so their hash differs on every build and says nothing about the distribution; the + # installer metadata names the build frontend that set up the env rather than the distribution + if file.hash is not None + and not file.as_posix().startswith("../") + and not ( + file.parent.suffix == ".dist-info" and file.name in {"INSTALLER", "REQUESTED", "direct_url.json"} + ) + ] + tools.append(component) + tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) + return tools, tool_dependencies + + +def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) -> list[str]: + refs = set() + for requirement in map(Requirement, distribution.requires or []): + if (requirement.marker is None or requirement.marker.evaluate()) and ( + dependency := installed.get(purl(requirement.name)) + ): + refs.add(f"tool:{purl(dependency.metadata['Name'], dependency.version)}") + return sorted(refs) + + +__all__ = [ + "Body", + "Component", + "Composition", + "Contact", + "Dependency", + "Document", + "Evidence", + "ExternalReference", + "Formula", + "Hash", + "License", + "LicenseInfo", + "Metadata", + "OrganizationalEntity", + "Property", + "Workflow", + "build_tools", + "component_from_metadata", + "contacts", + "document", + "external_reference", + "file_component", + "purl", + "timestamp", +] diff --git a/src/hatch_pep770/hooks.py b/src/hatch_pep770/hooks.py new file mode 100644 index 0000000..9138d5b --- /dev/null +++ b/src/hatch_pep770/hooks.py @@ -0,0 +1,17 @@ +"""Register the build hook with hatchling.""" + +from __future__ import annotations + +from hatchling.plugin import hookimpl + +from .plugin import Pep770BuildHook + + +@hookimpl +def hatch_register_build_hook() -> type[Pep770BuildHook]: + return Pep770BuildHook + + +__all__ = [ + "hatch_register_build_hook", +] diff --git a/src/hatch_pep770/plugin.py b/src/hatch_pep770/plugin.py new file mode 100644 index 0000000..ccc706d --- /dev/null +++ b/src/hatch_pep770/plugin.py @@ -0,0 +1,453 @@ +"""The hatchling build hook that writes a PEP 770 CycloneDX SBOM into the wheel.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import shutil +import subprocess +import tempfile +import uuid +import zipfile +from email.parser import Parser +from functools import cached_property +from io import StringIO +from itertools import starmap +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Any, Final +from urllib.parse import urlsplit + +from hatchling.__about__ import __version__ as hatchling_version +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.builders.utils import get_reproducible_timestamp +from hatchling.builders.wheel import WheelBuilder +from hatchling.utils.constants import DEFAULT_BUILD_SCRIPT +from packaging.requirements import Requirement + +from .cyclonedx import ( + build_tools, + component_from_metadata, + contacts, + document, + external_reference, + file_component, + purl, + timestamp, +) + +if TYPE_CHECKING: + from email.message import Message + + from hatchling.metadata.core import CoreMetadata + from packaging.specifiers import Specifier + + from .cyclonedx import ( + Body, + Component, + Contact, + Dependency, + Document, + ExternalReference, + License, + OrganizationalEntity, + Property, + Workflow, + ) + + +class Pep770BuildHook(BuildHookInterface): + """ + Write a PEP 770 CycloneDX 1.6 SBOM into the wheel's ``.dist-info/sboms/`` directory. + + The document describes the project from its core metadata, the runtime dependencies it declares, any wheels it + bundles as data files, and the build toolchain. It holds nothing about the build machine and takes its timestamp + from ``SOURCE_DATE_EPOCH``, so a rebuild with the same source, toolchain and epoch produces the same bytes on any + operating system. + + Subclass it in a hatchling custom build hook to add project-specific detail; see :meth:`bundled_wheel_properties`. + + """ + + PLUGIN_NAME = "pep770" + _sbom_directory: Path | None = None + + def initialize(self, version: str, build_data: dict[str, Any]) -> None: # ruff: ignore[unused-method-argument] + # `version` is hatchling's build variant name (e.g. "standard"), not the package version + if not isinstance(builder := self.build_config.builder, WheelBuilder): + return + if "sbom_files" not in build_data: # pragma: <3.10 cover # hatchling 1.28, the first with it, needs 3.10 + self.app.display_warning( + f"hatchling {hatchling_version} cannot embed SBOMs (1.28 adds it), so none written" + ) + return + tag = build_data.get("tag") or ( + builder.get_best_matching_tag() if build_data["infer_tag"] else builder.get_default_tag() + ) + content = json.dumps(self._document(f"{builder.artifact_project_id}-{tag}.whl"), indent=2) + # written outside self.directory, which hatchling also uses as the wheel's output directory, so no stray copy + # lands next to the built artifacts + self._sbom_directory = Path(tempfile.mkdtemp(prefix="hatch-pep770-")) + # bytes, so the line endings do not depend on the build machine's OS + (out := self._sbom_directory / f"{self._namespace}.cdx.json").write_bytes(f"{content}\n".encode()) + build_data["sbom_files"].append(str(out)) + + def finalize(self, version: str, build_data: dict[str, Any], artifact_path: str) -> None: # ruff: ignore[unused-method-argument] + if self._sbom_directory is not None: + shutil.rmtree(self._sbom_directory) + self._sbom_directory = None + + def bundled_wheel_properties(self, wheel: Path) -> list[Property]: # ruff: ignore[unused-method-argument, no-self-use] + """ + Return extra properties for one wheel the project bundles as a data file. + + The default adds none. A subclass overrides it to say what the project does with the wheel, for example which + Python versions receive it. The properties follow ``:bundled-wheel`` and precede the ones taken from + the wheel's metadata. + + :param wheel: the bundled wheel, an absolute path under the project root + :returns: the properties to add + + """ + return [] + + @property + def _namespace(self) -> str: + # hatchling normalizes the name; it also prefixes the CycloneDX property names this hook writes + return self.metadata.core.name + + def _document(self, wheel_name: str) -> Document: + core: Final[CoreMetadata] = self.metadata.core + version: Final[str] = self.metadata.version + root = self._root_component(core, version, wheel_name) + bundled = [self._bundled_component(wheel) for wheel in self._bundled_wheels] + declared = [self._declared_dependency(requirement) for requirement in core.dependencies] + tools, tool_dependencies = build_tools() + if script := self._build_script_tool(version): + tools.insert(0, script) + bundled_dependencies: Final[list[Dependency]] = [ + {"ref": component["bom-ref"], "dependsOn": references} + for component in bundled + if (references := [child["bom-ref"] for child in component["components"] if child["type"] == "library"]) + ] + supplier = self._supplier + body: Final[Body] = { + "metadata": { + "timestamp": timestamp(), + "lifecycles": [{"phase": "build"}], + "tools": {"components": tools}, + **({"manufacturer": supplier} if supplier else {}), + **({"authors": self._authors} if self._authors else {}), + **({"supplier": supplier} if supplier else {}), + "component": root, + **({"licenses": self._licenses} if self._licenses else {}), + "properties": [{"name": f"{self._namespace}:sbom:scope", "value": self._scope}], + }, + "components": [*bundled, *declared], + "dependencies": [ + {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in [*bundled, *declared]]}, + *bundled_dependencies, + *tool_dependencies, + ], + # transitive runtime dependencies are unknown until install time + "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], + "formulation": [{"bom-ref": "formula:wheel-build", "workflows": [self._workflow(root, tools)]}], + } + base: Final[str] = self._repository or f"https://pypi.org/project/{self._namespace}" + return document(body, uuid.uuid5(uuid.NAMESPACE_URL, f"{base}/sboms")) + + def _root_component(self, core: CoreMetadata, version: str, wheel_name: str) -> Component: + package = purl(core.name, version) + references: Final[list[ExternalReference]] = list(starmap(external_reference, core.urls.items())) + references.extend([ + {"type": "distribution", "url": f"https://pypi.org/project/{core.name}/{version}/"}, + {"type": "attestation", "url": f"https://pypi.org/integrity/{core.name}/{version}/{wheel_name}/provenance"}, + ]) + if (repository := self._repository) and urlsplit(repository).netloc == "github.com": + references.extend([ + {"type": "security-contact", "url": f"{repository}/security/policy"}, + {"type": "advisories", "url": f"{repository}/security/advisories"}, + ]) + if repository and core.license_files: + references.append({"type": "license", "url": f"{repository}/blob/{self._branch}/{core.license_files[0]}"}) + properties: Final[list[Property]] = [] + if core.requires_python: + properties.append({"name": f"{self._namespace}:requires-python", "value": core.requires_python}) + properties.extend({"name": "python:classifier", "value": classifier} for classifier in core.classifiers) + properties.extend({"name": "python:keyword", "value": keyword} for keyword in core.keywords) + if commit := self._commit(): + if repository: + references.append({ + "type": "vcs", + "url": f"{repository}/tree/{commit}", + "comment": "exact source revision", + }) + properties.append({"name": f"{self._namespace}:vcs-commit", "value": commit}) + supplier = self._supplier + copyright_line = self._copyright(core.license_files) + return { + "type": "application", + "bom-ref": package, + **({"supplier": supplier} if supplier else {}), + **({"authors": self._authors} if self._authors else {}), + "name": core.name, + "version": version, + **({"description": core.description} if core.description else {}), + **({"licenses": self._licenses} if self._licenses else {}), + **({"copyright": copyright_line} if copyright_line else {}), + "purl": package, + "externalReferences": references, + "properties": properties, + } + + @cached_property + def _supplier(self) -> OrganizationalEntity | None: + if (supplier := self.config.get("supplier")) is None: + return None + if not ( + isinstance(supplier, dict) + and isinstance(name := supplier.get("name"), str) + and isinstance(urls := supplier.get("url", []), list) + and all(isinstance(url, str) for url in urls) + and supplier.keys() <= {"name", "url"} + ): + msg = f"{self._option('supplier')} must be a table with a string `name` and an optional `url` string array" + raise TypeError(msg) + return {"name": name, "url": urls} + + def _option(self, name: str) -> str: + return f"Option `{name}` of build hook `{self.PLUGIN_NAME}`" + + @cached_property + def _authors(self) -> list[Contact]: + maintainers = self.metadata.core.maintainers_data + authors = self.metadata.core.authors_data + people = maintainers if maintainers["name"] or maintainers["email"] else authors + return contacts(people["name"], people["email"]) + + @cached_property + def _licenses(self) -> list[License]: + if expression := self.metadata.core.license_expression: + return [{"expression": expression, "acknowledgement": "declared"}] + return [] + + @cached_property + def _repository(self) -> str | None: + if (repository := self.config.get("repository")) is not None: + if not isinstance(repository, str): + msg = f"{self._option('repository')} must be a string" + raise TypeError(msg) + return repository.rstrip("/") + return next( + ( + url.rstrip("/") + for label, url in self.metadata.core.urls.items() + if external_reference(label, url)["type"] == "vcs" + ), + None, + ) + + @cached_property + def _branch(self) -> str: + if not isinstance(branch := self.config.get("default-branch", "main"), str): + msg = f"{self._option('default-branch')} must be a string" + raise TypeError(msg) + return branch + + def _commit(self) -> str | None: + if not (Path(self.root) / ".git").exists() or (git := shutil.which("git")) is None: # building from an sdist + return None + try: + result = subprocess.run( + [git, "rev-parse", "HEAD"], check=True, capture_output=True, text=True, cwd=self.root + ) + except subprocess.CalledProcessError as error: + self.app.display_warning(f"git rev-parse HEAD failed, so the SBOM names no source revision: {error.stderr}") + return None + return result.stdout.strip() + + def _copyright(self, license_files: list[str]) -> str | None: + for license_file in license_files: + for line in (Path(self.root) / license_file).read_text(encoding="utf-8").splitlines(): + if line.startswith("Copyright"): + return line + return None + + @cached_property + def _bundled_wheels(self) -> list[Path]: + root = Path(self.root) + found = {path for pattern in self._bundled_patterns for path in root.glob(pattern)} + return sorted(found, key=lambda path: path.relative_to(root).as_posix()) + + @cached_property + def _bundled_patterns(self) -> list[str]: + patterns = self.config.get("bundled-wheels", []) + if not (isinstance(patterns, list) and all(isinstance(pattern, str) for pattern in patterns)): + msg = f"{self._option('bundled-wheels')} must be an array of strings" + raise TypeError(msg) + return patterns + + def _bundled_component(self, wheel: Path) -> Component: + with zipfile.ZipFile(wheel) as archive: + members: Final[list[str]] = sorted(archive.namelist()) + metadata_name: Final[str] = next( + name for name in members if name.count("/") == 1 and name.endswith(".dist-info/METADATA") + ) + metadata = Parser().parsestr(archive.read(metadata_name).decode("utf-8")) + record = archive.read(metadata_name.replace("METADATA", "RECORD")).decode("utf-8") + vendored_metadata: Final[list[tuple[str, Message]]] = [ + *self._vendor_manifest(wheel, archive, members), + *( + (name, Parser().parsestr(archive.read(name).decode("utf-8"))) + for name in members + if "/_vendor/" in name and name.endswith(".dist-info/METADATA") + ), + ] + component = component_from_metadata(metadata, "library") + if ( + (supplier := self._supplier) + and (owner := self._repository_owner) + and any(reference["url"].startswith(owner) for reference in component["externalReferences"]) + ): + component["supplier"] = supplier + sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + component["hashes"] = [{"alg": "SHA-256", "content": sha256}] + component["externalReferences"].append( + {"type": "distribution", "url": f"https://pypi.org/project/{metadata['Name']}/{metadata['Version']}/"}, + ) + component["properties"] = [ + {"name": f"{self._namespace}:bundled-wheel", "value": wheel.relative_to(self.root).as_posix()}, + *self.bundled_wheel_properties(wheel), + *component["properties"], + ] + component["evidence"] = { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": metadata_name}], + }, + { + "field": "hash", + "confidence": 1, + "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], + }, + ], + } + component["components"] = [ + file_component(component["bom-ref"], path, digest, size) + for path, digest, size in (row for row in csv.reader(StringIO(record, newline="")) if row) + if digest + ] + vendored_components: Final[dict[str, Component]] = {} + for source, vendored in vendored_metadata: + child: Final[Component] = component_from_metadata(vendored, "library") + child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['purl']}" + child["properties"].append({"name": f"{self._namespace}:vendored-manifest", "value": source}) + child["evidence"] = { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], + } + ], + } + # a distribution's own .dist-info wins over the pip-style manifest line naming the same purl + vendored_components[child["purl"]] = child + component["components"].extend(vendored_components[key] for key in sorted(vendored_components)) + return component + + @staticmethod + def _vendor_manifest(wheel: Path, archive: zipfile.ZipFile, members: list[str]) -> list[tuple[str, Message]]: + # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info + found: Final[list[tuple[str, Message]]] = [] + for name in members: + if not name.endswith("/_vendor/vendor.txt"): + continue + for line in archive.read(name).decode("utf-8").splitlines(): + if not (requirement := line.partition("#")[0].strip()): + continue + parsed: Final[Requirement] = Requirement(requirement) + pins: Final[list[Specifier]] = list(parsed.specifier) + if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: + msg: Final[str] = f"Unpinned vendored dependency in {wheel.name}:{name}: {requirement}" + raise ValueError(msg) + found.append((name, Parser().parsestr(f"Name: {parsed.name}\nVersion: {pins[0].version}\n"))) + return found + + @cached_property + def _repository_owner(self) -> str | None: + # a bundled wheel published from the same forge account as this project shares its supplier + if (repository := self._repository) is None: + return None + parts = urlsplit(repository) + return f"{parts.scheme}://{parts.netloc}/{parts.path.strip('/').split('/')[0]}/" + + def _declared_dependency(self, requirement: str) -> Component: + parsed = Requirement(requirement) + component: Final[Component] = { + "type": "library", + # the same distribution can be declared more than once with different markers, so the purl is not unique + "bom-ref": f"requires-dist:{requirement}", + "name": parsed.name, + "purl": purl(parsed.name), + "externalReferences": [{"type": "distribution", "url": f"https://pypi.org/project/{parsed.name}/"}], + "properties": [{"name": f"{self._namespace}:requires-dist", "value": requirement}], + } + if parsed.marker is not None: + component["properties"].append({ + "name": f"{self._namespace}:environment-marker", + "value": str(parsed.marker), + }) + return component + + def _build_script_tool(self, version: str) -> Component | None: + # a custom build hook script shapes the document, so it is part of the toolchain a rebuild has to match + if self.PLUGIN_NAME != "custom": + return None + script = PurePosixPath(self.config.get("path", DEFAULT_BUILD_SCRIPT)) + tool: Final[Component] = { + "type": "application", + "bom-ref": f"tool:{script}", + "name": script.name, + "version": version, + "hashes": [ + {"alg": "SHA-256", "content": hashlib.sha256((Path(self.root) / script).read_bytes()).hexdigest()} + ], + } + if repository := self._repository: + tool["externalReferences"] = [{"type": "vcs", "url": f"{repository}/blob/{self._branch}/{script}"}] + return tool + + @staticmethod + def _workflow(root: Component, tools: list[Component]) -> Workflow: + return { + "bom-ref": "workflow:wheel-build", + "uid": "wheel-build", + "name": "build the wheel and this SBOM", + "taskTypes": ["build"], + "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], + "outputs": [{"type": "artifact", "resource": {"ref": root["bom-ref"]}}], + "inputs": [ + {"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": str(get_reproducible_timestamp())}]}, + ], + } + + @cached_property + def _scope(self) -> str: + declared = ( + "the direct runtime dependencies declared in the wheel's core metadata. The latter carry no version" + " because they are resolved at install time, not at build time." + ) + if directories := sorted({str(PurePosixPath(pattern).parent) for pattern in self._bundled_patterns}): + return f"Components are the wheels bundled as data files under {' and '.join(directories)} and {declared}" + return ( + "Components are the direct runtime dependencies declared in the wheel's core metadata. They carry no" + " version because they are resolved at install time, not at build time." + ) + + +__all__ = [ + "Pep770BuildHook", +] diff --git a/src/hatch_pep770/spdx.py b/src/hatch_pep770/spdx.py new file mode 100644 index 0000000..afcb2c7 --- /dev/null +++ b/src/hatch_pep770/spdx.py @@ -0,0 +1,213 @@ +"""Render a CycloneDX SBOM written by the build hook as an SPDX 2.3 JSON document.""" + +from __future__ import annotations + +import re +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Final, TypedDict + +from .version import __version__ + +if TYPE_CHECKING: + from collections.abc import Iterator + + from .cyclonedx import Component, Document, License + + +class _Checksum(TypedDict): + algorithm: str + checksumValue: str + + +class _ExternalRef(TypedDict): + referenceCategory: str + referenceType: str + referenceLocator: str + + +class _PackageKeys(TypedDict): + SPDXID: str + name: str + downloadLocation: str + filesAnalyzed: bool + licenseConcluded: str + licenseDeclared: str + primaryPackagePurpose: str + + +class _Package(_PackageKeys, total=False): + versionInfo: str + supplier: str + checksums: list[_Checksum] + homepage: str + copyrightText: str + summary: str + comment: str + externalRefs: list[_ExternalRef] + licenseComments: str + + +class _Relationship(TypedDict): + spdxElementId: str + relationshipType: str + relatedSpdxElement: str + + +class _CreationInfo(TypedDict): + created: str + creators: list[str] + + +class Spdx(TypedDict): + spdxVersion: str + dataLicense: str + SPDXID: str + name: str + documentNamespace: str + comment: str + creationInfo: _CreationInfo + packages: list[_Package] + relationships: list[_Relationship] + + +_PURPOSES: Final[dict[str, str]] = { + "application": "APPLICATION", + "library": "LIBRARY", + "platform": "OTHER", +} + + +def to_spdx(bom: Document) -> Spdx: + """ + Render a CycloneDX document as SPDX 2.3. + + Packages keep their hashes, licenses and purls, and the containment, dependency and build tool relationships carry + over. Per-file hashes stay behind, because SPDX 2.3 requires a SHA-1 for every file and wheel ``RECORD`` files hold + SHA-256, and so does the build formulation, which SPDX 2.3 has no field for. + + :param bom: a CycloneDX document as the build hook writes it + :returns: the SPDX document, whose namespace ends in the CycloneDX serial number so both name the same build + + """ + root: Final[Component] = bom["metadata"]["component"] + tools: Final[list[Component]] = bom["metadata"]["tools"]["components"] + name: Final[str] = f"{root['name']}-{root['version']}" + namespace: Final[str] = re.sub(r"[-_.]+", "-", root["name"]).lower() + repository: Final[str] = next( + ( + reference["url"] + for reference in root.get("externalReferences", []) + if reference["type"] == "vcs" and "Project-URL" in reference.get("comment", "") + ), + "https://spdx.org/spdxdocs", + ) + components: Final[list[Component]] = list(_walk([root, *bom["components"], *tools])) + supplier: Final[list[str]] = ( + [f"Organization: {bom['metadata']['supplier']['name']}"] if "supplier" in bom["metadata"] else [] + ) + return { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": name, + "documentNamespace": f"{repository}/sboms/{name}-{bom['serialNumber'].removeprefix('urn:uuid:')}", + "comment": f"Rendered from the CycloneDX SBOM {bom['serialNumber']} ({namespace}.cdx.json), which also carries" + " per-file hashes and the build formulation.", + "creationInfo": { + "created": datetime + .fromisoformat(bom["metadata"]["timestamp"]) + .astimezone(timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%SZ"), + "creators": [ + *supplier, + *(f"Tool: {tool['name']}-{tool['version']}" for tool in tools if tool["type"] == "application"), + f"Tool: hatch-pep770-{__version__}", + ], + }, + "packages": [_package(component, f"{namespace}:") for component in components], + "relationships": [ + _relationship("SPDXRef-DOCUMENT", "DESCRIBES", _spdx_id(root["bom-ref"])), + *( + _relationship(_spdx_id(parent["bom-ref"]), "CONTAINS", _spdx_id(child["bom-ref"])) + for parent in components + for child in parent.get("components", []) + if child["type"] != "file" + ), + *( + _relationship(_spdx_id(entry["ref"]), "DEPENDS_ON", _spdx_id(dependency)) + for entry in bom["dependencies"] + for dependency in entry["dependsOn"] + ), + *(_relationship(_spdx_id(tool["bom-ref"]), "BUILD_TOOL_OF", _spdx_id(root["bom-ref"])) for tool in tools), + ], + } + + +def _walk(components: list[Component]) -> Iterator[Component]: + # file components mirror wheel RECORD entries, which carry no SHA-1 and so cannot become SPDX files + for component in components: + if component["type"] != "file": + yield component + yield from _walk(component.get("components", [])) + + +def _package(component: Component, property_prefix: str) -> _Package: + # the hook writes either one SPDX expression, or the free-form names from License/classifier metadata plus the SPDX + # ids it concluded from them + licenses: Final[list[License]] = component.get("licenses", []) + concluded: Final[list[str]] = [ + entry["license"]["id"] for entry in licenses if "license" in entry and "id" in entry["license"] + ] + package: Final[_Package] = { + "SPDXID": _spdx_id(component["bom-ref"]), + "name": component["name"], + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "licenseConcluded": " AND ".join(concluded) or "NOASSERTION", + "licenseDeclared": next((entry["expression"] for entry in licenses if "expression" in entry), "NOASSERTION"), + "primaryPackagePurpose": _PURPOSES[component["type"]], + } + if version := component.get("version"): + package["versionInfo"] = version + if supplier := component.get("supplier"): + package["supplier"] = f"Organization: {supplier['name']}" + if hashes := component.get("hashes"): + package["checksums"] = [ + {"algorithm": entry["alg"].replace("-", ""), "checksumValue": entry["content"]} for entry in hashes + ] + references = component.get("externalReferences", []) + if homepage := next((reference["url"] for reference in references if reference["type"] == "website"), None): + package["homepage"] = homepage + if copyright_text := component.get("copyright"): + package["copyrightText"] = copyright_text + if description := component.get("description"): + package["summary"] = description + if comment := "\n".join( + f"{prop['name']}: {prop['value']}" + for prop in component.get("properties", []) + if prop["name"].startswith(property_prefix) + ): + package["comment"] = comment + if purl := component.get("purl"): + package["externalRefs"] = [ + {"referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", "referenceLocator": purl} + ] + if package["licenseDeclared"] == "NOASSERTION" and ( + names := [entry["license"]["name"] for entry in licenses if "license" in entry and "name" in entry["license"]] + ): + package["licenseComments"] = f"Declared in package metadata as: {'; '.join(names)}" + return package + + +def _relationship(element: str, kind: str, related: str) -> _Relationship: + return {"spdxElementId": element, "relationshipType": kind, "relatedSpdxElement": related} + + +def _spdx_id(bom_ref: str) -> str: + return f"SPDXRef-{re.sub(r'[^A-Za-z0-9.]+', '-', bom_ref).strip('-')}" + + +__all__ = [ + "Spdx", + "to_spdx", +] diff --git a/src/hatch_pep770/validate.py b/src/hatch_pep770/validate.py new file mode 100644 index 0000000..04cba3e --- /dev/null +++ b/src/hatch_pep770/validate.py @@ -0,0 +1,238 @@ +"""Check an SBOM against the CycloneDX 1.6 schema, its own references, its SPDX rendering and a license policy.""" + +from __future__ import annotations + +import json +import re +import zipfile +from typing import TYPE_CHECKING, Final + +from cyclonedx.schema import SchemaVersion +from cyclonedx.validation.json import JsonStrictValidator +from spdx_tools.spdx.parser.error import SPDXParsingError +from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser +from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document + +from .spdx import to_spdx + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from .cyclonedx import Component, Document, License + +_SERIAL_PATTERN: Final[re.Pattern[str]] = re.compile( + r""" + ^urn:uuid: + (?P[0-9a-f]{8})- + (?P[0-9a-f]{4})- + (?P[0-9a-f]{4})- + (?P[0-9a-f]{4})- + (?P[0-9a-f]{12}) + $ + """, + re.VERBOSE, +) +# SPDX matches operators only in upper case +_OPERATORS: Final[frozenset[str]] = frozenset({"AND", "OR", "WITH", "(", ")"}) +_TOKEN: Final[re.Pattern[str]] = re.compile(r"[()]|[^\s()]+") + + +def wheel_problems(wheel: Path, allowed_licenses: frozenset[str] | None) -> list[str]: + """ + Check every PEP 770 CycloneDX SBOM in a wheel. + + :param wheel: the wheel to read + :param allowed_licenses: SPDX license ids the shipped components may use, or ``None`` to skip the license policy + :returns: one message per problem, empty when the wheel carries at least one SBOM and every SBOM is valid + + """ + with zipfile.ZipFile(wheel) as archive: + sboms = { + name: archive.read(name).decode("utf-8") + for name in sorted(archive.namelist()) + if re.fullmatch(r"[^/]+\.dist-info/sboms/[^/]+\.cdx\.json", name) + } + if not sboms: + return ["no .dist-info/sboms/*.cdx.json found in the wheel"] + return [f"{name}: {problem}" for name, raw in sboms.items() for problem in document_problems(raw, allowed_licenses)] + + +def document_problems(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: + """ + Check one CycloneDX SBOM. + + Beyond the schema it requires the ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, + unique and resolvable ``bom-ref`` values, a root that depends on exactly the top-level components, and an SPDX 2.3 + rendering that validates. + + :param raw: the JSON text of the document + :param allowed_licenses: SPDX license ids the shipped components may use, or ``None`` to skip the license policy + :returns: one message per problem, empty when the document is valid + + """ + if schema_error := JsonStrictValidator(SchemaVersion.V1_6).validate_str(raw): + return [f"does not conform to the CycloneDX 1.6 schema: {schema_error}"] + document: Final[Document] = json.loads(raw) + if problems := _serial_problems(document.get("serialNumber", "")): + return problems + problems = _reference_problems(document) + problems += _spdx_problems(document) + if allowed_licenses is not None: + problems += license_problems(document["components"], allowed_licenses) + return problems + + +def _serial_problems(serial: str) -> list[str]: + # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize the document + # as CycloneDX at all, and silently rejects anything missing it as an unknown format + return [] if _SERIAL_PATTERN.match(serial) else [f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {serial!r}"] + + +def _reference_problems(document: Document) -> list[str]: + root_ref = document["metadata"]["component"]["bom-ref"] + tools = document["metadata"]["tools"]["components"] + every_ref = list(_bom_refs([document["metadata"]["component"], *document["components"], *tools])) + if duplicates := {ref for ref in every_ref if every_ref.count(ref) > 1}: + return [f"bom-ref values are not unique: {sorted(duplicates)}"] + problems = [] + dependencies = {entry["ref"]: set(entry["dependsOn"]) for entry in document["dependencies"]} + if unknown := (set(dependencies) | set().union(*dependencies.values())) - set(every_ref): + problems.append(f"dependencies reference bom-refs that do not exist: {sorted(unknown)}") + if dependencies.get(root_ref) != (component_refs := {component["bom-ref"] for component in document["components"]}): + problems.append( + f"root dependsOn {sorted(dependencies.get(root_ref, []))} != components {sorted(component_refs)}" + ) + tool_refs = {tool["bom-ref"] for tool in tools} + problems.extend( + f"workflow {workflow['uid']} references unknown tools: {sorted(unknown)}" + for workflow in (workflow for formula in document["formulation"] for workflow in formula["workflows"]) + if (unknown := {reference["ref"] for reference in workflow["resourceReferences"]} - tool_refs) + ) + return problems + + +def _bom_refs(components: list[Component]) -> Iterator[str]: + for component in components: + yield component["bom-ref"] + yield from _bom_refs(component.get("components", [])) + + +def _spdx_problems(document: Document) -> list[str]: + try: + spdx = JsonLikeDictParser().parse(dict(to_spdx(document))) + except SPDXParsingError as error: + return [f"SPDX rendering does not parse: {message}" for message in error.get_messages()] + return [f"SPDX rendering invalid: {message.validation_message}" for message in validate_full_spdx_document(spdx)] + + +def license_problems(components: list[Component], allowed_licenses: frozenset[str]) -> list[str]: + """ + Judge the licenses of what an artifact ships against a policy. + + Only components the artifact carries bytes for are judged: a bundled wheel, or a component with files. A declared + requirement resolves at install time, and a library a bundled wheel vendors ships under that wheel's terms. + + :param components: the document's top-level components + :param allowed_licenses: SPDX license ids a shipped component may use; an expression passes when the ids it needs + are allowed, and ``WITH`` exceptions only grant extra permissions so the license they modify decides + :returns: one message per component that fails the policy + + """ + return [ + f"{component['bom-ref']}: {problem}" + for component in _shipped(components) + if (problem := _judge(component, allowed_licenses)) + ] + + +def _shipped(components: list[Component]) -> Iterator[Component]: + for component in components: + if component["type"] == "library" and ( + "hashes" in component or any(child["type"] == "file" for child in component.get("components", [])) + ): + yield component + yield from _shipped(component.get("components", [])) + + +def _judge(component: Component, allowed: frozenset[str]) -> str | None: + licenses: Final[list[License]] = component.get("licenses", []) + terms: Final[list[str]] = [ + entry["expression"] if "expression" in entry else entry["license"]["id"] + for entry in licenses + if "expression" in entry or "id" in entry["license"] + ] + if not terms: + if names := [entry["license"]["name"] for entry in licenses if "license" in entry]: + return f"declares license names {names} with no SPDX id" + return "declares no license" + try: + denied: Final[list[str]] = [term for term in terms if not _allowed(term, allowed)] + except ValueError as error: + return str(error) + if denied: + return f"license {' AND '.join(denied)} is outside the allowed licenses {sorted(allowed)}" + return None + + +def _allowed(expression: str, allowed: frozenset[str]) -> bool: + tokens: Final[list[str]] = _TOKEN.findall(expression)[::-1] + try: + verdict: Final[bool] = _any_of(tokens, allowed) + except ValueError as error: + msg = f"license expression {expression!r} is malformed: {error}" + raise ValueError(msg) from error + if tokens: + msg = f"license expression {expression!r} is malformed: unexpected {tokens[-1]!r}" + raise ValueError(msg) + return verdict + + +def _any_of(tokens: list[str], allowed: frozenset[str]) -> bool: + verdict = _all_of(tokens, allowed) + while tokens and tokens[-1] == "OR": + tokens.pop() + verdict |= _all_of(tokens, allowed) + return verdict + + +def _all_of(tokens: list[str], allowed: frozenset[str]) -> bool: + verdict = _term(tokens, allowed) + while tokens and tokens[-1] == "AND": + tokens.pop() + verdict &= _term(tokens, allowed) + return verdict + + +def _term(tokens: list[str], allowed: frozenset[str]) -> bool: + if (token := _pop(tokens)) == "(": + verdict: Final[bool] = _any_of(tokens, allowed) + if (closing := _pop(tokens)) != ")": + msg = f"expected ')', got {closing!r}" + raise ValueError(msg) + return verdict + if token in _OPERATORS: + msg = f"unexpected {token!r}" + raise ValueError(msg) + if tokens and tokens[-1] == "WITH": + tokens.pop() + # an exception grants extra permissions on top of the license it modifies, so that license alone decides + if (exception := _pop(tokens)) in _OPERATORS: + msg = f"unexpected {exception!r}" + raise ValueError(msg) + # SPDX matches license ids without regard to case + return token.casefold() in {identifier.casefold() for identifier in allowed} + + +def _pop(tokens: list[str]) -> str: + if not tokens: + msg: Final[str] = "it ends early" + raise ValueError(msg) + return tokens.pop() + + +__all__ = [ + "document_problems", + "license_problems", + "wheel_problems", +] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..19055f8 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import base64 +import csv +import hashlib +import json +import zipfile +from io import StringIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, Final, Protocol + +import pytest +from hatchling.builders.wheel import WheelBuilder + +if TYPE_CHECKING: + from collections.abc import Callable + + from hatch_pep770.cyclonedx import Document + + +class BundleWheel(Protocol): + def __call__(self, files: dict[str, str], metadata: str = "", name: str = "pip") -> Path: ... + + +class InstallTool(Protocol): + def __call__(self, site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> None: ... + + +@pytest.fixture(autouse=True) +def _clean_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + (package := tmp_path / "project" / "src" / "example").mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (tmp_path / "project" / "LICENSE").write_text("Copyright (c) 2026 Example\n\nMIT terms\n", encoding="utf-8") + return tmp_path / "project" + + +@pytest.fixture +def config() -> dict[str, Any]: + return { + "project": { + "name": "Example", + "version": "1.0", + "description": "an example", + "requires-python": ">=3.9", + "license": "MIT", + "license-files": ["LICENSE"], + "maintainers": [{"name": "Maintainer", "email": "maintainer@example.com"}], + "keywords": ["sample"], + "classifiers": ["Programming Language :: Python :: 3"], + "dependencies": ["packaging>=24", "tomli>=2; python_version<'3.11'"], + "urls": {"Changelog": "https://example.com/changelog", "Source": "https://github.com/example/example"}, + }, + "tool": { + "hatch": { + "build": { + "targets": {"wheel": {"packages": ["src/example"]}}, + "hooks": {"pep770": {}}, + } + } + }, + } + + +@pytest.fixture +def build_sbom(root: Path) -> Callable[[dict[str, Any]], str]: + # drives the hook through hatchling's own wheel builder, also on hatchling releases that predate `sbom_files` + def build(config: dict[str, Any]) -> str: + builder: Final[WheelBuilder] = WheelBuilder(str(root), config=config) + build_data: Final[dict[str, Any]] = {**builder.get_default_build_data(), "sbom_files": []} + (hook,) = builder.get_build_hooks(str(root / "dist")).values() + hook.initialize("standard", build_data) + content = Path(build_data["sbom_files"][0]).read_text(encoding="utf-8") + hook.finalize("standard", build_data, str(root / "dist" / "example.whl")) + return content + + return build + + +@pytest.fixture +def build_wheel(root: Path, tmp_path: Path) -> Callable[[dict[str, Any], str], Path]: + def build(config: dict[str, Any], out: str) -> Path: + (artifact,) = WheelBuilder(str(root), config=config).build(directory=str(tmp_path / out), versions=["standard"]) + return Path(artifact) + + return build + + +@pytest.fixture +def bundle_wheel(root: Path) -> BundleWheel: + def bundle(files: dict[str, str], metadata: str = "", name: str = "pip") -> Path: + (wheels := root / "src" / "example" / "wheels").mkdir(parents=True, exist_ok=True) + record: Final[StringIO] = StringIO(newline="") + csv.writer(record).writerows([ + (f"{name}/example.py", "sha256=ungWv48Bz-pBQUDeXa4iI7ADYaOWF3qctBD_YfIAFa0", "3"), + (f"{name}-1.0.dist-info/RECORD", "", ""), + ]) + with zipfile.ZipFile(wheel := wheels / f"{name}-1.0-py3-none-any.whl", "w") as archive: + for path, content in files.items(): + archive.writestr(zipfile.ZipInfo(path), content) + archive.writestr( + zipfile.ZipInfo(f"{name}-1.0.dist-info/METADATA"), + f"Metadata-Version: 2.4\nName: {name}\nVersion: 1.0\n{metadata}", + ) + archive.writestr(zipfile.ZipInfo(f"{name}-1.0.dist-info/RECORD"), record.getvalue()) + archive.writestr(zipfile.ZipInfo(f"{name}/example.py"), b"abc") + return wheel + + return bundle + + +@pytest.fixture +def install_tool() -> InstallTool: + def install(site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> None: + record: Final[StringIO] = StringIO(newline="") + for path, content in files.items(): + (target := site / path).parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + digest = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() + csv.writer(record).writerow((path, f"sha256={digest}", len(content))) + (dist_info := site / f"{name}-1.0.dist-info").mkdir(exist_ok=True) + (dist_info / "METADATA").write_text(f"Metadata-Version: 2.4\nName: {name}\nVersion: 1.0\n", encoding="utf-8") + (dist_info / "WHEEL").write_text( + f"Wheel-Version: 1.0\nRoot-Is-Purelib: {str(purelib).lower()}\n", encoding="utf-8" + ) + (dist_info / "RECORD").write_bytes(record.getvalue().encode()) + + return install + + +@pytest.fixture +def sbom_document(build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any]) -> Document: + return json.loads(build_sbom(config)) diff --git a/tests/test___main__.py b/tests/test___main__.py new file mode 100644 index 0000000..0469465 --- /dev/null +++ b/tests/test___main__.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import json +import sys +import zipfile +from typing import TYPE_CHECKING + +import pytest + +from hatch_pep770.__main__ import main +from hatch_pep770.spdx import to_spdx + +if TYPE_CHECKING: + from pathlib import Path + + from hatch_pep770.cyclonedx import Document + + +@pytest.fixture +def wheel(sbom_document: Document, tmp_path: Path) -> Path: + (dist := tmp_path / "dist").mkdir() + with zipfile.ZipFile(path := dist / "example-1.0-py3-none-any.whl", "w") as archive: + archive.writestr("example-1.0.dist-info/sboms/example.cdx.json", json.dumps(sbom_document)) + return path + + +def test_main_spdx(sbom_document: Document, tmp_path: Path) -> None: + (source := tmp_path / "example.cdx.json").write_text(json.dumps(sbom_document), encoding="utf-8") + assert (main(["spdx", str(source), str(target := tmp_path / "example.spdx.json")]), target.read_bytes()) == ( + 0, + f"{json.dumps(to_spdx(sbom_document), indent=2)}\n".encode(), + ) + + +@pytest.mark.parametrize( + "target", [pytest.param("example-1.0-py3-none-any.whl", id="wheel"), pytest.param(".", id="directory")] +) +def test_main_validate(wheel: Path, capsys: pytest.CaptureFixture[str], target: str) -> None: + assert (main(["validate", str(wheel.parent / target), "--allow-license", "MIT"]), capsys.readouterr().out) == ( + 0, + "SBOM in example-1.0-py3-none-any.whl is valid\n", + ) + + +def test_main_validate_problems(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: + archive.writestr("example-1.0.dist-info/METADATA", "Name: example\n") + assert (main(["validate", str(wheel)]), capsys.readouterr().err) == ( + 1, + "SBOM invalid: example-1.0-py3-none-any.whl: no .dist-info/sboms/*.cdx.json found in the wheel\n", + ) + + +def test_main_validate_no_wheel(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert (main(["validate", str(tmp_path)]), capsys.readouterr().err) == (1, f"no wheel found in {tmp_path}\n") + + +def test_main_module(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sys, "argv", ["hatch-pep770", "--help"]) + with pytest.raises(SystemExit): + main() + assert "usage: hatch-pep770" in capsys.readouterr().out diff --git a/tests/test_plugin.py b/tests/test_plugin.py new file mode 100644 index 0000000..52f91d4 --- /dev/null +++ b/tests/test_plugin.py @@ -0,0 +1,829 @@ +from __future__ import annotations + +import csv +import hashlib +import json +import os +import subprocess +import sys +import zipfile +from io import StringIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, Final + +import pytest +from hatchling.builders.sdist import SdistBuilder +from hatchling.builders.wheel import WheelBuilder + +from hatch_pep770.validate import wheel_problems + +if TYPE_CHECKING: + from collections.abc import Callable + + from pytest_mock import MockerFixture + + from hatch_pep770.cyclonedx import Component, Document, Workflow + from tests.conftest import BundleWheel, InstallTool + +_BUNDLED: Final[str] = "src/example/wheels/*.whl" +_URLS: Final[dict[str, str]] = {"Source": "https://github.com/example/example"} + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason="hatchling 1.28 needs Python 3.10") +def test_wheel_embeds_sbom( + build_wheel: Callable[[dict[str, Any], str], Path], config: dict[str, Any] +) -> None: # pragma: >=3.10 cover + with zipfile.ZipFile(build_wheel(config, "dist")) as archive: + names: Final[list[str]] = archive.namelist() + record: Final[str] = archive.read("example-1.0.dist-info/RECORD").decode("utf-8") + assert ("example-1.0.dist-info/sboms/example.cdx.json" in names, "sboms/example.cdx.json," in record) == ( + True, + True, + ) + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason="hatchling 1.28 needs Python 3.10") +def test_wheel_sbom_valid( + build_wheel: Callable[[dict[str, Any], str], Path], + bundle_wheel: BundleWheel, + config: dict[str, Any], +) -> None: # pragma: >=3.10 cover + bundle_wheel({"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}, "License-Expression: MIT\n") + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + assert wheel_problems(build_wheel(config, "dist"), frozenset({"MIT"})) == [] + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason="hatchling 1.28 needs Python 3.10") +def test_wheel_reproducible( + build_wheel: Callable[[dict[str, Any], str], Path], config: dict[str, Any] +) -> None: # pragma: >=3.10 cover + assert build_wheel(config, "first").read_bytes() == build_wheel(config, "second").read_bytes() + + +@pytest.mark.skipif(sys.version_info >= (3, 10), reason="Python 3.10 and later get hatchling 1.28") +def test_wheel_old_hatchling( + build_wheel: Callable[[dict[str, Any], str], Path], config: dict[str, Any], capsys: pytest.CaptureFixture[str] +) -> None: # pragma: <3.10 cover + with zipfile.ZipFile(build_wheel(config, "dist")) as archive: + sboms: Final[list[str]] = [name for name in archive.namelist() if "/sboms/" in name] + assert (sboms, "cannot embed SBOMs" in capsys.readouterr().err) == ([], True) + + +def test_sdist_target_skipped(root: Path, config: dict[str, Any], capsys: pytest.CaptureFixture[str]) -> None: + (artifact,) = SdistBuilder(str(root), config=config).build(directory=str(root / "dist")) + assert (Path(artifact).name, capsys.readouterr().err) == ("example-1.0.tar.gz", "") + + +def test_root_component(sbom_document: Document) -> None: + attestation: Final[str] = "https://pypi.org/integrity/example/1.0/example-1.0-py3-none-any.whl/provenance" + assert sbom_document["metadata"]["component"] == { + "type": "application", + "bom-ref": "pkg:pypi/example@1.0", + "authors": [{"name": "Maintainer", "email": "maintainer@example.com"}], + "name": "example", + "version": "1.0", + "description": "an example", + "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], + "copyright": "Copyright (c) 2026 Example", + "purl": "pkg:pypi/example@1.0", + "externalReferences": [ + {"type": "release-notes", "url": "https://example.com/changelog", "comment": "Project-URL: Changelog"}, + {"type": "vcs", "url": "https://github.com/example/example", "comment": "Project-URL: Source"}, + {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, + {"type": "attestation", "url": attestation}, + {"type": "security-contact", "url": "https://github.com/example/example/security/policy"}, + {"type": "advisories", "url": "https://github.com/example/example/security/advisories"}, + {"type": "license", "url": "https://github.com/example/example/blob/main/LICENSE"}, + ], + "properties": [ + {"name": "example:requires-python", "value": ">=3.9"}, + {"name": "python:classifier", "value": "Programming Language :: Python :: 3"}, + {"name": "python:keyword", "value": "sample"}, + ], + } + + +def test_root_component_minimal_metadata( + root: Path, build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any] +) -> None: + (root / "LICENSE").write_text("MIT terms\n", encoding="utf-8") + config["project"] = { + "name": "example", + "version": "1.0", + "license-files": ["LICENSE"], + "authors": [{"name": "Author"}], + } + assert json.loads(build_sbom(config))["metadata"]["component"] == { + "type": "application", + "bom-ref": "pkg:pypi/example@1.0", + "authors": [{"name": "Author"}], + "name": "example", + "version": "1.0", + "purl": "pkg:pypi/example@1.0", + "externalReferences": [ + {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, + { + "type": "attestation", + "url": "https://pypi.org/integrity/example/1.0/example-1.0-py2.py3-none-any.whl/provenance", + }, + ], + "properties": [], + } + + +def test_document_metadata(sbom_document: Document) -> None: + assert {key: value for key, value in sbom_document["metadata"].items() if key not in {"tools", "component"}} == { + "timestamp": "2020-02-02T00:00:00+00:00", + "lifecycles": [{"phase": "build"}], + "authors": [{"name": "Maintainer", "email": "maintainer@example.com"}], + "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], + "properties": [ + { + "name": "example:sbom:scope", + "value": "Components are the direct runtime dependencies declared in the wheel's core metadata. They" + " carry no version because they are resolved at install time, not at build time.", + } + ], + } + + +def test_supplier(build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any]) -> None: + supplier: Final[dict[str, Any]] = {"name": "Example Org", "url": ["https://example.org"]} + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"supplier": supplier} + document: Final[dict[str, Any]] = json.loads(build_sbom(config)) + assert [ + document["metadata"]["manufacturer"], + document["metadata"]["supplier"], + document["metadata"]["component"]["supplier"], + ] == [supplier] * 3 + + +def test_declared_dependencies(sbom_document: Document) -> None: + assert (sbom_document["components"], sbom_document["dependencies"][0]) == ( + [ + { + "type": "library", + "bom-ref": "requires-dist:packaging>=24", + "name": "packaging", + "purl": "pkg:pypi/packaging", + "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/packaging/"}], + "properties": [{"name": "example:requires-dist", "value": "packaging>=24"}], + }, + { + "type": "library", + "bom-ref": "requires-dist:tomli>=2; python_version < '3.11'", + "name": "tomli", + "purl": "pkg:pypi/tomli", + "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/tomli/"}], + "properties": [ + {"name": "example:requires-dist", "value": "tomli>=2; python_version < '3.11'"}, + {"name": "example:environment-marker", "value": 'python_version < "3.11"'}, + ], + }, + ], + { + "ref": "pkg:pypi/example@1.0", + "dependsOn": ["requires-dist:packaging>=24", "requires-dist:tomli>=2; python_version < '3.11'"], + }, + ) + + +def test_composition_incomplete(sbom_document: Document) -> None: + assert sbom_document["compositions"] == [{"aggregate": "incomplete", "dependencies": ["pkg:pypi/example@1.0"]}] + + +def test_build_workflow(sbom_document: Document) -> None: + workflow: Final[Workflow] = sbom_document["formulation"][0]["workflows"][0] + assert {**workflow, "resourceReferences": len(workflow["resourceReferences"])} == { + "bom-ref": "workflow:wheel-build", + "uid": "wheel-build", + "name": "build the wheel and this SBOM", + "taskTypes": ["build"], + "resourceReferences": len(sbom_document["metadata"]["tools"]["components"]), + "outputs": [{"type": "artifact", "resource": {"ref": "pkg:pypi/example@1.0"}}], + "inputs": [{"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": "1580601600"}]}], + } + + +def test_build_tools_name_interpreter_and_hook(sbom_document: Document) -> None: + tools: Final[list[Component]] = sbom_document["metadata"]["tools"]["components"] + assert (tools[0]["type"], any(tool["name"] == "hatch-pep770" for tool in tools)) == ("platform", True) + + +def test_bundled_wheel( + build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] +) -> None: + wheel: Final[Path] = bundle_wheel({}, "License-Expression: MIT\n") + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + sha256: Final[str] = hashlib.sha256(wheel.read_bytes()).hexdigest() + assert json.loads(build_sbom(config))["components"][0] == { + "type": "library", + "bom-ref": "pkg:pypi/pip@1.0", + "name": "pip", + "version": "1.0", + "purl": "pkg:pypi/pip@1.0", + "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], + "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/pip/1.0/"}], + "properties": [{"name": "example:bundled-wheel", "value": "src/example/wheels/pip-1.0-py3-none-any.whl"}], + "hashes": [{"alg": "SHA-256", "content": sha256}], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [ + {"technique": "manifest-analysis", "confidence": 1, "value": "pip-1.0.dist-info/METADATA"} + ], + }, + { + "field": "hash", + "confidence": 1, + "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], + }, + ] + }, + "components": [ + { + "type": "file", + "bom-ref": "pkg:pypi/pip@1.0#pip/example.py", + "name": "pip/example.py", + "hashes": [ + {"alg": "SHA-256", "content": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} + ], + "properties": [{"name": "size", "value": "3"}], + } + ], + } + + +def test_bundled_wheel_scope( + build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] +) -> None: + bundle_wheel({}) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + assert json.loads(build_sbom(config))["metadata"]["properties"] == [ + { + "name": "example:sbom:scope", + "value": "Components are the wheels bundled as data files under src/example/wheels and the direct runtime" + " dependencies declared in the wheel's core metadata. The latter carry no version because they are" + " resolved at install time, not at build time.", + } + ] + + +def test_bundled_wheels_sorted_and_first( + build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] +) -> None: + bundle_wheel({}, name="setuptools") + bundle_wheel({}, name="pip") + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED, "src/example/wheels/pip-*"]} + document: Final[dict[str, Any]] = json.loads(build_sbom(config)) + assert document["dependencies"][0]["dependsOn"] == [ + "pkg:pypi/pip@1.0", + "pkg:pypi/setuptools@1.0", + "requires-dist:packaging>=24", + "requires-dist:tomli>=2; python_version < '3.11'", + ] + + +@pytest.mark.parametrize( + ("source", "urls", "expected"), + [ + pytest.param("https://github.com/example/pip", _URLS, True, id="same-account"), + pytest.param("https://github.com/other/pip", _URLS, False, id="other-account"), + pytest.param("https://github.com/example/pip", {}, False, id="no-repository"), + ], +) +def test_bundled_wheel_supplier( + build_sbom: Callable[[dict[str, Any]], str], + bundle_wheel: BundleWheel, + config: dict[str, Any], + source: str, + urls: dict[str, str], + expected: bool, +) -> None: + bundle_wheel({}, f"Project-URL: Source, {source}\n") + config["project"]["urls"] = urls + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = { + "bundled-wheels": [_BUNDLED], + "supplier": {"name": "Example Org"}, + } + assert ("supplier" in json.loads(build_sbom(config))["components"][0]) is expected + + +@pytest.mark.parametrize( + "filename", + [ + pytest.param("pip/example.py", id="plain"), + pytest.param("pip/a,b.py", id="comma"), + pytest.param('pip/a"b.py', id="quote"), + pytest.param("pip/a\nb.py", id="newline"), + ], +) +def test_bundled_wheel_record_paths( + root: Path, build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], filename: str +) -> None: + record: Final[StringIO] = StringIO(newline="") + csv.writer(record).writerows([ + (filename, "sha256=ungWv48Bz-pBQUDeXa4iI7ADYaOWF3qctBD_YfIAFa0", "3"), + ("pip-1.0.dist-info/RECORD", "", ""), + ]) + (wheels := root / "src" / "example" / "wheels").mkdir(parents=True) + with zipfile.ZipFile(wheels / "pip-1.0-py3-none-any.whl", "w") as archive: + archive.writestr("pip-1.0.dist-info/METADATA", "Metadata-Version: 2.4\nName: pip\nVersion: 1.0\n") + archive.writestr("pip-1.0.dist-info/RECORD", record.getvalue()) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + assert json.loads(build_sbom(config))["components"][0]["components"] == [ + { + "type": "file", + "bom-ref": f"pkg:pypi/pip@1.0#{filename}", + "name": filename, + "hashes": [ + {"alg": "SHA-256", "content": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} + ], + "properties": [{"name": "size", "value": "3"}], + }, + ] + + +@pytest.mark.parametrize( + ("source", "content"), + [ + pytest.param( + "pip/_vendor/vendor.txt", "# bundled\n\n urllib3==1.26.4 # patched upstream\n", id="pip-manifest" + ), + pytest.param( + "pip/_vendor/urllib3-1.26.4.dist-info/METADATA", "Name: urllib3\nVersion: 1.26.4\n", id="nested-metadata" + ), + ], +) +def test_vendored_identity( + build_sbom: Callable[[dict[str, Any]], str], + bundle_wheel: BundleWheel, + config: dict[str, Any], + source: str, + content: str, +) -> None: + bundle_wheel({source: content}) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + document: Final[dict[str, Any]] = json.loads(build_sbom(config)) + assert [child for child in document["components"][0]["components"] if child["type"] == "library"] == [ + { + "type": "library", + "bom-ref": "pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4", + "name": "urllib3", + "version": "1.26.4", + "purl": "pkg:pypi/urllib3@1.26.4", + "externalReferences": [], + "properties": [{"name": "example:vendored-manifest", "value": source}], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], + } + ] + }, + } + ] + + +def test_vendored_relationship( + build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] +) -> None: + bundle_wheel({"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + assert json.loads(build_sbom(config))["dependencies"][1] == { + "ref": "pkg:pypi/pip@1.0", + "dependsOn": ["pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4"], + } + + +def test_vendored_metadata_precedence( + build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] +) -> None: + bundle_wheel({ + "pip/_vendor/vendor.txt": "urllib3==1.26.4\n", + "pip/_vendor/urllib3-1.26.4.dist-info/METADATA": "Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n", + }) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + document: Final[dict[str, Any]] = json.loads(build_sbom(config)) + assert [child["licenses"] for child in document["components"][0]["components"] if child["type"] == "library"] == [ + [{"expression": "MIT", "acknowledgement": "declared"}], + ] + + +@pytest.mark.parametrize( + "requirement", + [ + pytest.param("urllib3", id="missing"), + pytest.param("urllib3>=1", id="range"), + pytest.param("urllib3==1.*", id="wildcard"), + pytest.param("urllib3 @ https://example.com/package.whl", id="url"), + ], +) +def test_vendored_unpinned_rejected( + build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any], requirement: str +) -> None: + bundle_wheel({"pip/_vendor/vendor.txt": requirement}) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + with pytest.raises(ValueError, match="Unpinned vendored dependency"): + build_sbom(config) + + +@pytest.mark.parametrize( + ("metadata", "expected"), + [ + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\n", + [ + {"license": {"name": "MIT License", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ], + id="classifier", + ), + pytest.param( + "Classifier: License :: OSI Approved :: Python Software Foundation License\nLicense: PSF-2.0\n", + [ + {"license": {"name": "Python Software Foundation License", "acknowledgement": "declared"}}, + {"license": {"name": "PSF-2.0", "acknowledgement": "declared"}}, + {"license": {"id": "PSF-2.0", "acknowledgement": "concluded"}}, + ], + id="classifier-and-id-agree", + ), + pytest.param( + "License: mit\n", + [ + {"license": {"name": "mit", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ], + id="id-canonical-case", + ), + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\n" + "Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)\n", + [ + {"license": {"name": "MIT License", "acknowledgement": "declared"}}, + {"license": {"name": "GNU Lesser General Public License v3 (LGPLv3)", "acknowledgement": "declared"}}, + ], + id="one-unresolved", + ), + pytest.param( + "License: MIT OR Apache-2.0\n", + [{"license": {"name": "MIT OR Apache-2.0", "acknowledgement": "declared"}}], + id="expression-in-license-field", + ), + pytest.param( + "License: LicenseRef-Proprietary\n", + [{"license": {"name": "LicenseRef-Proprietary", "acknowledgement": "declared"}}], + id="license-ref", + ), + ], +) +def test_license_resolution( + build_sbom: Callable[[dict[str, Any]], str], + bundle_wheel: BundleWheel, + config: dict[str, Any], + metadata: str, + expected: list[dict[str, Any]], +) -> None: + bundle_wheel({}, metadata) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + assert json.loads(build_sbom(config))["components"][0]["licenses"] == expected + + +def test_bundled_wheel_metadata_fields( + build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] +) -> None: + bundle_wheel( + {}, + "Summary: installer\nHome-page: https://pip.pypa.io\nAuthor: Someone\nAuthor-email: dev@example.com\n" + "Maintainer-email: Team <>\nProject-URL: Bug Tracker, https://example.com/issues\n", + ) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + component: Final[dict[str, Any]] = json.loads(build_sbom(config))["components"][0] + assert {key: component[key] for key in ("description", "authors", "externalReferences")} == { + "description": "installer", + "authors": [{"name": "Someone"}, {"email": "dev@example.com"}, {"name": "Team"}], + "externalReferences": [ + {"type": "website", "url": "https://pip.pypa.io"}, + {"type": "issue-tracker", "url": "https://example.com/issues", "comment": "Project-URL: Bug Tracker"}, + {"type": "distribution", "url": "https://pypi.org/project/pip/1.0/"}, + ], + } + + +@pytest.mark.parametrize( + ("environment", "expected"), + [ + pytest.param({}, ("2020-02-02T00:00:00+00:00", "1580601600"), id="hatchling-default"), + pytest.param( + {"SOURCE_DATE_EPOCH": "1700000000"}, ("2023-11-14T22:13:20+00:00", "1700000000"), id="source-date-epoch" + ), + ], +) +def test_timestamp( + build_sbom: Callable[[dict[str, Any]], str], + config: dict[str, Any], + mocker: MockerFixture, + environment: dict[str, str], + expected: tuple[str, str], +) -> None: + mocker.patch.dict(os.environ, environment) + document: Final[dict[str, Any]] = json.loads(build_sbom(config)) + workflow: Final[dict[str, Any]] = document["formulation"][0]["workflows"][0] + assert (document["metadata"]["timestamp"], workflow["inputs"][0]["environmentVars"][0]["value"]) == expected + + +def test_build_host_independent( + build_sbom: Callable[[dict[str, Any]], str], + config: dict[str, Any], + install_tool: InstallTool, + mocker: MockerFixture, + tmp_path: Path, +) -> None: + sboms: Final[list[str]] = [] + for host, extension, installer, sys_version in ( + ( + { + "system": "Linux", + "release": "6.17.0-1022-azure", + "version": "#22~24.04.1-Ubuntu SMP", + "machine": "x86_64", + "platform": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", + "python_compiler": "GCC 13.3.0", + "python_build": ("main", "Aug 5 2026 10:00:00"), + }, + "cpython-314-x86_64-linux-gnu.so", + b"uv\n", + "3.14.7 (main, Aug 5 2026, 10:00:00) [GCC 13.3.0]", + ), + ( + { + "system": "Linux", + "release": "7.0.12-linuxkit", + "version": "#1 SMP PREEMPT", + "machine": "aarch64", + "platform": "Linux-7.0.12-linuxkit-aarch64-with-glibc2.36", + "python_compiler": "Clang 20.1.4", + "python_build": ("main", "Sep 1 2026 08:00:00"), + }, + "cpython-314-aarch64-linux-gnu.so", + b"pip\n", + "3.14.7 (main, Sep 1 2026, 08:00:00) [Clang 20.1.4]", + ), + ): + for name, value in host.items(): + mocker.patch(f"platform.{name}", autospec=True, return_value=value) + mocker.patch.object(sys, "version", sys_version) + install_tool(site := tmp_path / extension, "native", False, {f"native/_speedups.{extension}": b"\0"}) + install_tool(site, "pure", True, {"pure-1.0.dist-info/INSTALLER": installer, "pure/__init__.py": b""}) + mocker.patch.object(sys, "path", [str(site), *sys.path]) + sboms.append(build_sbom(config)) + mocker.stopall() + assert sboms[0] == sboms[1] + + +def test_build_tools_pure_files( + build_sbom: Callable[[dict[str, Any]], str], + config: dict[str, Any], + install_tool: InstallTool, + mocker: MockerFixture, + tmp_path: Path, +) -> None: + install_tool( + site := tmp_path / "site", + "pure", + True, + {"pure-1.0.dist-info/INSTALLER": b"uv\n", "pure/__init__.py": b"", "../../bin/pure": b"#!python\n"}, + ) + install_tool(site, "native", False, {"native/_speedups.so": b"\0"}) + mocker.patch.object(sys, "path", [str(site), *sys.path]) + tools: Final[dict[str, dict[str, Any]]] = { + tool["name"]: tool for tool in json.loads(build_sbom(config))["metadata"]["tools"]["components"] + } + assert ([file["name"] for file in tools["pure"]["components"]], "components" in tools["native"]) == ( + ["pure/__init__.py"], + False, + ) + + +def test_ci_rerun( + build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("GITHUB_SHA", "a" * 40) + monkeypatch.setenv("GITHUB_RUN_ID", "100") + first: Final[str] = build_sbom(config) + monkeypatch.setenv("GITHUB_SHA", "b" * 40) + monkeypatch.setenv("GITHUB_RUN_ID", "200") + assert build_sbom(config) == first + + +def test_source_revision( + root: Path, build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + for name in ("GIT_AUTHOR_NAME", "GIT_COMMITTER_NAME"): + monkeypatch.setenv(name, "Example") + for name in ("GIT_AUTHOR_EMAIL", "GIT_COMMITTER_EMAIL"): + monkeypatch.setenv(name, "example@example.com") + for command in (["init", "-q"], ["add", "."], ["commit", "-q", "--no-gpg-sign", "-m", "init"]): + subprocess.run(["git", *command], cwd=root, check=True) + commit: Final[str] = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=root, check=True, capture_output=True, text=True + ).stdout.strip() + component: Final[dict[str, Any]] = json.loads(build_sbom(config))["metadata"]["component"] + assert (component["externalReferences"][-1], component["properties"][-1]) == ( + {"type": "vcs", "url": f"https://github.com/example/example/tree/{commit}", "comment": "exact source revision"}, + {"name": "example:vcs-commit", "value": commit}, + ) + + +def test_source_revision_without_repository( + root: Path, build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], mocker: MockerFixture +) -> None: + (root / ".git").mkdir() + mocker.patch( + "subprocess.run", + autospec=True, + return_value=subprocess.CompletedProcess(["git"], 0, stdout="c0ffee\n", stderr=""), + ) + del config["project"]["urls"] + component: Final[dict[str, Any]] = json.loads(build_sbom(config))["metadata"]["component"] + assert ([ref["type"] for ref in component["externalReferences"]], component["properties"][-1]) == ( + ["distribution", "attestation"], + {"name": "example:vcs-commit", "value": "c0ffee"}, + ) + + +def test_source_revision_unreadable( + root: Path, + build_sbom: Callable[[dict[str, Any]], str], + config: dict[str, Any], + mocker: MockerFixture, + capsys: pytest.CaptureFixture[str], +) -> None: + (root / ".git").mkdir() + mocker.patch( + "subprocess.run", + autospec=True, + side_effect=subprocess.CalledProcessError(128, ["git"], stderr="fatal: detected dubious ownership"), + ) + component: Final[dict[str, Any]] = json.loads(build_sbom(config))["metadata"]["component"] + assert ( + [prop["name"] for prop in component["properties"]], + "names no source revision: fatal: detected dubious ownership" in capsys.readouterr().err, + ) == (["example:requires-python", "python:classifier", "python:keyword"], True) + + +@pytest.mark.parametrize( + ("hook_config", "expected"), + [ + pytest.param( + {"repository": "https://gitlab.com/example/example/"}, + [ + {"type": "release-notes", "url": "https://example.com/changelog", "comment": "Project-URL: Changelog"}, + {"type": "vcs", "url": "https://github.com/example/example", "comment": "Project-URL: Source"}, + {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, + {"type": "license", "url": "https://gitlab.com/example/example/blob/main/LICENSE"}, + ], + id="repository-off-github", + ), + pytest.param( + {"default-branch": "trunk"}, + [ + {"type": "release-notes", "url": "https://example.com/changelog", "comment": "Project-URL: Changelog"}, + {"type": "vcs", "url": "https://github.com/example/example", "comment": "Project-URL: Source"}, + {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, + {"type": "security-contact", "url": "https://github.com/example/example/security/policy"}, + {"type": "advisories", "url": "https://github.com/example/example/security/advisories"}, + {"type": "license", "url": "https://github.com/example/example/blob/trunk/LICENSE"}, + ], + id="default-branch", + ), + ], +) +def test_repository_links( + build_sbom: Callable[[dict[str, Any]], str], + config: dict[str, Any], + hook_config: dict[str, Any], + expected: list[dict[str, str]], +) -> None: + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = hook_config + references: Final[list[dict[str, str]]] = json.loads(build_sbom(config))["metadata"]["component"][ + "externalReferences" + ] + assert [reference for reference in references if reference["type"] != "attestation"] == expected + + +def test_serial_number_per_project(build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any]) -> None: + first: Final[str] = json.loads(build_sbom(config))["serialNumber"] + del config["project"]["urls"] + assert json.loads(build_sbom(config))["serialNumber"] != first + + +@pytest.mark.parametrize( + ("build_data", "tag"), + [ + pytest.param({"tag": "py3-none-linux_x86_64"}, "py3-none-linux_x86_64", id="set-by-another-hook"), + pytest.param({"infer_tag": True}, None, id="inferred"), + ], +) +def test_attestation_wheel_tag(root: Path, config: dict[str, Any], build_data: dict[str, Any], tag: str | None) -> None: + builder: Final[WheelBuilder] = WheelBuilder(str(root), config=config) + data: Final[dict[str, Any]] = {**builder.get_default_build_data(), "sbom_files": [], **build_data} + (hook,) = builder.get_build_hooks(str(root / "dist")).values() + hook.initialize("standard", data) + references: Final[list[dict[str, str]]] = json.loads(Path(data["sbom_files"][0]).read_text(encoding="utf-8"))[ + "metadata" + ]["component"]["externalReferences"] + expected: Final[str] = tag or builder.get_best_matching_tag() + assert [ref["url"] for ref in references if ref["type"] == "attestation"] == [ + f"https://pypi.org/integrity/example/1.0/example-1.0-{expected}.whl/provenance" + ] + + +def test_finalize_removes_sbom(root: Path, config: dict[str, Any]) -> None: + builder: Final[WheelBuilder] = WheelBuilder(str(root), config=config) + build_data: Final[dict[str, Any]] = {**builder.get_default_build_data(), "sbom_files": []} + (hook,) = builder.get_build_hooks(str(root / "dist")).values() + hook.initialize("standard", build_data) + hook.finalize("standard", build_data, "") + assert Path(build_data["sbom_files"][0]).parent.exists() is False + + +@pytest.mark.parametrize( + ("hook_config", "message"), + [ + pytest.param( + {"supplier": "Example"}, "Option `supplier` of build hook `pep770` must be a table", id="supplier" + ), + pytest.param({"supplier": {"name": "Example", "email": "x"}}, "Option `supplier`", id="supplier-unknown-key"), + pytest.param({"repository": 1}, "Option `repository` of build hook `pep770` must be a string", id="repository"), + pytest.param( + {"default-branch": 1}, "Option `default-branch` of build hook `pep770` must be a string", id="branch" + ), + pytest.param( + {"bundled-wheels": "*.whl"}, + "Option `bundled-wheels` of build hook `pep770` must be an array of strings", + id="bundled-wheels", + ), + ], +) +def test_invalid_option( + build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], hook_config: dict[str, Any], message: str +) -> None: + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = hook_config + with pytest.raises(TypeError, match=message): + build_sbom(config) + + +@pytest.mark.parametrize( + ("urls", "references"), + [ + pytest.param( + _URLS, + { + "externalReferences": [ + {"type": "vcs", "url": "https://github.com/example/example/blob/main/hatch_build.py"} + ] + }, + id="repository", + ), + pytest.param({}, {}, id="no-repository"), + ], +) +def test_custom_hook_subclass( + root: Path, + build_sbom: Callable[[dict[str, Any]], str], + bundle_wheel: BundleWheel, + config: dict[str, Any], + urls: dict[str, str], + references: dict[str, Any], +) -> None: + bundle_wheel({}) + (script := root / "hatch_build.py").write_text( + "from __future__ import annotations\n\nfrom hatch_pep770 import plugin\n\n\n" + "class Hook(plugin.Pep770BuildHook):\n" + " def bundled_wheel_properties(self, wheel):\n" + ' return [{"name": "example:installed-for-python", "value": wheel.name}]\n\n\n' + "def get_build_hook():\n return Hook\n", + encoding="utf-8", + ) + config["project"]["urls"] = urls + config["tool"]["hatch"]["build"]["hooks"] = {"custom": {"bundled-wheels": [_BUNDLED]}} + document: Final[dict[str, Any]] = json.loads(build_sbom(config)) + assert (document["metadata"]["tools"]["components"][0], document["components"][0]["properties"][:2]) == ( + { + "type": "application", + "bom-ref": "tool:hatch_build.py", + "name": "hatch_build.py", + "version": "1.0", + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(script.read_bytes()).hexdigest()}], + **references, + }, + [ + {"name": "example:bundled-wheel", "value": "src/example/wheels/pip-1.0-py3-none-any.whl"}, + {"name": "example:installed-for-python", "value": "pip-1.0-py3-none-any.whl"}, + ], + ) diff --git a/tests/test_spdx.py b/tests/test_spdx.py new file mode 100644 index 0000000..e72aeab --- /dev/null +++ b/tests/test_spdx.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Final + +import pytest + +from hatch_pep770.spdx import to_spdx +from hatch_pep770.version import __version__ + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hatch_pep770.cyclonedx import Document + from tests.conftest import BundleWheel + +_BUNDLED: Final[str] = "src/example/wheels/*.whl" + + +@pytest.fixture +def bundled_config(bundle_wheel: BundleWheel, config: dict[str, Any]) -> dict[str, Any]: + bundle_wheel( + {"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}, "Project-URL: Source, https://github.com/example/pip\n" + ) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = { + "bundled-wheels": [_BUNDLED], + "supplier": {"name": "Example Org"}, + } + return config + + +def test_spdx_bundled_wheel(build_sbom: Callable[[dict[str, Any]], str], bundled_config: dict[str, Any]) -> None: + cyclonedx: Final[Document] = json.loads(build_sbom(bundled_config)) + assert to_spdx(cyclonedx)["packages"][1] == { + "SPDXID": "SPDXRef-pkg-pypi-pip-1.0", + "name": "pip", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "primaryPackagePurpose": "LIBRARY", + "versionInfo": "1.0", + "supplier": "Organization: Example Org", + "checksums": [{"algorithm": "SHA256", "checksumValue": cyclonedx["components"][0]["hashes"][0]["content"]}], + "comment": "example:bundled-wheel: src/example/wheels/pip-1.0-py3-none-any.whl", + "externalRefs": [ + {"referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", "referenceLocator": "pkg:pypi/pip@1.0"} + ], + } + + +def test_spdx_root_package(sbom_document: Document) -> None: + assert to_spdx(sbom_document)["packages"][0] == { + "SPDXID": "SPDXRef-pkg-pypi-example-1.0", + "name": "example", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "primaryPackagePurpose": "APPLICATION", + "versionInfo": "1.0", + "copyrightText": "Copyright (c) 2026 Example", + "summary": "an example", + "comment": "example:requires-python: >=3.9", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:pypi/example@1.0", + } + ], + } + + +def test_spdx_relationships(build_sbom: Callable[[dict[str, Any]], str], bundled_config: dict[str, Any]) -> None: + assert [ + relationship + for relationship in to_spdx(_load(build_sbom(bundled_config)))["relationships"] + if not relationship["spdxElementId"].startswith("SPDXRef-tool-") + ] == [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-pkg-pypi-example-1.0", + }, + { + "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0-vendored-pkg-pypi-urllib3-1.26.4", + }, + { + "spdxElementId": "SPDXRef-pkg-pypi-example-1.0", + "relationshipType": "DEPENDS_ON", + "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0", + }, + { + "spdxElementId": "SPDXRef-pkg-pypi-example-1.0", + "relationshipType": "DEPENDS_ON", + "relatedSpdxElement": "SPDXRef-requires-dist-packaging-24", + }, + { + "spdxElementId": "SPDXRef-pkg-pypi-example-1.0", + "relationshipType": "DEPENDS_ON", + "relatedSpdxElement": "SPDXRef-requires-dist-tomli-2-python-version-3.11", + }, + { + "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0", + "relationshipType": "DEPENDS_ON", + "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0-vendored-pkg-pypi-urllib3-1.26.4", + }, + ] + + +def test_spdx_build_tools(sbom_document: Document) -> None: + assert { + relationship["relatedSpdxElement"] + for relationship in to_spdx(sbom_document)["relationships"] + if relationship["relationshipType"] == "BUILD_TOOL_OF" + } == {"SPDXRef-pkg-pypi-example-1.0"} + + +@pytest.mark.parametrize( + ("metadata", "expected"), + [ + pytest.param("License-Expression: MIT\n", ("MIT", "NOASSERTION", None), id="expression"), + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\nLicense: MIT\n", + ("NOASSERTION", "MIT", "Declared in package metadata as: MIT License; MIT"), + id="names", + ), + pytest.param( + "License: LGPLv3\n", + ("NOASSERTION", "NOASSERTION", "Declared in package metadata as: LGPLv3"), + id="unresolved-name", + ), + pytest.param("", ("NOASSERTION", "NOASSERTION", None), id="undeclared"), + ], +) +def test_spdx_declared_license( + build_sbom: Callable[[dict[str, Any]], str], + bundle_wheel: BundleWheel, + config: dict[str, Any], + metadata: str, + expected: tuple[str, str, str | None], +) -> None: + bundle_wheel({}, metadata) + config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} + package: Final[dict[str, Any]] = dict(to_spdx(_load(build_sbom(config)))["packages"][1]) + assert (package["licenseDeclared"], package["licenseConcluded"], package.get("licenseComments")) == expected + + +@pytest.mark.parametrize( + ("urls", "hooks", "expected"), + [ + pytest.param( + {"Source": "https://github.com/example/example"}, + {"pep770": {"supplier": {"name": "Example Org"}}}, + ("https://github.com/example/example/sboms/example-1.0-", ["Organization: Example Org"]), + id="repository-and-supplier", + ), + pytest.param({}, {"pep770": {}}, ("https://spdx.org/spdxdocs/sboms/example-1.0-", []), id="neither"), + pytest.param( + {}, + {"custom": {}}, + ("https://spdx.org/spdxdocs/sboms/example-1.0-", ["Tool: hatch_build.py-1.0"]), + id="custom-build-script", + ), + ], +) +def test_spdx_creation_info( + root: Path, + build_sbom: Callable[[dict[str, Any]], str], + config: dict[str, Any], + monkeypatch: pytest.MonkeyPatch, + urls: dict[str, str], + hooks: dict[str, Any], + expected: tuple[str, list[str]], +) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + (root / "hatch_build.py").write_text( + "from hatch_pep770 import plugin\n\n\ndef get_build_hook():\n return plugin.Pep770BuildHook\n", + encoding="utf-8", + ) + config["project"]["urls"] = urls + config["tool"]["hatch"]["build"]["hooks"] = hooks + cyclonedx: Final[Document] = json.loads(build_sbom(config)) + document: Final[dict[str, Any]] = dict(to_spdx(cyclonedx)) + namespace, creators = expected + assert ( + document["documentNamespace"], + document["creationInfo"], + document["comment"], + ) == ( + f"{namespace}{cyclonedx['serialNumber'].removeprefix('urn:uuid:')}", + {"created": "2023-11-14T22:13:20Z", "creators": [*creators, f"Tool: hatch-pep770-{__version__}"]}, + ( + f"Rendered from the CycloneDX SBOM {cyclonedx['serialNumber']} (example.cdx.json), which also carries" + " per-file hashes and the build formulation." + ), + ) + + +def _load(content: str) -> Document: + return json.loads(content) diff --git a/tests/test_validate.py b/tests/test_validate.py new file mode 100644 index 0000000..d526e3b --- /dev/null +++ b/tests/test_validate.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import json +import zipfile +from typing import TYPE_CHECKING, Final + +import pytest + +from hatch_pep770.validate import document_problems, license_problems, wheel_problems + +if TYPE_CHECKING: + from pathlib import Path + + from hatch_pep770.cyclonedx import Component, Document, License + +_ALLOWED: Final[frozenset[str]] = frozenset({"Apache-2.0", "BSD-3-Clause", "MIT"}) + + +def _expression(expression: str) -> License: + return {"expression": expression, "acknowledgement": "declared"} + + +def _named(name: str) -> License: + return {"license": {"name": name, "acknowledgement": "declared"}} + + +def _concluded(identifier: str) -> License: + return {"license": {"id": identifier, "acknowledgement": "concluded"}} + + +def _denied(expression: str, bom_ref: str = "pkg:pypi/example@1.0") -> str: + return f"{bom_ref}: license {expression} is outside the allowed licenses ['Apache-2.0', 'BSD-3-Clause', 'MIT']" + + +def test_document_valid(sbom_document: Document) -> None: + assert document_problems(json.dumps(sbom_document), _ALLOWED) == [] + + +def test_document_schema_violation() -> None: + problems: Final[list[str]] = document_problems( + json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "1"}), None + ) + assert [problem.splitlines()[0] for problem in problems] == [ + "does not conform to the CycloneDX 1.6 schema: '1' is not of type 'integer'" + ] + + +def test_document_serial_number_missing(sbom_document: Document) -> None: + unnumbered: Final[dict[str, object]] = {key: value for key, value in sbom_document.items() if key != "serialNumber"} + assert document_problems(json.dumps(unnumbered), None) == ["serialNumber must be a urn:uuid: RFC 4122 UUID, got ''"] + + +def test_document_duplicate_bom_ref(sbom_document: Document) -> None: + sbom_document["components"][1]["bom-ref"] = sbom_document["components"][0]["bom-ref"] + assert document_problems(json.dumps(sbom_document), None)[0] == ( + "bom-ref values are not unique: ['requires-dist:packaging>=24']" + ) + + +def test_document_dangling_dependency(sbom_document: Document) -> None: + sbom_document["dependencies"].append({"ref": "pkg:pypi/example@1.0", "dependsOn": ["pkg:pypi/missing"]}) + problems: Final[list[str]] = document_problems(json.dumps(sbom_document), None) + assert problems[0] == "dependencies reference bom-refs that do not exist: ['pkg:pypi/missing']" + + +def test_document_root_dependencies(sbom_document: Document) -> None: + sbom_document["dependencies"][0]["dependsOn"] = ["requires-dist:packaging>=24"] + assert document_problems(json.dumps(sbom_document), None) == [ + ( + "root dependsOn ['requires-dist:packaging>=24'] != components" + " ['requires-dist:packaging>=24', \"requires-dist:tomli>=2; python_version < '3.11'\"]" + ) + ] + + +def test_document_workflow_unknown_tool(sbom_document: Document) -> None: + sbom_document["formulation"][0]["workflows"][0]["resourceReferences"].append({"ref": "tool:missing"}) + assert document_problems(json.dumps(sbom_document), None) == [ + "workflow wheel-build references unknown tools: ['tool:missing']" + ] + + +def test_document_spdx_unparsable(sbom_document: Document) -> None: + sbom_document["metadata"]["component"]["hashes"] = [{"alg": "BLAKE2b-256", "content": "0" * 64}] + assert document_problems(json.dumps(sbom_document), None)[0].startswith("SPDX rendering does not parse:") + + +def test_document_spdx_invalid(sbom_document: Document) -> None: + sbom_document["metadata"]["component"]["purl"] = "not a purl" + assert document_problems(json.dumps(sbom_document), None)[0].startswith("SPDX rendering invalid:") + + +def test_wheel_without_sbom(tmp_path: Path) -> None: + with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: + archive.writestr("example-1.0.dist-info/METADATA", "Name: example\n") + assert wheel_problems(wheel, None) == ["no .dist-info/sboms/*.cdx.json found in the wheel"] + + +def test_wheel_names_each_sbom(sbom_document: Document, tmp_path: Path) -> None: + unnumbered: Final[dict[str, object]] = {key: value for key, value in sbom_document.items() if key != "serialNumber"} + with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: + archive.writestr("example-1.0.dist-info/sboms/example.cdx.json", json.dumps(unnumbered)) + assert wheel_problems(wheel, None) == [ + "example-1.0.dist-info/sboms/example.cdx.json: serialNumber must be a urn:uuid: RFC 4122 UUID, got ''" + ] + + +@pytest.mark.parametrize( + ("licenses", "expected"), + [ + pytest.param([_expression("MIT")], [], id="allowed"), + pytest.param([_expression("mit")], [], id="allowed-any-case"), + pytest.param([_expression("GPL-3.0-only")], [_denied("GPL-3.0-only")], id="denied"), + pytest.param([_expression("Apache-2.0 OR GPL-3.0-only")], [], id="or-one-allowed"), + pytest.param( + [_expression("GPL-3.0-only OR MPL-2.0")], [_denied("GPL-3.0-only OR MPL-2.0")], id="or-none-allowed" + ), + pytest.param([_expression("MIT AND BSD-3-Clause")], [], id="and-all-allowed"), + pytest.param([_expression("MIT AND GPL-3.0-only")], [_denied("MIT AND GPL-3.0-only")], id="and-one-denied"), + pytest.param([_expression("MIT OR GPL-3.0-only AND LGPL-3.0-only")], [], id="and-binds-tighter"), + pytest.param( + [_expression("(MIT OR GPL-3.0-only) AND LGPL-3.0-only")], + [_denied("(MIT OR GPL-3.0-only) AND LGPL-3.0-only")], + id="parentheses", + ), + pytest.param([_expression("Apache-2.0 WITH LLVM-exception")], [], id="with-allowed"), + pytest.param( + [_expression("GPL-2.0-only WITH Classpath-exception-2.0")], + [_denied("GPL-2.0-only WITH Classpath-exception-2.0")], + id="with-denied", + ), + pytest.param([_named("MIT License"), _concluded("MIT")], [], id="concluded-allowed"), + pytest.param([_named("MPL 2.0"), _concluded("MPL-2.0")], [_denied("MPL-2.0")], id="concluded-denied"), + pytest.param( + [_concluded("MIT"), _concluded("LGPL-3.0-only")], [_denied("LGPL-3.0-only")], id="concluded-all-apply" + ), + pytest.param( + [_named("MIT License")], + ["pkg:pypi/example@1.0: declares license names ['MIT License'] with no SPDX id"], + id="name-only", + ), + pytest.param([], ["pkg:pypi/example@1.0: declares no license"], id="undeclared"), + ], +) +def test_license_policy(licenses: list[License], expected: list[str]) -> None: + component: Final[Component] = { + "type": "library", + "bom-ref": "pkg:pypi/example@1.0", + "name": "example", + "hashes": [{"alg": "SHA-256", "content": "0" * 64}], + "licenses": licenses, + } + assert license_problems([component], _ALLOWED) == expected + + +@pytest.mark.parametrize( + ("expression", "reason"), + [ + pytest.param("MIT AND", "it ends early", id="trailing-operator"), + pytest.param("OR MIT", "unexpected 'OR'", id="leading-operator"), + pytest.param("(MIT", "it ends early", id="unclosed"), + pytest.param("MIT)", "unexpected ')'", id="unopened"), + pytest.param("(MIT Apache-2.0)", "expected ')', got 'Apache-2.0'", id="missing-operator"), + pytest.param("MIT and Apache-2.0", "unexpected 'and'", id="lower-case-operator"), + pytest.param("Apache-2.0 WITH OR", "unexpected 'OR'", id="with-operator"), + ], +) +def test_license_policy_malformed(expression: str, reason: str) -> None: + component: Final[Component] = { + "type": "library", + "bom-ref": "pkg:pypi/example@1.0", + "name": "example", + "hashes": [{"alg": "SHA-256", "content": "0" * 64}], + "licenses": [_expression(expression)], + } + assert license_problems([component], _ALLOWED) == [ + f"pkg:pypi/example@1.0: license expression {expression!r} is malformed: {reason}" + ] + + +@pytest.mark.parametrize( + "component", + [ + pytest.param( + {"type": "library", "bom-ref": "requires-dist:distlib", "name": "distlib"}, id="declared-requirement" + ), + pytest.param( + { + "type": "library", + "bom-ref": "pkg:pypi/pip@1.0", + "name": "pip", + "hashes": [{"alg": "SHA-256", "content": "0" * 64}], + "licenses": [_expression("MIT")], + "components": [ + {"type": "library", "bom-ref": "pkg:pypi/pip@1.0#vendored/pkg:pypi/autocommand@2.2.2", "name": "a"}, + ], + }, + id="vendored-by-bundled-wheel", + ), + pytest.param( + { + "type": "file", + "bom-ref": "loader", + "name": "loader", + "hashes": [{"alg": "SHA-256", "content": "0" * 64}], + }, + id="file", + ), + ], +) +def test_license_policy_skips_unshipped(component: Component) -> None: + assert license_problems([component], _ALLOWED) == [] + + +def test_license_policy_judges_files_owner() -> None: + component: Final[Component] = { + "type": "library", + "bom-ref": "pkg:pypi/distlib@0.4", + "name": "distlib", + "licenses": [_concluded("LGPL-3.0-only")], + "components": [{"type": "file", "bom-ref": "pkg:pypi/distlib@0.4#distlib/__init__.py", "name": "x"}], + } + assert license_problems([component], _ALLOWED) == [_denied("LGPL-3.0-only", "pkg:pypi/distlib@0.4")] From 041efedf4754d2f6cf2fe8a0ac5441a4f820b3e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 18:35:03 -0800 Subject: [PATCH 003/115] =?UTF-8?q?=F0=9F=91=B7=20ci:=20add=20check,=20rel?= =?UTF-8?q?ease=20and=20security=20workflows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/check.yaml | 63 ++++++++++++++++++++++++++ .github/workflows/codeql.yaml | 22 +++++++++ .github/workflows/news.yaml | 30 ++++++++++++ .github/workflows/prepare-release.yaml | 33 ++++++++++++++ .github/workflows/release.yaml | 56 +++++++++++++++++++++++ .github/workflows/scorecard.yaml | 26 +++++++++++ 6 files changed, 230 insertions(+) create mode 100644 .github/workflows/check.yaml create mode 100644 .github/workflows/codeql.yaml create mode 100644 .github/workflows/news.yaml create mode 100644 .github/workflows/prepare-release.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/scorecard.yaml diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml new file mode 100644 index 0000000..55ebfc3 --- /dev/null +++ b/.github/workflows/check.yaml @@ -0,0 +1,63 @@ +name: ๐Ÿงช check +on: + workflow_dispatch: + push: + branches: ["main"] + tags-ignore: ["**"] + pull_request: + types: [opened, synchronize, reopened] + schedule: + - cron: "27 8 * * *" +concurrency: + group: check-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +permissions: + contents: read +jobs: + test: + name: ๐Ÿงช Test + uses: tox-dev/github-workflows/.github/workflows/check.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + permissions: + contents: read + with: + include: | + [ + {"env": "3.15"}, + {"env": "3.15t", "python": "3.15t"}, + {"env": "3.14", "python": "3.14"}, + {"env": "3.13", "python": "3.13"}, + {"env": "3.12", "python": "3.12"}, + {"env": "3.11", "python": "3.11"}, + {"env": "3.10", "python": "3.10"}, + {"env": "3.9", "python": "3.9"}, + {"env": "type"}, + {"env": "dev"}, + {"env": "pkg_meta"}, + {"env": "3.15", "os": "windows-2025-vs2026"}, + {"env": "3.14", "python": "3.14", "os": "windows-2025-vs2026"}, + {"env": "3.9", "python": "3.9", "os": "windows-2025-vs2026"}, + {"env": "3.15", "os": "macos-26"} + ] + release: + name: ๐Ÿท๏ธ Release dry run + uses: ./.github/workflows/prepare-release.yaml # zizmor: ignore[self-repository] actionlint rejects the $/ form + permissions: + contents: read + pull-requests: read + with: + dry-run: true + hooks: + name: ๐Ÿงน Hooks under the cooldown + uses: tox-dev/github-workflows/.github/workflows/cooled-hooks.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + permissions: + contents: read + gate: + name: ๐Ÿšฆ ${{ github.event_name == 'pull_request' && 'Required checks pass' || 'Non-PR checks pass' }} + if: ${{ !cancelled() }} # codespell:ignore cancelled + needs: [test, release, hooks] + runs-on: ubuntu-26.04 + steps: + - name: ๐Ÿšฆ Require all needed jobs to pass + uses: tox-dev/github-workflows/required-checks@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + with: + needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 0000000..de17f24 --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,22 @@ +name: ๐Ÿ›ก๏ธ CodeQL +on: + workflow_dispatch: + push: + branches: ["main"] + pull_request: + schedule: + - cron: "35 8 * * 2" +concurrency: + group: codeql-${{ github.ref }} + cancel-in-progress: true +permissions: {} +jobs: + analyze: + name: ๐Ÿ›ก๏ธ Analyze + uses: tox-dev/github-workflows/.github/workflows/codeql.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + permissions: + contents: read + security-events: write + with: + languages: '["actions","python"]' + paths-ignore: '["tests"]' diff --git a/.github/workflows/news.yaml b/.github/workflows/news.yaml new file mode 100644 index 0000000..c707bc0 --- /dev/null +++ b/.github/workflows/news.yaml @@ -0,0 +1,30 @@ +name: ๐Ÿ“ Changelog +on: + pull_request: + types: [opened, reopened, synchronize, labeled, unlabeled] +permissions: + contents: read + pull-requests: read +jobs: + changelog: + name: ๐Ÿ“ News fragment + runs-on: ubuntu-26.04 + steps: + - name: ๐Ÿ“ฅ Check out code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: ๐Ÿ Install uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: latest-known + python-version: "3.15" + enable-cache: false + - name: ๐Ÿ“ฆ Install locked release tools + uses: tox-dev/github-workflows/install-locked-tools@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + with: + groups: release-news + venv: ${{ runner.temp }}/release-tools + - name: ๐Ÿ” Validate news fragments + uses: tox-dev/github-workflows/validate-news@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 diff --git a/.github/workflows/prepare-release.yaml b/.github/workflows/prepare-release.yaml new file mode 100644 index 0000000..3daaefb --- /dev/null +++ b/.github/workflows/prepare-release.yaml @@ -0,0 +1,33 @@ +name: ๐Ÿท๏ธ Prepare release +on: + workflow_dispatch: + inputs: + bump: + description: Version bump strategy + required: true + type: choice + options: + - auto + - major + - minor + - patch + default: auto + workflow_call: + inputs: + dry-run: + description: Assemble the release without pushing, for the pull request gate + type: boolean + required: true +permissions: {} +jobs: + prepare: + name: ๐Ÿท๏ธ Prepare Towncrier release + uses: tox-dev/github-workflows/.github/workflows/prepare-towncrier-release.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + permissions: + contents: read + pull-requests: read + secrets: + RELEASE_APP_PRIVATE_KEY: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + with: + bump: ${{ inputs.bump || 'auto' }} + dry-run: ${{ inputs.dry-run == true }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..df5858b --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,56 @@ +name: ๐Ÿš€ Release to PyPI +on: + push: + tags: ["*"] +permissions: + contents: read +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false +jobs: + build: + name: ๐Ÿ—๏ธ Build + uses: tox-dev/github-workflows/.github/workflows/validate-python-release.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + permissions: + contents: read + notes: + name: ๐Ÿ“ Render GitHub Release notes + uses: tox-dev/github-workflows/.github/workflows/release-notes.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + permissions: + contents: read + with: + tag: ${{ github.ref_name }} + changelog: CHANGELOG.md + artifact-url: https://pypi.org/project/hatch-pep770/${{ github.ref_name }}/ + publish-pypi: + name: ๐Ÿš€ Publish to PyPI + needs: [build, notes] + runs-on: ubuntu-26.04 + environment: + name: release + url: https://pypi.org/project/hatch-pep770/${{ github.ref_name }} + permissions: + id-token: write + steps: + - name: ๐Ÿ“ฉ Download distributions + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ needs.build.outputs.distributions-artifact-id }} + path: dist + - name: ๐Ÿš€ Publish to PyPI + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + attestations: true + packages-dir: dist + github-release: + name: ๐Ÿš€ Publish GitHub Release + needs: [build, notes, publish-pypi] + uses: tox-dev/github-workflows/.github/workflows/github-release.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + permissions: + contents: write + with: + project: hatch-pep770 + version: ${{ github.ref_name }} + tag: ${{ github.ref_name }} + artifact-pattern: python-package-distributions + notes-artifact-id: ${{ needs.notes.outputs.artifact-id }} diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml new file mode 100644 index 0000000..a219ba2 --- /dev/null +++ b/.github/workflows/scorecard.yaml @@ -0,0 +1,26 @@ +name: ๐Ÿ›ก๏ธ scorecard +on: + branch_protection_rule: + schedule: + - cron: "43 8 * * 2" + push: + branches: ["main"] + workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +permissions: {} +jobs: + analysis: + name: ๐Ÿ›ก๏ธ Scorecard analysis + if: github.repository == 'tox-dev/hatch-pep770' + permissions: + actions: read # read workflow runs for the Packaging check + checks: read # read check runs for the CI-Tests and SAST checks + contents: read # check out and scan the repository + id-token: write # authenticate the results published to the scorecard API + issues: read # read issue activity for the Maintained check + pull-requests: read # read reviews for the Code-Review check + security-events: write # upload the SARIF results to code scanning + statuses: read # read commit statuses for the CI-Tests check + uses: tox-dev/github-workflows/.github/workflows/scorecard.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 From a6922926b9dca96fc2cc6c392d76d56601791081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 18:37:56 -0800 Subject: [PATCH 004/115] =?UTF-8?q?=F0=9F=93=9D=20docs:=20pin=20the=20firs?= =?UTF-8?q?t=20release=20in=20the=20example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 13319d8..a8171e4 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Add the hook to the build requirements and enable it: ```toml [build-system] build-backend = "hatchling.build" -requires = ["hatch-pep770>=1", "hatchling>=1.28"] +requires = ["hatch-pep770>=0.1", "hatchling>=1.28"] [tool.hatch.build.hooks.pep770] ``` From b57cafb7d817391ca5eae0d67e91ad122bd1add8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 19:54:34 -0800 Subject: [PATCH 005/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20split?= =?UTF-8?q?=20into=20private=20modules,=20add=20embed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generate the SBOM from the wheel's METADATA so the hatchling hook and the new embed command share one code path, and expose the hook, the Project settings, the building blocks and the SPDX and validation helpers from the package root. Raise the floor to Python 3.11. --- src/hatch_pep770/__init__.py | 49 +- src/hatch_pep770/__main__.py | 58 +- src/hatch_pep770/_cli.py | 86 ++ .../{cyclonedx.py => _cyclonedx.py} | 399 ++++----- src/hatch_pep770/_embed.py | 74 ++ src/hatch_pep770/_extra.py | 37 + src/hatch_pep770/_hook.py | 107 +++ src/hatch_pep770/_project.py | 484 ++++++++++ src/hatch_pep770/_run.py | 47 + src/hatch_pep770/{spdx.py => _spdx.py} | 52 +- .../{validate.py => _validate.py} | 102 +-- src/hatch_pep770/hooks.py | 11 +- src/hatch_pep770/plugin.py | 453 ---------- src/hatch_pep770/py.typed | 0 tests/conftest.py | 163 ++-- tests/seeder/LICENSE | 3 + tests/seeder/hatch_build.py | 29 + tests/seeder/pyproject.toml | 26 + tests/seeder/src/seeder/__init__.py | 0 tests/seeder/src/seeder/embed/__init__.py | 0 tests/seeder/src/seeder/embed/bundle.py | 9 + .../src/seeder/embed/pip-1.0-py3-none-any.whl | Bin 0 -> 1080 bytes tests/test___main__.py | 62 -- tests/test_cli.py | 71 ++ tests/test_embed.py | 126 +++ tests/test_hook.py | 144 +++ tests/test_main.py | 21 + tests/test_plugin.py | 829 ------------------ tests/test_project.py | 546 ++++++++++++ tests/test_spdx.py | 101 +-- tests/test_validate.py | 91 +- tests/test_version.py | 9 + 32 files changed, 2311 insertions(+), 1878 deletions(-) create mode 100644 src/hatch_pep770/_cli.py rename src/hatch_pep770/{cyclonedx.py => _cyclonedx.py} (62%) create mode 100644 src/hatch_pep770/_embed.py create mode 100644 src/hatch_pep770/_extra.py create mode 100644 src/hatch_pep770/_hook.py create mode 100644 src/hatch_pep770/_project.py create mode 100644 src/hatch_pep770/_run.py rename src/hatch_pep770/{spdx.py => _spdx.py} (88%) rename src/hatch_pep770/{validate.py => _validate.py} (69%) delete mode 100644 src/hatch_pep770/plugin.py create mode 100644 src/hatch_pep770/py.typed create mode 100644 tests/seeder/LICENSE create mode 100644 tests/seeder/hatch_build.py create mode 100644 tests/seeder/pyproject.toml create mode 100644 tests/seeder/src/seeder/__init__.py create mode 100644 tests/seeder/src/seeder/embed/__init__.py create mode 100644 tests/seeder/src/seeder/embed/bundle.py create mode 100644 tests/seeder/src/seeder/embed/pip-1.0-py3-none-any.whl delete mode 100644 tests/test___main__.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_embed.py create mode 100644 tests/test_hook.py create mode 100644 tests/test_main.py delete mode 100644 tests/test_plugin.py create mode 100644 tests/test_project.py create mode 100644 tests/test_version.py diff --git a/src/hatch_pep770/__init__.py b/src/hatch_pep770/__init__.py index 1aa6c39..b8d369b 100644 --- a/src/hatch_pep770/__init__.py +++ b/src/hatch_pep770/__init__.py @@ -1 +1,48 @@ -"""Embed a PEP 770 CycloneDX SBOM in wheels built with hatchling.""" +"""Write PEP 770 CycloneDX SBOMs into wheels, from a hatchling build hook or the command line.""" + +from __future__ import annotations + +from ._cli import Options, main +from ._cyclonedx import ( + Body, + Component, + Dependency, + Document, + OrganizationalEntity, + Property, + build_tools, + component_from_metadata, + document, + timestamp, +) +from ._embed import embed +from ._hook import Pep770BuildHook +from ._project import BundledWheelProperties, Project, Toml +from ._spdx import Spdx, to_spdx +from ._validate import validate, validate_wheel +from .version import version as __version__ + +__all__ = [ + "Body", + "BundledWheelProperties", + "Component", + "Dependency", + "Document", + "Options", + "OrganizationalEntity", + "Pep770BuildHook", + "Project", + "Property", + "Spdx", + "Toml", + "__version__", + "build_tools", + "component_from_metadata", + "document", + "embed", + "main", + "timestamp", + "to_spdx", + "validate", + "validate_wheel", +] diff --git a/src/hatch_pep770/__main__.py b/src/hatch_pep770/__main__.py index e57defc..fd5e81a 100644 --- a/src/hatch_pep770/__main__.py +++ b/src/hatch_pep770/__main__.py @@ -1,62 +1,10 @@ -"""Command line: render a CycloneDX SBOM as SPDX, or validate the SBOMs in built wheels.""" +"""Main entry point.""" from __future__ import annotations -import json import sys -from argparse import ArgumentParser, Namespace -from pathlib import Path - -from .spdx import to_spdx - - -class _Options(Namespace): - command: str - source: Path - target: Path - paths: list[Path] - allow_license: list[str] | None - - -def main(argv: list[str] | None = None) -> int: - parser = ArgumentParser(prog="hatch-pep770", description="Work with the PEP 770 SBOMs hatch-pep770 writes.") - commands = parser.add_subparsers(dest="command", required=True) - spdx = commands.add_parser("spdx", help="render a CycloneDX SBOM as SPDX 2.3 JSON") - spdx.add_argument("source", type=Path, help="the CycloneDX JSON file to read") - spdx.add_argument("target", type=Path, help="the SPDX JSON file to write") - validate = commands.add_parser( - "validate", help="validate the SBOMs in wheels, needs the validate extra: hatch-pep770[validate]" - ) - validate.add_argument("paths", type=Path, nargs="+", help="wheel files, or directories holding them") - validate.add_argument( - "--allow-license", - action="append", - metavar="SPDX-ID", - help="SPDX license id bundled components may use; repeat it per id, leave it out to skip the license policy", - ) - options = parser.parse_args(argv, namespace=_Options()) - if options.command == "spdx": - bom = json.loads(options.source.read_text(encoding="utf-8")) - options.target.write_bytes(f"{json.dumps(to_spdx(bom), indent=2)}\n".encode()) - return 0 - # validate pulls in the optional extra, so the spdx command works without it - from .validate import wheel_problems # ruff: ignore[import-outside-top-level] - - allowed = None if options.allow_license is None else frozenset(options.allow_license) - wheels = sorted(wheel for path in options.paths for wheel in (path.glob("*.whl") if path.is_dir() else [path])) - if not wheels: - sys.stderr.write(f"no wheel found in {', '.join(map(str, options.paths))}\n") - return 1 - failed = False - for wheel in wheels: - for problem in (problems := wheel_problems(wheel, allowed)): - sys.stderr.write(f"SBOM invalid: {wheel.name}: {problem}\n") - if problems: - failed = True - else: - sys.stdout.write(f"SBOM in {wheel.name} is valid\n") - return int(failed) +from ._cli import main if __name__ == "__main__": - sys.exit(main()) + main(sys.argv[1:]) diff --git a/src/hatch_pep770/_cli.py b/src/hatch_pep770/_cli.py new file mode 100644 index 0000000..006d087 --- /dev/null +++ b/src/hatch_pep770/_cli.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter +from pathlib import Path +from typing import TYPE_CHECKING + +from ._run import run +from .version import version + +if TYPE_CHECKING: + from collections.abc import Sequence + + +def main(args: Sequence[str] | None = None) -> None: + """ + Run the command-line interface. + + :param args: command-line arguments, or ``sys.argv[1:]`` when None + :raises SystemExit: with status 1 when a command fails + + """ + if not run(_build_parser().parse_args(args, namespace=Options())): + raise SystemExit(1) + + +def _build_parser() -> ArgumentParser: + parser = ArgumentParser( + prog="hatch-pep770", + description="Write, convert and check PEP 770 CycloneDX SBOMs of wheels.", + formatter_class=RawDescriptionHelpFormatter, + epilog=f"running {version} at {Path(__file__).parent}", + ) + commands = parser.add_subparsers(dest="command", required=True) + embed = commands.add_parser( + "embed", + help="write the SBOM into wheels another build backend built", + description="Write the SBOM the hatchling hook would write into wheels another build backend built, such as" + " meson-python. Settings come from the [tool.hatch-pep770] table of the project's pyproject.toml.", + ) + embed.add_argument("wheels", type=Path, nargs="+", metavar="wheel", help="wheel to rewrite in place") + embed.add_argument( + "--root", + type=Path, + default=Path.cwd(), + help="project directory the wheels were built from (default: %(default)s)", + ) + spdx = commands.add_parser("spdx", help="render a CycloneDX SBOM as SPDX 2.3 JSON") + spdx.add_argument("source", type=Path, help="CycloneDX JSON file to read") + spdx.add_argument("target", type=Path, help="SPDX JSON file to write") + validate = commands.add_parser( + "validate", help="validate the SBOMs in wheels", description="Needs the validate extra: hatch-pep770[validate]." + ) + validate.add_argument("paths", type=Path, nargs="+", metavar="path", help="wheel, or directory holding wheels") + validate.add_argument( + "--allow-license", + dest="allowed_licenses", + action="append", + metavar="spdx-id", + help="SPDX license id bundled components may use; repeat it per id, leave it out to skip the license policy", + ) + return parser + + +class Options(Namespace): + """The parsed command line.""" + + command: str + """The subcommand: ``embed``, ``spdx`` or ``validate``.""" + wheels: list[Path] + """Wheels to write an SBOM into, for ``embed``.""" + root: Path + """Project directory the wheels were built from, for ``embed``.""" + source: Path + """CycloneDX document to read, for ``spdx``.""" + target: Path + """SPDX document to write, for ``spdx``.""" + paths: list[Path] + """Wheels, or directories holding them, for ``validate``.""" + allowed_licenses: list[str] | None + """SPDX license ids bundled components may use, for ``validate``; None skips the license policy.""" + + +__all__ = [ + "Options", + "main", +] diff --git a/src/hatch_pep770/cyclonedx.py b/src/hatch_pep770/_cyclonedx.py similarity index 62% rename from src/hatch_pep770/cyclonedx.py rename to src/hatch_pep770/_cyclonedx.py index 171b825..1888094 100644 --- a/src/hatch_pep770/cyclonedx.py +++ b/src/hatch_pep770/_cyclonedx.py @@ -1,22 +1,22 @@ -"""CycloneDX 1.6 building blocks shared by the build hook and by projects that describe other artifacts.""" - from __future__ import annotations import base64 +import hashlib import json import platform import re import sys import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from email.parser import Parser from email.utils import getaddresses from importlib.metadata import distributions -from typing import TYPE_CHECKING, Final, TypedDict +from typing import TYPE_CHECKING, Final, NotRequired, TypedDict from hatchling.builders.utils import get_reproducible_timestamp from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression from packaging.requirements import Requirement +from packaging.utils import canonicalize_name if TYPE_CHECKING: from email.message import Message @@ -24,48 +24,45 @@ class OrganizationalEntity(TypedDict): + """An organization that supplies or makes a component.""" + name: str url: list[str] -class Contact(TypedDict, total=False): - name: str - email: str +class Contact(TypedDict): + name: NotRequired[str] + email: NotRequired[str] -class Hash(TypedDict): +class _Hash(TypedDict): alg: str content: str class Property(TypedDict): + """A name and value pair; names this package writes start with the project name or ``python:``.""" + name: str value: str -class _ExternalReferenceKeys(TypedDict): +class ExternalReference(TypedDict): type: str url: str + comment: NotRequired[str] -class ExternalReference(_ExternalReferenceKeys, total=False): - comment: str - - -class _LicenseIdentity(TypedDict, total=False): - id: str - name: str - - -class LicenseInfo(_LicenseIdentity): +class LicenseInfo(TypedDict): + id: NotRequired[str] + name: NotRequired[str] acknowledgement: str -class License(TypedDict, total=False): - # a CycloneDX license choice holds either a `license` or an `expression` with its `acknowledgement` - license: LicenseInfo - expression: str - acknowledgement: str +class License(TypedDict): + license: NotRequired[LicenseInfo] + expression: NotRequired[str] + acknowledgement: NotRequired[str] class _Method(TypedDict): @@ -80,86 +77,88 @@ class _Identity(TypedDict): methods: list[_Method] -class Evidence(TypedDict): +class _Evidence(TypedDict): identity: list[_Identity] -_ComponentKeys = TypedDict("_ComponentKeys", {"type": str, "bom-ref": str, "name": str}) - - -class _ComponentFields(TypedDict, total=False): - version: str - purl: str - supplier: OrganizationalEntity - authors: list[Contact] - description: str - licenses: list[License] - copyright: str - hashes: list[Hash] - externalReferences: list[ExternalReference] - properties: list[Property] - evidence: Evidence - components: list[Component] - - -class Component(_ComponentKeys, _ComponentFields): - pass +#: A CycloneDX component: a distribution, an application, the interpreter or a file. +Component = TypedDict( + "Component", + { + "type": str, + "bom-ref": str, + "name": str, + "version": NotRequired[str], + "purl": NotRequired[str], + "supplier": NotRequired[OrganizationalEntity], + "authors": NotRequired[list[Contact]], + "description": NotRequired[str], + "licenses": NotRequired[list[License]], + "copyright": NotRequired[str], + "hashes": NotRequired[list[_Hash]], + "externalReferences": NotRequired[list[ExternalReference]], + "properties": NotRequired[list[Property]], + "evidence": NotRequired[_Evidence], + "components": NotRequired[list["Component"]], + }, +) class Dependency(TypedDict): + """The components one component depends on, by ``bom-ref``.""" + ref: str dependsOn: list[str] -class _Ref(TypedDict): +class _Reference(TypedDict): ref: str class _Output(TypedDict): type: str - resource: _Ref + resource: _Reference class _Input(TypedDict): environmentVars: list[Property] -Workflow = TypedDict( - "Workflow", +_Workflow = TypedDict( + "_Workflow", { "bom-ref": str, "uid": str, "name": str, "taskTypes": list[str], - "resourceReferences": list[_Ref], - "inputs": list[_Input], + "resourceReferences": list[_Reference], "outputs": list[_Output], + "inputs": list[_Input], }, ) -Formula = TypedDict("Formula", {"bom-ref": str, "workflows": list[Workflow]}) -class _Lifecycle(TypedDict): +_Formula = TypedDict("_Formula", {"bom-ref": str, "workflows": list[_Workflow]}) + + +class Lifecycle(TypedDict): phase: str -class _Tools(TypedDict): +class Tools(TypedDict): components: list[Component] -class _MetadataKeys(TypedDict): +class Metadata(TypedDict): timestamp: str - lifecycles: list[_Lifecycle] - tools: _Tools + lifecycles: list[Lifecycle] + tools: Tools + manufacturer: NotRequired[OrganizationalEntity] + authors: NotRequired[list[Contact]] + supplier: NotRequired[OrganizationalEntity] component: Component - - -class Metadata(_MetadataKeys, total=False): - manufacturer: OrganizationalEntity - supplier: OrganizationalEntity - authors: list[Contact] - licenses: list[License] - properties: list[Property] + licenses: NotRequired[list[License]] + properties: NotRequired[list[Property]] class Composition(TypedDict): @@ -167,26 +166,33 @@ class Composition(TypedDict): dependencies: list[str] -class _BodyKeys(TypedDict): +class Body(TypedDict): + """The content of a document, everything but the header :func:`document` adds.""" + metadata: Metadata components: list[Component] dependencies: list[Dependency] - formulation: list[Formula] - + compositions: NotRequired[list[Composition]] + formulation: list[_Formula] -class Body(_BodyKeys, total=False): - compositions: list[Composition] - -_Header = TypedDict( - "_Header", {"$schema": str, "bomFormat": str, "specVersion": str, "serialNumber": str, "version": int} +#: A complete CycloneDX 1.6 JSON document. +Document = TypedDict( + "Document", + { + "$schema": str, + "bomFormat": str, + "specVersion": str, + "serialNumber": str, + "version": int, + "metadata": Metadata, + "components": list[Component], + "dependencies": list[Dependency], + "compositions": NotRequired[list[Composition]], + "formulation": list[_Formula], + }, ) - -class Document(_Header, Body): - pass - - # the label normalization and mapping cyclonedx-py applies to Project-URL entries, plus "source", so the same metadata # yields the same reference types whichever generator produced the document _URL_LABEL_TO_REFERENCE_TYPE: Final[dict[str, str]] = { @@ -207,40 +213,15 @@ class Document(_Header, Body): "github": "vcs", "chat": "chat", } -# core metadata headers copied verbatim onto a component as properties -_METADATA_PROPERTIES: Final[tuple[str, ...]] = ( - "Requires-Python", - "Requires-Dist", - "Provides-Extra", - "Classifier", - "Keywords", -) -# only the license classifiers that name exactly one SPDX license; "BSD License" or "Apache Software License" leave the -# version or variant open -_CLASSIFIER_LICENSES: Final[dict[str, str]] = { - "ISC License (ISCL)": "ISC", - "MIT License": "MIT", - "MIT No Attribution License (MIT-0)": "MIT-0", - "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", - "Python Software Foundation License": "PSF-2.0", - "The Unlicense (Unlicense)": "Unlicense", -} -_RECORD_HASH_ALGORITHMS: Final[dict[str, str]] = { - "md5": "MD5", - "sha1": "SHA-1", - "sha256": "SHA-256", - "sha384": "SHA-384", - "sha512": "SHA-512", -} def document(body: Body, namespace: uuid.UUID) -> Document: """ - Wrap a document body in the CycloneDX 1.6 header. + Add the CycloneDX 1.6 header to a document body. - :param body: the metadata, components, dependencies and formulation of the document - :param namespace: UUID namespace for the serial number, one per project - :returns: the complete document, whose serial number derives from the body so identical bodies share it + :param body: the metadata, components, dependencies and formulation + :param namespace: UUID namespace of the serial number, one per project + :returns: the document; its serial number derives from the body, so identical bodies share one """ return { @@ -255,58 +236,64 @@ def document(body: Body, namespace: uuid.UUID) -> Document: def timestamp() -> str: """ - Format the build time for the document. + Format the build time. - :returns: the ISO 8601 time from ``SOURCE_DATE_EPOCH``, or hatchling's fixed default when unset + :returns: ISO 8601 time from ``SOURCE_DATE_EPOCH``, or hatchling's fixed default when it is unset """ - return datetime.fromtimestamp(get_reproducible_timestamp(), tz=timezone.utc).isoformat() + return datetime.fromtimestamp(get_reproducible_timestamp(), tz=UTC).isoformat() -def purl(name: str, version: str | None = None) -> str: - """ - Build the package URL of a Python distribution. - - :param name: distribution name, normalized here - :param version: version, or ``None`` for an unversioned purl - :returns: the ``pkg:pypi`` package URL - - """ - normalized = re.sub(r"[-_.]+", "-", name).lower() - return f"pkg:pypi/{normalized}@{version}" if version else f"pkg:pypi/{normalized}" - - -def external_reference(label: str, url: str) -> ExternalReference: +def build_tools() -> tuple[list[Component], list[Dependency]]: """ - Turn a ``Project-URL`` entry into an external reference. - - :param label: the entry's label, which picks the reference type - :param url: the entry's URL - :returns: the reference, typed ``other`` when the label is not a known one + Describe the running environment: the interpreter and every installed distribution. - """ - reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") - return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} + Nothing about the machine goes in, so the same toolchain yields the same record on any operating system. A + pure-Python distribution lists its files with their hashes; a platform one does not, because its files differ per + machine. + :returns: the tool components, interpreter first, and the dependency graph between the distributions -def contacts(names: list[str], addresses: list[str]) -> list[Contact]: """ - Turn core metadata author or maintainer fields into CycloneDX contacts. + interpreter = f"pkg:generic/{sys.implementation.name}@{platform.python_version()}" + tools: Final[list[Component]] = [ + { + "type": "platform", + "bom-ref": f"tool:{interpreter}", + "name": sys.implementation.name, + "version": platform.python_version(), + "purl": interpreter, + "properties": [{"name": "python:implementation", "value": platform.python_implementation()}], + }, + ] + # bom-refs are prefixed because the same distribution can be both a build tool and a component of the artifact + installed = {purl(distribution.metadata["Name"]): distribution for distribution in distributions()} + tool_dependencies: Final[list[Dependency]] = [] + for distribution in (installed[key] for key in sorted(installed)): + component = component_from_metadata(distribution.metadata, "library") + component["bom-ref"] = f"tool:{component['bom-ref']}" + if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] == "true": + component["components"] = [ + file_component( + component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) + ) + for file in distribution.files or [] + # console-script launchers live outside site-packages and embed the environment's interpreter path in + # their shebang, so their hash differs on every build; the installer metadata names the frontend that + # set up the environment rather than the distribution + if file.hash is not None + and not file.as_posix().startswith("../") + and not ( + file.parent.suffix == ".dist-info" and file.name in {"INSTALLER", "REQUESTED", "direct_url.json"} + ) + ] + tools.append(component) + tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) + return tools, tool_dependencies - :param names: bare names, from ``Author`` or ``Maintainer`` - :param addresses: RFC 5322 addresses, from ``Author-email`` or ``Maintainer-email`` - :returns: one contact per name, then one per address - """ - found: list[Contact] = [{"name": name} for name in names] - for name, email in getaddresses(addresses): - contact: Contact = {} - if name: - contact["name"] = name - if email: - contact["email"] = email - found.append(contact) - return found +def purl(name: str, version: str | None = None) -> str: + return f"pkg:pypi/{canonicalize_name(name)}@{version}" if version else f"pkg:pypi/{canonicalize_name(name)}" def component_from_metadata(metadata: PackageMetadata | Message, component_type: str) -> Component: @@ -319,13 +306,9 @@ def component_from_metadata(metadata: PackageMetadata | Message, component_type: """ name, version = metadata["Name"], metadata["Version"] - component: Final[Component] = { - "type": component_type, - "bom-ref": (package := purl(name, version)), - "name": name, - "version": version, - "purl": package, - } + package = purl(name, version) + component: Final[Component] = {"type": component_type, "bom-ref": package, "name": name, "version": version} + component["purl"] = package if summary := metadata.get("Summary"): component["description"] = summary if authors := contacts( @@ -341,14 +324,27 @@ def component_from_metadata(metadata: PackageMetadata | Message, component_type: ] if home_page := metadata.get("Home-page"): component["externalReferences"].insert(0, {"type": "website", "url": home_page}) + # core metadata headers copied verbatim onto the component component["properties"] = [ {"name": f"python:{header.lower()}", "value": value} - for header in _METADATA_PROPERTIES + for header in ("Requires-Python", "Requires-Dist", "Provides-Extra", "Classifier", "Keywords") for value in metadata.get_all(header, []) ] return component +def contacts(names: list[str], addresses: list[str]) -> list[Contact]: + found: Final[list[Contact]] = [{"name": name} for name in names] + for name, email in getaddresses(addresses): + contact: Contact = {} + if name: + contact["name"] = name + if email: + contact["email"] = email + found.append(contact) + return found + + def _licenses(metadata: PackageMetadata | Message) -> list[License]: if expression := metadata.get("License-Expression"): return [{"expression": expression, "acknowledgement": "declared"}] @@ -369,7 +365,16 @@ def _licenses(metadata: PackageMetadata | Message) -> list[License]: def _spdx_license(name: str) -> str | None: - if identifier := _CLASSIFIER_LICENSES.get(name): + # only the license classifiers that name exactly one SPDX license; "BSD License" or "Apache Software License" leave + # the version or variant open + if identifier := { + "ISC License (ISCL)": "ISC", + "MIT License": "MIT", + "MIT No Attribution License (MIT-0)": "MIT-0", + "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", + "Python Software Foundation License": "PSF-2.0", + "The Unlicense (Unlicense)": "Unlicense", + }.get(name): return identifier try: canonical = canonicalize_license_expression(name) @@ -379,76 +384,31 @@ def _spdx_license(name: str) -> str | None: return canonical if re.fullmatch(r"[A-Za-z0-9.-]+", canonical) and not canonical.startswith("LicenseRef-") else None -def file_component(parent_ref: str, path: str, digest: str, size: str) -> Component: - """ - Describe one file from a wheel ``RECORD`` row. +def external_reference(label: str, url: str) -> ExternalReference: + reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") + return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} - :param parent_ref: ``bom-ref`` of the component that holds the file - :param path: the file's path inside the wheel - :param digest: the ``RECORD`` digest, ``=`` - :param size: the file's size in bytes - :returns: a ``file`` component with the digest as a lowercase hex hash - """ +def file_component(parent_ref: str, path: str, digest: str, size: str) -> Component: + # RECORD stores "=", CycloneDX wants lowercase hex algorithm, _, encoded = digest.partition("=") + algorithms: Final[dict[str, str]] = { + "md5": "MD5", + "sha1": "SHA-1", + "sha256": "SHA-256", + "sha384": "SHA-384", + "sha512": "SHA-512", + } raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) return { "type": "file", "bom-ref": f"{parent_ref}#{path}", "name": path, - "hashes": [{"alg": _RECORD_HASH_ALGORITHMS[algorithm], "content": raw.hex()}], + "hashes": [{"alg": algorithms[algorithm], "content": raw.hex()}], "properties": [{"name": "size", "value": size}], } -def build_tools() -> tuple[list[Component], list[Dependency]]: - """ - Describe the running build environment: the interpreter and every installed distribution. - - Nothing about the build machine goes in, so the same toolchain yields the same record on any operating system or - architecture. A pure-Python distribution lists its files with their hashes; a platform one does not, because its - files differ per machine. - - :returns: the tool components, interpreter first, and the dependency graph between the distributions - - """ - interpreter = f"pkg:generic/{sys.implementation.name}@{platform.python_version()}" - tools: Final[list[Component]] = [ - { - "type": "platform", - "bom-ref": f"tool:{interpreter}", - "name": sys.implementation.name, - "version": platform.python_version(), - "purl": interpreter, - "properties": [{"name": "python:implementation", "value": platform.python_implementation()}], - }, - ] - # bom-refs are prefixed because the same distribution can be both a build tool and a component of the artifact - installed = {purl(distribution.metadata["Name"]): distribution for distribution in distributions()} - tool_dependencies: Final[list[Dependency]] = [] - for distribution in (installed[key] for key in sorted(installed)): - component = component_from_metadata(distribution.metadata, "library") - component["bom-ref"] = f"tool:{component['purl']}" - if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] == "true": - component["components"] = [ - file_component( - component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) - ) - for file in distribution.files or [] - # console-script launchers live outside site-packages and embed the build env's interpreter path in - # their shebang, so their hash differs on every build and says nothing about the distribution; the - # installer metadata names the build frontend that set up the env rather than the distribution - if file.hash is not None - and not file.as_posix().startswith("../") - and not ( - file.parent.suffix == ".dist-info" and file.name in {"INSTALLER", "REQUESTED", "direct_url.json"} - ) - ] - tools.append(component) - tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) - return tools, tool_dependencies - - def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) -> list[str]: refs = set() for requirement in map(Requirement, distribution.requires or []): @@ -459,6 +419,10 @@ def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) return sorted(refs) +def record_digest(content: bytes) -> str: + return f"sha256={base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b'=').decode()}" + + __all__ = [ "Body", "Component", @@ -466,16 +430,14 @@ def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) "Contact", "Dependency", "Document", - "Evidence", "ExternalReference", - "Formula", - "Hash", "License", "LicenseInfo", + "Lifecycle", "Metadata", "OrganizationalEntity", "Property", - "Workflow", + "Tools", "build_tools", "component_from_metadata", "contacts", @@ -483,5 +445,6 @@ def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) "external_reference", "file_component", "purl", + "record_digest", "timestamp", ] diff --git a/src/hatch_pep770/_embed.py b/src/hatch_pep770/_embed.py new file mode 100644 index 0000000..c9da7c3 --- /dev/null +++ b/src/hatch_pep770/_embed.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import json +import os +import tempfile +import time +import zipfile +from email.parser import Parser +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from hatchling.builders.utils import get_reproducible_timestamp + +from ._cyclonedx import record_digest +from ._project import sbom_file_name + +if TYPE_CHECKING: + from ._project import Project + + +def embed(wheel: Path, project: Project) -> str: + """ + Write the SBOM into a wheel some other build backend already built, as the build hook would have. + + The SBOM goes under ``.dist-info/sboms/`` with a ``RECORD`` row, replacing an SBOM of the same name. Every other + entry keeps its name, order, timestamp and permissions, and the new entry takes its timestamp from + ``SOURCE_DATE_EPOCH``, so embedding into the same wheel twice gives the same bytes. + + :param wheel: the wheel to rewrite in place + :param project: the source tree the wheel was built from + :returns: the SBOM's path inside the wheel + + """ + with zipfile.ZipFile(wheel) as archive: + entries: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [ + (info, archive.read(info)) for info in archive.infolist() + ] + dist_info: Final[str] = next( + info.filename.split("/")[0] + for info, _ in entries + if info.filename.count("/") == 1 and info.filename.endswith(".dist-info/METADATA") + ) + contents: Final[dict[str, bytes]] = {info.filename: content for info, content in entries} + metadata = Parser().parsestr(contents[f"{dist_info}/METADATA"].decode("utf-8")) + sbom: Final[str] = f"{dist_info}/sboms/{sbom_file_name(metadata)}" + content: Final[bytes] = f"{json.dumps(project.describe(metadata, wheel.name), indent=2)}\n".encode() + record: Final[str] = f"{dist_info}/RECORD" + lines: Final[list[str]] = [ + line for line in contents[record].decode("utf-8").splitlines(keepends=True) if not line.startswith(f"{sbom},") + ] + newline = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" + # the RECORD row of RECORD itself stays last + at = next((index for index, line in enumerate(lines) if line.startswith(f"{record},")), len(lines)) + lines.insert(at, f"{sbom},{record_digest(content)},{len(content)}{newline}") + entry = zipfile.ZipInfo(sbom, time.gmtime(max(get_reproducible_timestamp(), 315532800))[:6]) # zip starts at 1980 + entry.compress_type = zipfile.ZIP_DEFLATED + entry.external_attr = 0o644 << 16 + # written next to the wheel and moved over it, so a failure leaves the wheel as it was + descriptor, temporary = tempfile.mkstemp(dir=wheel.parent, suffix=".whl") + os.close(descriptor) + with zipfile.ZipFile(temporary, "w") as archive: + for info, data in entries: + if info.filename == record: + archive.writestr(entry, content) + archive.writestr(info, "".join(lines).encode()) + elif info.filename != sbom: + archive.writestr(info, data) + Path(temporary).replace(wheel) + return sbom + + +__all__ = [ + "embed", +] diff --git a/src/hatch_pep770/_extra.py b/src/hatch_pep770/_extra.py new file mode 100644 index 0000000..15536ad --- /dev/null +++ b/src/hatch_pep770/_extra.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from functools import cache +from typing import TYPE_CHECKING + +from cyclonedx.schema import SchemaVersion +from cyclonedx.validation.json import JsonStrictValidator +from spdx_tools.spdx.parser.error import SPDXParsingError +from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser +from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document + +if TYPE_CHECKING: + from ._spdx import Spdx + + +def schema_error(raw: str) -> str | None: + error = _validator().validate_str(raw, all_errors=False) + return None if error is None else str(error) + + +@cache # building the validator loads the whole schema, which takes a second +def _validator() -> JsonStrictValidator: + return JsonStrictValidator(SchemaVersion.V1_6) + + +def spdx_problems(spdx: Spdx) -> list[str]: + try: + parsed = JsonLikeDictParser().parse(dict(spdx)) + except SPDXParsingError as error: + return [f"SPDX rendering does not parse: {message}" for message in error.get_messages()] + return [f"SPDX rendering invalid: {message.validation_message}" for message in validate_full_spdx_document(parsed)] + + +__all__ = [ + "schema_error", + "spdx_problems", +] diff --git a/src/hatch_pep770/_hook.py b/src/hatch_pep770/_hook.py new file mode 100644 index 0000000..7ac1677 --- /dev/null +++ b/src/hatch_pep770/_hook.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import json +import shutil +import tempfile +from email.parser import Parser +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Any + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.builders.wheel import WheelBuilder +from hatchling.utils.constants import DEFAULT_BUILD_SCRIPT + +from ._project import Project, sbom_file_name + +if TYPE_CHECKING: + from ._cyclonedx import Property + + +class Pep770BuildHook(BuildHookInterface): + """ + Hatchling build hook that writes a PEP 770 CycloneDX SBOM into the wheel's ``.dist-info/sboms/``. + + Enable it with a ``[tool.hatch.build.hooks.pep770]`` table, or subclass it in a hatchling custom build hook to add + project-specific detail through :meth:`bundled_wheel_properties`. + + """ + + PLUGIN_NAME = "pep770" + _sbom_directory: Path | None = None + + def initialize( + self, + version: str, # ruff: ignore[unused-method-argument] # hatchling's signature + build_data: dict[str, Any], # simplint: ignore[SMP602] # hatchling API + ) -> None: + """ + Write the SBOM and register it with the wheel build. + + :param version: hatchling's build variant name, such as ``standard`` + :param build_data: the build data, whose ``sbom_files`` receives the SBOM + + """ + if not isinstance(builder := self.build_config.builder, WheelBuilder): + return + metadata = Parser().parsestr( + builder.config.core_metadata_constructor(self.metadata, extra_dependencies=build_data["dependencies"]) + ) + tag = build_data.get("tag") or ( + builder.get_best_matching_tag() if build_data["infer_tag"] else builder.get_default_tag() + ) + project = Project.from_table( + self.config, + root=Path(self.root), + where=f"build hook `{self.PLUGIN_NAME}`", + # a custom build hook script shapes the document, so the SBOM lists it among the build tools + build_script=PurePosixPath(self.config.get("path", DEFAULT_BUILD_SCRIPT)) + if self.PLUGIN_NAME == "custom" + else None, + bundled_wheel_properties=self.bundled_wheel_properties, + ) + content = json.dumps(project.describe(metadata, f"{builder.artifact_project_id}-{tag}.whl"), indent=2) + # written outside self.directory, which is also the wheel's output directory, so no stray copy lands next to + # the built artifacts + self._sbom_directory = Path(tempfile.mkdtemp(prefix="hatch-pep770-")) + # bytes, so the line endings do not depend on the build machine's OS + (sbom := self._sbom_directory / sbom_file_name(metadata)).write_bytes(f"{content}\n".encode()) + build_data["sbom_files"].append(str(sbom)) + + def bundled_wheel_properties( # ruff: ignore[no-self-use] # subclasses override it + self, + wheel: Path, # ruff: ignore[unused-method-argument] # subclasses use it + ) -> list[Property]: + """ + Return extra properties for a wheel the project bundles as a data file. + + Override it to say what the project does with the wheel, for example which Python versions receive it. The + properties follow ``:bundled-wheel`` and precede the ones taken from the wheel's metadata. + + :param wheel: the bundled wheel, an absolute path under the project root + :returns: the properties to add; the default adds none + + """ + return [] + + def finalize( + self, + version: str, # ruff: ignore[unused-method-argument] # hatchling's signature + build_data: dict[str, Any], # ruff: ignore[unused-method-argument] # simplint: ignore[SMP602] # hatchling API + artifact_path: str, # ruff: ignore[unused-method-argument] # hatchling's signature + ) -> None: + """ + Remove the SBOM's temporary directory once hatchling copied the file into the wheel. + + :param version: hatchling's build variant name + :param build_data: the build data + :param artifact_path: the built wheel + + """ + if self._sbom_directory is not None: + shutil.rmtree(self._sbom_directory) + self._sbom_directory = None + + +__all__ = [ + "Pep770BuildHook", +] diff --git a/src/hatch_pep770/_project.py b/src/hatch_pep770/_project.py new file mode 100644 index 0000000..1d1c865 --- /dev/null +++ b/src/hatch_pep770/_project.py @@ -0,0 +1,484 @@ +from __future__ import annotations + +import csv +import hashlib +import shutil +import subprocess # ruff: ignore[suspicious-subprocess-import] # runs git to read the source revision +import uuid +import zipfile +from dataclasses import dataclass +from email.parser import Parser +from io import StringIO +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias +from urllib.parse import urlsplit + +from hatchling.builders.utils import get_reproducible_timestamp +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +from ._cyclonedx import ( + build_tools, + component_from_metadata, + contacts, + document, + external_reference, + file_component, + purl, + timestamp, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + from datetime import date, datetime, time + from email.message import Message + + from ._cyclonedx import ( + Component, + Contact, + Dependency, + Document, + ExternalReference, + License, + OrganizationalEntity, + Property, + ) + +#: a value of a TOML table, as :mod:`tomllib` or hatchling hand it over +Toml: TypeAlias = "str | int | float | bool | datetime | date | time | list[Toml] | dict[str, Toml]" + + +class BundledWheelProperties(Protocol): + """Extra properties for a wheel the project bundles as a data file.""" + + def __call__(self, wheel: Path, /) -> list[Property]: + """ + Describe one bundled wheel. + + :param wheel: the bundled wheel, an absolute path under the project root + :returns: the properties to add after ``:bundled-wheel`` + + """ + + +def _no_bundled_wheel_properties(_wheel: Path, /) -> list[Property]: + return [] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Project: + """ + The source tree a wheel comes from, and what its SBOM should say about it. + + :param root: the project directory, where ``.git`` and the license files are + :param supplier: the organization that supplies the project, or ``None`` to leave the supplier out + :param repository: the repository URL, or ``None`` to take the first ``Source``, ``Repository`` or ``GitHub`` + project URL of the metadata + :param default_branch: branch used in links to files in the repository + :param bundled_wheels: glob patterns, relative to ``root``, of wheels the project ships as data files + :param build_script: path, relative to ``root``, of a build script to list among the build tools, or ``None`` + :param bundled_wheel_properties: extra properties for each bundled wheel + + """ + + root: Path + supplier: OrganizationalEntity | None + repository: str | None + default_branch: str + bundled_wheels: tuple[str, ...] + build_script: PurePosixPath | None + bundled_wheel_properties: BundledWheelProperties + + @classmethod + def from_table( + cls, + table: Mapping[str, Toml], + *, + root: Path, + where: str, + build_script: PurePosixPath | None = None, + bundled_wheel_properties: BundledWheelProperties = _no_bundled_wheel_properties, + ) -> Project: + """ + Read the settings from a configuration table, ignoring keys it does not know. + + :param table: the ``supplier``, ``repository``, ``default-branch`` and ``bundled-wheels`` settings + :param root: the project directory + :param where: where the table lives, for error messages, such as ``[tool.hatch-pep770]`` + :param build_script: path, relative to ``root``, of a build script to list among the build tools + :param bundled_wheel_properties: extra properties for each bundled wheel + :returns: the project + :raises TypeError: when a setting has the wrong type + + """ + supplier = table.get("supplier") + if supplier is not None and not ( + isinstance(supplier, dict) + and supplier.keys() <= {"name", "url"} + and isinstance(name := supplier.get("name"), str) + and isinstance(urls := supplier.get("url", []), list) + and all(isinstance(url, str) for url in urls) + ): + msg = f"`supplier` in {where} must be a table with a string `name` and an optional `url` string array" + raise TypeError(msg) + if not isinstance(repository := table.get("repository"), str | None): + msg = f"`repository` in {where} must be a string" + raise TypeError(msg) + if not isinstance(default_branch := table.get("default-branch", "main"), str): + msg = f"`default-branch` in {where} must be a string" + raise TypeError(msg) + if not ( + isinstance(patterns := table.get("bundled-wheels", []), list) + and all(isinstance(pattern, str) for pattern in patterns) + ): + msg = f"`bundled-wheels` in {where} must be an array of strings" + raise TypeError(msg) + return cls( + root=root, + supplier=None if supplier is None else {"name": name, "url": [str(url) for url in urls]}, + repository=None if repository is None else repository.rstrip("/"), + default_branch=default_branch, + bundled_wheels=tuple(str(pattern) for pattern in patterns), + build_script=build_script, + bundled_wheel_properties=bundled_wheel_properties, + ) + + def describe(self, metadata: Message, wheel_name: str) -> Document: + """ + Write the SBOM of a wheel. + + :param metadata: the wheel's ``METADATA`` + :param wheel_name: the wheel's file name, for its attestation link + :returns: the CycloneDX 1.6 document + + """ + namespace = canonicalize_name(metadata["Name"]) + repository = self.repository or self._project_url_repository(metadata) + root = self._root_component(metadata, wheel_name, repository) + bundled = [self._bundled_component(wheel, namespace, repository) for wheel in self._bundled_wheel_paths()] + declared = [_declared_dependency(requirement, namespace) for requirement in _runtime_requirements(metadata)] + bundled_dependencies: Final[list[Dependency]] = [ + {"ref": component["bom-ref"], "dependsOn": references} + for component in bundled + if (references := [child["bom-ref"] for child in component["components"] if child["type"] == "library"]) + ] + tools, tool_dependencies = build_tools() + if self.build_script is not None: + tools.insert(0, self._build_script_tool(self.build_script, metadata["Version"], repository)) + authors = _authors(metadata) + licenses = _licenses(metadata) + base = repository or f"https://pypi.org/project/{namespace}" + return document( + { + "metadata": { + "timestamp": timestamp(), + "lifecycles": [{"phase": "build"}], + "tools": {"components": tools}, + **({"manufacturer": self.supplier} if self.supplier else {}), + **({"authors": authors} if authors else {}), + **({"supplier": self.supplier} if self.supplier else {}), + "component": root, + **({"licenses": licenses} if licenses else {}), + "properties": [{"name": f"{namespace}:sbom:scope", "value": self._scope()}], + }, + "components": [*bundled, *declared], + "dependencies": [ + { + "ref": root["bom-ref"], + "dependsOn": [component["bom-ref"] for component in [*bundled, *declared]], + }, + *bundled_dependencies, + *tool_dependencies, + ], + # transitive runtime dependencies are unknown until install time + "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], + "formulation": [ + { + "bom-ref": "formula:wheel-build", + "workflows": [ + { + "bom-ref": "workflow:wheel-build", + "uid": "wheel-build", + "name": "build the wheel and this SBOM", + "taskTypes": ["build"], + "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], + "outputs": [{"type": "artifact", "resource": {"ref": root["bom-ref"]}}], + "inputs": [ + { + "environmentVars": [ + {"name": "SOURCE_DATE_EPOCH", "value": str(get_reproducible_timestamp())} + ] + } + ], + } + ], + } + ], + }, + uuid.uuid5(uuid.NAMESPACE_URL, f"{base}/sboms"), + ) + + @staticmethod + def _project_url_repository(metadata: Message) -> str | None: + return next( + ( + url.strip().rstrip("/") + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) + if external_reference(label.strip(), url.strip())["type"] == "vcs" + ), + None, + ) + + def _root_component(self, metadata: Message, wheel_name: str, repository: str | None) -> Component: + name, version = metadata["Name"], metadata["Version"] + namespace = canonicalize_name(name) + references: Final[list[ExternalReference]] = [ + external_reference(label.strip(), url.strip()) + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) + ] + references.extend([ + {"type": "distribution", "url": f"https://pypi.org/project/{name}/{version}/"}, + {"type": "attestation", "url": f"https://pypi.org/integrity/{name}/{version}/{wheel_name}/provenance"}, + ]) + if repository and urlsplit(repository).netloc == "github.com": + references.extend([ + {"type": "security-contact", "url": f"{repository}/security/policy"}, + {"type": "advisories", "url": f"{repository}/security/advisories"}, + ]) + license_files: Final[list[str]] = metadata.get_all("License-File", []) + if repository and license_files: + references.append({"type": "license", "url": f"{repository}/blob/{self.default_branch}/{license_files[0]}"}) + properties: Final[list[Property]] = [] + if requires_python := metadata.get("Requires-Python"): + properties.append({"name": f"{namespace}:requires-python", "value": requires_python}) + properties.extend({"name": "python:classifier", "value": value} for value in metadata.get_all("Classifier", [])) + properties.extend( + {"name": "python:keyword", "value": keyword.strip()} + for keyword in (metadata.get("Keywords") or "").split(",") + if keyword.strip() + ) + if commit := self._commit(): + if repository: + references.append({ + "type": "vcs", + "url": f"{repository}/tree/{commit}", + "comment": "exact source revision", + }) + properties.append({"name": f"{namespace}:vcs-commit", "value": commit}) + authors = _authors(metadata) + licenses = _licenses(metadata) + copyright_line = self._copyright(license_files) + package = purl(name, version) + return { + "type": "application", + "bom-ref": package, + **({"supplier": self.supplier} if self.supplier else {}), + **({"authors": authors} if authors else {}), + "name": name, + "version": version, + **({"description": summary} if (summary := metadata.get("Summary")) else {}), + **({"licenses": licenses} if licenses else {}), + **({"copyright": copyright_line} if copyright_line else {}), + "purl": package, + "externalReferences": references, + "properties": properties, + } + + def _commit(self) -> str | None: + if not (self.root / ".git").exists() or (git := shutil.which("git")) is None: # building from an sdist + return None + return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] # fixed arguments + [git, "rev-parse", "HEAD"], check=True, capture_output=True, text=True, cwd=self.root + ).stdout.strip() + + def _copyright(self, license_files: list[str]) -> str | None: + return next( + ( + line + for license_file in license_files + if (path := self.root / license_file).is_file() + for line in path.read_text(encoding="utf-8").splitlines() + if line.startswith("Copyright") + ), + None, + ) + + def _bundled_component(self, wheel: Path, namespace: str, repository: str | None) -> Component: + with zipfile.ZipFile(wheel) as archive: + members: Final[list[str]] = sorted(archive.namelist()) + metadata_name: Final[str] = next( + name for name in members if name.count("/") == 1 and name.endswith(".dist-info/METADATA") + ) + metadata = Parser().parsestr(archive.read(metadata_name).decode("utf-8")) + record = archive.read(metadata_name.replace("METADATA", "RECORD")).decode("utf-8") + vendored: Final[list[tuple[str, Message]]] = [ + *_vendor_manifest(wheel, archive, members), + *( + (name, Parser().parsestr(archive.read(name).decode("utf-8"))) + for name in members + if "/_vendor/" in name and name.endswith(".dist-info/METADATA") + ), + ] + component = component_from_metadata(metadata, "library") + # a bundled wheel published from the same forge account as this project shares its supplier + if ( + self.supplier + and repository + and any(reference["url"].startswith(_owner(repository)) for reference in component["externalReferences"]) + ): + component["supplier"] = self.supplier + sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + component["hashes"] = [{"alg": "SHA-256", "content": sha256}] + component["externalReferences"].append( + {"type": "distribution", "url": f"https://pypi.org/project/{metadata['Name']}/{metadata['Version']}/"}, + ) + component["properties"] = [ + {"name": f"{namespace}:bundled-wheel", "value": wheel.relative_to(self.root).as_posix()}, + *self.bundled_wheel_properties(wheel), + *component["properties"], + ] + component["evidence"] = { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": metadata_name}], + }, + { + "field": "hash", + "confidence": 1, + "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], + }, + ], + } + component["components"] = [ + file_component(component["bom-ref"], path, digest, size) + for path, digest, size in (row for row in csv.reader(StringIO(record, newline="")) if row) + if digest + ] + children: Final[dict[str, Component]] = {} + for source, child_metadata in vendored: + child = component_from_metadata(child_metadata, "library") + child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['bom-ref']}" + child["properties"].append({"name": f"{namespace}:vendored-manifest", "value": source}) + child["evidence"] = { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], + } + ], + } + # a vendored .dist-info comes after the pip manifest line naming the same purl, so its metadata wins + children[purl(child["name"], child["version"])] = child + component["components"].extend(children[key] for key in sorted(children)) + return component + + def _bundled_wheel_paths(self) -> list[Path]: + found = {path for pattern in self.bundled_wheels for path in self.root.glob(pattern)} + return sorted(found, key=lambda path: path.relative_to(self.root).as_posix()) + + def _build_script_tool(self, script: PurePosixPath, version: str, repository: str | None) -> Component: + # a custom build hook shapes the document, so it is part of the toolchain a rebuild has to match + tool: Final[Component] = { + "type": "application", + "bom-ref": f"tool:{script}", + "name": script.name, + "version": version, + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256((self.root / script).read_bytes()).hexdigest()}], + } + if repository: + tool["externalReferences"] = [{"type": "vcs", "url": f"{repository}/blob/{self.default_branch}/{script}"}] + return tool + + def _scope(self) -> str: + if directories := sorted({str(PurePosixPath(pattern).parent) for pattern in self.bundled_wheels}): + return ( + f"Components are the wheels bundled as data files under {' and '.join(directories)} and the direct" + " runtime dependencies declared in the wheel's core metadata. The latter carry no version because" + " they are resolved at install time, not at build time." + ) + return ( + "Components are the direct runtime dependencies declared in the wheel's core metadata. They carry no" + " version because they are resolved at install time, not at build time." + ) + + +def sbom_file_name(metadata: Message) -> str: + return f"{canonicalize_name(metadata['Name'])}.cdx.json" + + +def _runtime_requirements(metadata: Message) -> list[str]: + # an extra's requirements apply only when the extra is requested, so they are not runtime dependencies of the wheel + return [ + requirement + for requirement in metadata.get_all("Requires-Dist", []) + if (marker := Requirement(requirement).marker) is None or "extra" not in str(marker) + ] + + +def _declared_dependency(requirement: str, namespace: str) -> Component: + parsed = Requirement(requirement) + component: Final[Component] = { + "type": "library", + # the same distribution can be declared more than once with different markers, so the purl is not unique + "bom-ref": f"requires-dist:{requirement}", + "name": parsed.name, + "purl": purl(parsed.name), + "externalReferences": [{"type": "distribution", "url": f"https://pypi.org/project/{parsed.name}/"}], + "properties": [{"name": f"{namespace}:requires-dist", "value": requirement}], + } + if parsed.marker is not None: + component["properties"].append({"name": f"{namespace}:environment-marker", "value": str(parsed.marker)}) + return component + + +def _authors(metadata: Message) -> list[Contact]: + # core metadata joins several names into one header + if people := contacts( + [name for entry in metadata.get_all("Maintainer", []) for name in entry.split(", ")], + metadata.get_all("Maintainer-email", []), + ): + return people + return contacts( + [name for entry in metadata.get_all("Author", []) for name in entry.split(", ")], + metadata.get_all("Author-email", []), + ) + + +def _licenses(metadata: Message) -> list[License]: + if expression := metadata.get("License-Expression"): + return [{"expression": expression, "acknowledgement": "declared"}] + return [] + + +def _vendor_manifest(wheel: Path, archive: zipfile.ZipFile, members: list[str]) -> list[tuple[str, Message]]: + # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info + found: Final[list[tuple[str, Message]]] = [] + for name in members: + if not name.endswith("/_vendor/vendor.txt"): + continue + for line in archive.read(name).decode("utf-8").splitlines(): + if not (requirement := line.partition("#")[0].strip()): + continue + parsed = Requirement(requirement) + pins = list(parsed.specifier) + if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: + msg = f"Unpinned vendored dependency in {wheel.name}:{name}: {requirement}" + raise ValueError(msg) + found.append((name, Parser().parsestr(f"Name: {parsed.name}\nVersion: {pins[0].version}\n"))) + return found + + +def _owner(repository: str) -> str: + parts = urlsplit(repository) + return f"{parts.scheme}://{parts.netloc}/{parts.path.strip('/').split('/')[0]}/" + + +__all__ = [ + "BundledWheelProperties", + "Project", + "Toml", + "sbom_file_name", +] diff --git a/src/hatch_pep770/_run.py b/src/hatch_pep770/_run.py new file mode 100644 index 0000000..21553a9 --- /dev/null +++ b/src/hatch_pep770/_run.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import json +import sys +import tomllib +from typing import TYPE_CHECKING + +from ._embed import embed +from ._project import Project +from ._spdx import to_spdx +from ._validate import validate_wheel + +if TYPE_CHECKING: + from ._cli import Options + + +def run(options: Options) -> bool: + if options.command == "embed": + pyproject = options.root / "pyproject.toml" + table = tomllib.loads(pyproject.read_text(encoding="utf-8")).get("tool", {}).get("hatch-pep770", {}) + project = Project.from_table(table, root=options.root, where=f"[tool.hatch-pep770] of {pyproject}") + for wheel in options.wheels: + sys.stdout.write(f"wrote {embed(wheel, project)} into {wheel.name}\n") + return True + if options.command == "spdx": + bom = json.loads(options.source.read_text(encoding="utf-8")) + options.target.write_bytes(f"{json.dumps(to_spdx(bom), indent=2)}\n".encode()) + return True + allowed = None if options.allowed_licenses is None else frozenset(options.allowed_licenses) + wheels = sorted(wheel for path in options.paths for wheel in (path.glob("*.whl") if path.is_dir() else [path])) + if not wheels: + sys.stderr.write(f"no wheel found in {', '.join(map(str, options.paths))}\n") + return False + valid = True + for wheel in wheels: + for problem in (problems := validate_wheel(wheel, allowed)): + sys.stderr.write(f"SBOM invalid: {wheel.name}: {problem}\n") + if problems: + valid = False + else: + sys.stdout.write(f"SBOM in {wheel.name} is valid\n") + return valid + + +__all__ = [ + "run", +] diff --git a/src/hatch_pep770/spdx.py b/src/hatch_pep770/_spdx.py similarity index 88% rename from src/hatch_pep770/spdx.py rename to src/hatch_pep770/_spdx.py index afcb2c7..3da6484 100644 --- a/src/hatch_pep770/spdx.py +++ b/src/hatch_pep770/_spdx.py @@ -1,17 +1,15 @@ -"""Render a CycloneDX SBOM written by the build hook as an SPDX 2.3 JSON document.""" - from __future__ import annotations import re -from datetime import datetime, timezone -from typing import TYPE_CHECKING, Final, TypedDict +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Final, NotRequired, TypedDict -from .version import __version__ +from .version import version if TYPE_CHECKING: from collections.abc import Iterator - from .cyclonedx import Component, Document, License + from ._cyclonedx import Component, Document, License class _Checksum(TypedDict): @@ -25,7 +23,7 @@ class _ExternalRef(TypedDict): referenceLocator: str -class _PackageKeys(TypedDict): +class _Package(TypedDict): SPDXID: str name: str downloadLocation: str @@ -33,18 +31,15 @@ class _PackageKeys(TypedDict): licenseConcluded: str licenseDeclared: str primaryPackagePurpose: str - - -class _Package(_PackageKeys, total=False): - versionInfo: str - supplier: str - checksums: list[_Checksum] - homepage: str - copyrightText: str - summary: str - comment: str - externalRefs: list[_ExternalRef] - licenseComments: str + versionInfo: NotRequired[str] + supplier: NotRequired[str] + checksums: NotRequired[list[_Checksum]] + homepage: NotRequired[str] + copyrightText: NotRequired[str] + summary: NotRequired[str] + comment: NotRequired[str] + externalRefs: NotRequired[list[_ExternalRef]] + licenseComments: NotRequired[str] class _Relationship(TypedDict): @@ -59,6 +54,8 @@ class _CreationInfo(TypedDict): class Spdx(TypedDict): + """An SPDX 2.3 JSON document.""" + spdxVersion: str dataLicense: str SPDXID: str @@ -70,13 +67,6 @@ class Spdx(TypedDict): relationships: list[_Relationship] -_PURPOSES: Final[dict[str, str]] = { - "application": "APPLICATION", - "library": "LIBRARY", - "platform": "OTHER", -} - - def to_spdx(bom: Document) -> Spdx: """ Render a CycloneDX document as SPDX 2.3. @@ -85,7 +75,7 @@ def to_spdx(bom: Document) -> Spdx: over. Per-file hashes stay behind, because SPDX 2.3 requires a SHA-1 for every file and wheel ``RECORD`` files hold SHA-256, and so does the build formulation, which SPDX 2.3 has no field for. - :param bom: a CycloneDX document as the build hook writes it + :param bom: a CycloneDX document as this package writes it :returns: the SPDX document, whose namespace ends in the CycloneDX serial number so both name the same build """ @@ -116,12 +106,12 @@ def to_spdx(bom: Document) -> Spdx: "creationInfo": { "created": datetime .fromisoformat(bom["metadata"]["timestamp"]) - .astimezone(timezone.utc) + .astimezone(UTC) .strftime("%Y-%m-%dT%H:%M:%SZ"), "creators": [ *supplier, *(f"Tool: {tool['name']}-{tool['version']}" for tool in tools if tool["type"] == "application"), - f"Tool: hatch-pep770-{__version__}", + f"Tool: hatch-pep770-{version}", ], }, "packages": [_package(component, f"{namespace}:") for component in components], @@ -165,7 +155,9 @@ def _package(component: Component, property_prefix: str) -> _Package: "filesAnalyzed": False, "licenseConcluded": " AND ".join(concluded) or "NOASSERTION", "licenseDeclared": next((entry["expression"] for entry in licenses if "expression" in entry), "NOASSERTION"), - "primaryPackagePurpose": _PURPOSES[component["type"]], + "primaryPackagePurpose": {"application": "APPLICATION", "library": "LIBRARY", "platform": "OTHER"}[ + component["type"] + ], } if version := component.get("version"): package["versionInfo"] = version diff --git a/src/hatch_pep770/validate.py b/src/hatch_pep770/_validate.py similarity index 69% rename from src/hatch_pep770/validate.py rename to src/hatch_pep770/_validate.py index 04cba3e..51b1d6c 100644 --- a/src/hatch_pep770/validate.py +++ b/src/hatch_pep770/_validate.py @@ -1,5 +1,3 @@ -"""Check an SBOM against the CycloneDX 1.6 schema, its own references, its SPDX rendering and a license policy.""" - from __future__ import annotations import json @@ -7,40 +5,22 @@ import zipfile from typing import TYPE_CHECKING, Final -from cyclonedx.schema import SchemaVersion -from cyclonedx.validation.json import JsonStrictValidator -from spdx_tools.spdx.parser.error import SPDXParsingError -from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser -from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document - -from .spdx import to_spdx +from ._spdx import to_spdx if TYPE_CHECKING: from collections.abc import Iterator from pathlib import Path - from .cyclonedx import Component, Document, License - -_SERIAL_PATTERN: Final[re.Pattern[str]] = re.compile( - r""" - ^urn:uuid: - (?P[0-9a-f]{8})- - (?P[0-9a-f]{4})- - (?P[0-9a-f]{4})- - (?P[0-9a-f]{4})- - (?P[0-9a-f]{12}) - $ - """, - re.VERBOSE, -) + from ._cyclonedx import Component, Document, License + # SPDX matches operators only in upper case _OPERATORS: Final[frozenset[str]] = frozenset({"AND", "OR", "WITH", "(", ")"}) _TOKEN: Final[re.Pattern[str]] = re.compile(r"[()]|[^\s()]+") -def wheel_problems(wheel: Path, allowed_licenses: frozenset[str] | None) -> list[str]: +def validate_wheel(wheel: Path, allowed_licenses: frozenset[str] | None) -> list[str]: """ - Check every PEP 770 CycloneDX SBOM in a wheel. + Check every PEP 770 CycloneDX SBOM in a wheel; needs the ``validate`` extra. :param wheel: the wheel to read :param allowed_licenses: SPDX license ids the shipped components may use, or ``None`` to skip the license policy @@ -55,12 +35,12 @@ def wheel_problems(wheel: Path, allowed_licenses: frozenset[str] | None) -> list } if not sboms: return ["no .dist-info/sboms/*.cdx.json found in the wheel"] - return [f"{name}: {problem}" for name, raw in sboms.items() for problem in document_problems(raw, allowed_licenses)] + return [f"{name}: {problem}" for name, raw in sboms.items() for problem in validate(raw, allowed_licenses)] -def document_problems(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: +def validate(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: """ - Check one CycloneDX SBOM. + Check one CycloneDX SBOM; needs the ``validate`` extra. Beyond the schema it requires the ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, unique and resolvable ``bom-ref`` values, a root that depends on exactly the top-level components, and an SPDX 2.3 @@ -71,32 +51,39 @@ def document_problems(raw: str, allowed_licenses: frozenset[str] | None) -> list :returns: one message per problem, empty when the document is valid """ - if schema_error := JsonStrictValidator(SchemaVersion.V1_6).validate_str(raw): - return [f"does not conform to the CycloneDX 1.6 schema: {schema_error}"] + from ._extra import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # optional extra + + if error := schema_error(raw): + return [f"does not conform to the CycloneDX 1.6 schema: {error}"] document: Final[Document] = json.loads(raw) - if problems := _serial_problems(document.get("serialNumber", "")): - return problems + # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize the document + # as CycloneDX at all, and silently rejects anything missing it as an unknown format + if not re.fullmatch( + r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + serial := document.get("serialNumber", ""), + ): + return [f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {serial!r}"] problems = _reference_problems(document) - problems += _spdx_problems(document) + problems.extend(spdx_problems(to_spdx(document))) if allowed_licenses is not None: - problems += license_problems(document["components"], allowed_licenses) + # only what the artifact carries bytes for is judged: a declared requirement resolves at install time, and a + # library a bundled wheel vendors ships under that wheel's terms + problems.extend( + f"{component['bom-ref']}: {problem}" + for component in _shipped(document["components"]) + if (problem := _judge(component, allowed_licenses)) + ) return problems -def _serial_problems(serial: str) -> list[str]: - # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize the document - # as CycloneDX at all, and silently rejects anything missing it as an unknown format - return [] if _SERIAL_PATTERN.match(serial) else [f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {serial!r}"] - - def _reference_problems(document: Document) -> list[str]: root_ref = document["metadata"]["component"]["bom-ref"] tools = document["metadata"]["tools"]["components"] every_ref = list(_bom_refs([document["metadata"]["component"], *document["components"], *tools])) if duplicates := {ref for ref in every_ref if every_ref.count(ref) > 1}: return [f"bom-ref values are not unique: {sorted(duplicates)}"] - problems = [] dependencies = {entry["ref"]: set(entry["dependsOn"]) for entry in document["dependencies"]} + problems = [] if unknown := (set(dependencies) | set().union(*dependencies.values())) - set(every_ref): problems.append(f"dependencies reference bom-refs that do not exist: {sorted(unknown)}") if dependencies.get(root_ref) != (component_refs := {component["bom-ref"] for component in document["components"]}): @@ -118,34 +105,6 @@ def _bom_refs(components: list[Component]) -> Iterator[str]: yield from _bom_refs(component.get("components", [])) -def _spdx_problems(document: Document) -> list[str]: - try: - spdx = JsonLikeDictParser().parse(dict(to_spdx(document))) - except SPDXParsingError as error: - return [f"SPDX rendering does not parse: {message}" for message in error.get_messages()] - return [f"SPDX rendering invalid: {message.validation_message}" for message in validate_full_spdx_document(spdx)] - - -def license_problems(components: list[Component], allowed_licenses: frozenset[str]) -> list[str]: - """ - Judge the licenses of what an artifact ships against a policy. - - Only components the artifact carries bytes for are judged: a bundled wheel, or a component with files. A declared - requirement resolves at install time, and a library a bundled wheel vendors ships under that wheel's terms. - - :param components: the document's top-level components - :param allowed_licenses: SPDX license ids a shipped component may use; an expression passes when the ids it needs - are allowed, and ``WITH`` exceptions only grant extra permissions so the license they modify decides - :returns: one message per component that fails the policy - - """ - return [ - f"{component['bom-ref']}: {problem}" - for component in _shipped(components) - if (problem := _judge(component, allowed_licenses)) - ] - - def _shipped(components: list[Component]) -> Iterator[Component]: for component in components: if component["type"] == "library" and ( @@ -232,7 +191,6 @@ def _pop(tokens: list[str]) -> str: __all__ = [ - "document_problems", - "license_problems", - "wheel_problems", + "validate", + "validate_wheel", ] diff --git a/src/hatch_pep770/hooks.py b/src/hatch_pep770/hooks.py index 9138d5b..a5a51db 100644 --- a/src/hatch_pep770/hooks.py +++ b/src/hatch_pep770/hooks.py @@ -4,14 +4,15 @@ from hatchling.plugin import hookimpl -from .plugin import Pep770BuildHook +from ._hook import Pep770BuildHook @hookimpl def hatch_register_build_hook() -> type[Pep770BuildHook]: - return Pep770BuildHook + """ + Return the build hook hatchling loads for ``[tool.hatch.build.hooks.pep770]``. + :returns: the build hook class -__all__ = [ - "hatch_register_build_hook", -] + """ + return Pep770BuildHook diff --git a/src/hatch_pep770/plugin.py b/src/hatch_pep770/plugin.py deleted file mode 100644 index ccc706d..0000000 --- a/src/hatch_pep770/plugin.py +++ /dev/null @@ -1,453 +0,0 @@ -"""The hatchling build hook that writes a PEP 770 CycloneDX SBOM into the wheel.""" - -from __future__ import annotations - -import csv -import hashlib -import json -import shutil -import subprocess -import tempfile -import uuid -import zipfile -from email.parser import Parser -from functools import cached_property -from io import StringIO -from itertools import starmap -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Any, Final -from urllib.parse import urlsplit - -from hatchling.__about__ import __version__ as hatchling_version -from hatchling.builders.hooks.plugin.interface import BuildHookInterface -from hatchling.builders.utils import get_reproducible_timestamp -from hatchling.builders.wheel import WheelBuilder -from hatchling.utils.constants import DEFAULT_BUILD_SCRIPT -from packaging.requirements import Requirement - -from .cyclonedx import ( - build_tools, - component_from_metadata, - contacts, - document, - external_reference, - file_component, - purl, - timestamp, -) - -if TYPE_CHECKING: - from email.message import Message - - from hatchling.metadata.core import CoreMetadata - from packaging.specifiers import Specifier - - from .cyclonedx import ( - Body, - Component, - Contact, - Dependency, - Document, - ExternalReference, - License, - OrganizationalEntity, - Property, - Workflow, - ) - - -class Pep770BuildHook(BuildHookInterface): - """ - Write a PEP 770 CycloneDX 1.6 SBOM into the wheel's ``.dist-info/sboms/`` directory. - - The document describes the project from its core metadata, the runtime dependencies it declares, any wheels it - bundles as data files, and the build toolchain. It holds nothing about the build machine and takes its timestamp - from ``SOURCE_DATE_EPOCH``, so a rebuild with the same source, toolchain and epoch produces the same bytes on any - operating system. - - Subclass it in a hatchling custom build hook to add project-specific detail; see :meth:`bundled_wheel_properties`. - - """ - - PLUGIN_NAME = "pep770" - _sbom_directory: Path | None = None - - def initialize(self, version: str, build_data: dict[str, Any]) -> None: # ruff: ignore[unused-method-argument] - # `version` is hatchling's build variant name (e.g. "standard"), not the package version - if not isinstance(builder := self.build_config.builder, WheelBuilder): - return - if "sbom_files" not in build_data: # pragma: <3.10 cover # hatchling 1.28, the first with it, needs 3.10 - self.app.display_warning( - f"hatchling {hatchling_version} cannot embed SBOMs (1.28 adds it), so none written" - ) - return - tag = build_data.get("tag") or ( - builder.get_best_matching_tag() if build_data["infer_tag"] else builder.get_default_tag() - ) - content = json.dumps(self._document(f"{builder.artifact_project_id}-{tag}.whl"), indent=2) - # written outside self.directory, which hatchling also uses as the wheel's output directory, so no stray copy - # lands next to the built artifacts - self._sbom_directory = Path(tempfile.mkdtemp(prefix="hatch-pep770-")) - # bytes, so the line endings do not depend on the build machine's OS - (out := self._sbom_directory / f"{self._namespace}.cdx.json").write_bytes(f"{content}\n".encode()) - build_data["sbom_files"].append(str(out)) - - def finalize(self, version: str, build_data: dict[str, Any], artifact_path: str) -> None: # ruff: ignore[unused-method-argument] - if self._sbom_directory is not None: - shutil.rmtree(self._sbom_directory) - self._sbom_directory = None - - def bundled_wheel_properties(self, wheel: Path) -> list[Property]: # ruff: ignore[unused-method-argument, no-self-use] - """ - Return extra properties for one wheel the project bundles as a data file. - - The default adds none. A subclass overrides it to say what the project does with the wheel, for example which - Python versions receive it. The properties follow ``:bundled-wheel`` and precede the ones taken from - the wheel's metadata. - - :param wheel: the bundled wheel, an absolute path under the project root - :returns: the properties to add - - """ - return [] - - @property - def _namespace(self) -> str: - # hatchling normalizes the name; it also prefixes the CycloneDX property names this hook writes - return self.metadata.core.name - - def _document(self, wheel_name: str) -> Document: - core: Final[CoreMetadata] = self.metadata.core - version: Final[str] = self.metadata.version - root = self._root_component(core, version, wheel_name) - bundled = [self._bundled_component(wheel) for wheel in self._bundled_wheels] - declared = [self._declared_dependency(requirement) for requirement in core.dependencies] - tools, tool_dependencies = build_tools() - if script := self._build_script_tool(version): - tools.insert(0, script) - bundled_dependencies: Final[list[Dependency]] = [ - {"ref": component["bom-ref"], "dependsOn": references} - for component in bundled - if (references := [child["bom-ref"] for child in component["components"] if child["type"] == "library"]) - ] - supplier = self._supplier - body: Final[Body] = { - "metadata": { - "timestamp": timestamp(), - "lifecycles": [{"phase": "build"}], - "tools": {"components": tools}, - **({"manufacturer": supplier} if supplier else {}), - **({"authors": self._authors} if self._authors else {}), - **({"supplier": supplier} if supplier else {}), - "component": root, - **({"licenses": self._licenses} if self._licenses else {}), - "properties": [{"name": f"{self._namespace}:sbom:scope", "value": self._scope}], - }, - "components": [*bundled, *declared], - "dependencies": [ - {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in [*bundled, *declared]]}, - *bundled_dependencies, - *tool_dependencies, - ], - # transitive runtime dependencies are unknown until install time - "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], - "formulation": [{"bom-ref": "formula:wheel-build", "workflows": [self._workflow(root, tools)]}], - } - base: Final[str] = self._repository or f"https://pypi.org/project/{self._namespace}" - return document(body, uuid.uuid5(uuid.NAMESPACE_URL, f"{base}/sboms")) - - def _root_component(self, core: CoreMetadata, version: str, wheel_name: str) -> Component: - package = purl(core.name, version) - references: Final[list[ExternalReference]] = list(starmap(external_reference, core.urls.items())) - references.extend([ - {"type": "distribution", "url": f"https://pypi.org/project/{core.name}/{version}/"}, - {"type": "attestation", "url": f"https://pypi.org/integrity/{core.name}/{version}/{wheel_name}/provenance"}, - ]) - if (repository := self._repository) and urlsplit(repository).netloc == "github.com": - references.extend([ - {"type": "security-contact", "url": f"{repository}/security/policy"}, - {"type": "advisories", "url": f"{repository}/security/advisories"}, - ]) - if repository and core.license_files: - references.append({"type": "license", "url": f"{repository}/blob/{self._branch}/{core.license_files[0]}"}) - properties: Final[list[Property]] = [] - if core.requires_python: - properties.append({"name": f"{self._namespace}:requires-python", "value": core.requires_python}) - properties.extend({"name": "python:classifier", "value": classifier} for classifier in core.classifiers) - properties.extend({"name": "python:keyword", "value": keyword} for keyword in core.keywords) - if commit := self._commit(): - if repository: - references.append({ - "type": "vcs", - "url": f"{repository}/tree/{commit}", - "comment": "exact source revision", - }) - properties.append({"name": f"{self._namespace}:vcs-commit", "value": commit}) - supplier = self._supplier - copyright_line = self._copyright(core.license_files) - return { - "type": "application", - "bom-ref": package, - **({"supplier": supplier} if supplier else {}), - **({"authors": self._authors} if self._authors else {}), - "name": core.name, - "version": version, - **({"description": core.description} if core.description else {}), - **({"licenses": self._licenses} if self._licenses else {}), - **({"copyright": copyright_line} if copyright_line else {}), - "purl": package, - "externalReferences": references, - "properties": properties, - } - - @cached_property - def _supplier(self) -> OrganizationalEntity | None: - if (supplier := self.config.get("supplier")) is None: - return None - if not ( - isinstance(supplier, dict) - and isinstance(name := supplier.get("name"), str) - and isinstance(urls := supplier.get("url", []), list) - and all(isinstance(url, str) for url in urls) - and supplier.keys() <= {"name", "url"} - ): - msg = f"{self._option('supplier')} must be a table with a string `name` and an optional `url` string array" - raise TypeError(msg) - return {"name": name, "url": urls} - - def _option(self, name: str) -> str: - return f"Option `{name}` of build hook `{self.PLUGIN_NAME}`" - - @cached_property - def _authors(self) -> list[Contact]: - maintainers = self.metadata.core.maintainers_data - authors = self.metadata.core.authors_data - people = maintainers if maintainers["name"] or maintainers["email"] else authors - return contacts(people["name"], people["email"]) - - @cached_property - def _licenses(self) -> list[License]: - if expression := self.metadata.core.license_expression: - return [{"expression": expression, "acknowledgement": "declared"}] - return [] - - @cached_property - def _repository(self) -> str | None: - if (repository := self.config.get("repository")) is not None: - if not isinstance(repository, str): - msg = f"{self._option('repository')} must be a string" - raise TypeError(msg) - return repository.rstrip("/") - return next( - ( - url.rstrip("/") - for label, url in self.metadata.core.urls.items() - if external_reference(label, url)["type"] == "vcs" - ), - None, - ) - - @cached_property - def _branch(self) -> str: - if not isinstance(branch := self.config.get("default-branch", "main"), str): - msg = f"{self._option('default-branch')} must be a string" - raise TypeError(msg) - return branch - - def _commit(self) -> str | None: - if not (Path(self.root) / ".git").exists() or (git := shutil.which("git")) is None: # building from an sdist - return None - try: - result = subprocess.run( - [git, "rev-parse", "HEAD"], check=True, capture_output=True, text=True, cwd=self.root - ) - except subprocess.CalledProcessError as error: - self.app.display_warning(f"git rev-parse HEAD failed, so the SBOM names no source revision: {error.stderr}") - return None - return result.stdout.strip() - - def _copyright(self, license_files: list[str]) -> str | None: - for license_file in license_files: - for line in (Path(self.root) / license_file).read_text(encoding="utf-8").splitlines(): - if line.startswith("Copyright"): - return line - return None - - @cached_property - def _bundled_wheels(self) -> list[Path]: - root = Path(self.root) - found = {path for pattern in self._bundled_patterns for path in root.glob(pattern)} - return sorted(found, key=lambda path: path.relative_to(root).as_posix()) - - @cached_property - def _bundled_patterns(self) -> list[str]: - patterns = self.config.get("bundled-wheels", []) - if not (isinstance(patterns, list) and all(isinstance(pattern, str) for pattern in patterns)): - msg = f"{self._option('bundled-wheels')} must be an array of strings" - raise TypeError(msg) - return patterns - - def _bundled_component(self, wheel: Path) -> Component: - with zipfile.ZipFile(wheel) as archive: - members: Final[list[str]] = sorted(archive.namelist()) - metadata_name: Final[str] = next( - name for name in members if name.count("/") == 1 and name.endswith(".dist-info/METADATA") - ) - metadata = Parser().parsestr(archive.read(metadata_name).decode("utf-8")) - record = archive.read(metadata_name.replace("METADATA", "RECORD")).decode("utf-8") - vendored_metadata: Final[list[tuple[str, Message]]] = [ - *self._vendor_manifest(wheel, archive, members), - *( - (name, Parser().parsestr(archive.read(name).decode("utf-8"))) - for name in members - if "/_vendor/" in name and name.endswith(".dist-info/METADATA") - ), - ] - component = component_from_metadata(metadata, "library") - if ( - (supplier := self._supplier) - and (owner := self._repository_owner) - and any(reference["url"].startswith(owner) for reference in component["externalReferences"]) - ): - component["supplier"] = supplier - sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() - component["hashes"] = [{"alg": "SHA-256", "content": sha256}] - component["externalReferences"].append( - {"type": "distribution", "url": f"https://pypi.org/project/{metadata['Name']}/{metadata['Version']}/"}, - ) - component["properties"] = [ - {"name": f"{self._namespace}:bundled-wheel", "value": wheel.relative_to(self.root).as_posix()}, - *self.bundled_wheel_properties(wheel), - *component["properties"], - ] - component["evidence"] = { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": metadata_name}], - }, - { - "field": "hash", - "confidence": 1, - "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], - }, - ], - } - component["components"] = [ - file_component(component["bom-ref"], path, digest, size) - for path, digest, size in (row for row in csv.reader(StringIO(record, newline="")) if row) - if digest - ] - vendored_components: Final[dict[str, Component]] = {} - for source, vendored in vendored_metadata: - child: Final[Component] = component_from_metadata(vendored, "library") - child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['purl']}" - child["properties"].append({"name": f"{self._namespace}:vendored-manifest", "value": source}) - child["evidence"] = { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], - } - ], - } - # a distribution's own .dist-info wins over the pip-style manifest line naming the same purl - vendored_components[child["purl"]] = child - component["components"].extend(vendored_components[key] for key in sorted(vendored_components)) - return component - - @staticmethod - def _vendor_manifest(wheel: Path, archive: zipfile.ZipFile, members: list[str]) -> list[tuple[str, Message]]: - # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info - found: Final[list[tuple[str, Message]]] = [] - for name in members: - if not name.endswith("/_vendor/vendor.txt"): - continue - for line in archive.read(name).decode("utf-8").splitlines(): - if not (requirement := line.partition("#")[0].strip()): - continue - parsed: Final[Requirement] = Requirement(requirement) - pins: Final[list[Specifier]] = list(parsed.specifier) - if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: - msg: Final[str] = f"Unpinned vendored dependency in {wheel.name}:{name}: {requirement}" - raise ValueError(msg) - found.append((name, Parser().parsestr(f"Name: {parsed.name}\nVersion: {pins[0].version}\n"))) - return found - - @cached_property - def _repository_owner(self) -> str | None: - # a bundled wheel published from the same forge account as this project shares its supplier - if (repository := self._repository) is None: - return None - parts = urlsplit(repository) - return f"{parts.scheme}://{parts.netloc}/{parts.path.strip('/').split('/')[0]}/" - - def _declared_dependency(self, requirement: str) -> Component: - parsed = Requirement(requirement) - component: Final[Component] = { - "type": "library", - # the same distribution can be declared more than once with different markers, so the purl is not unique - "bom-ref": f"requires-dist:{requirement}", - "name": parsed.name, - "purl": purl(parsed.name), - "externalReferences": [{"type": "distribution", "url": f"https://pypi.org/project/{parsed.name}/"}], - "properties": [{"name": f"{self._namespace}:requires-dist", "value": requirement}], - } - if parsed.marker is not None: - component["properties"].append({ - "name": f"{self._namespace}:environment-marker", - "value": str(parsed.marker), - }) - return component - - def _build_script_tool(self, version: str) -> Component | None: - # a custom build hook script shapes the document, so it is part of the toolchain a rebuild has to match - if self.PLUGIN_NAME != "custom": - return None - script = PurePosixPath(self.config.get("path", DEFAULT_BUILD_SCRIPT)) - tool: Final[Component] = { - "type": "application", - "bom-ref": f"tool:{script}", - "name": script.name, - "version": version, - "hashes": [ - {"alg": "SHA-256", "content": hashlib.sha256((Path(self.root) / script).read_bytes()).hexdigest()} - ], - } - if repository := self._repository: - tool["externalReferences"] = [{"type": "vcs", "url": f"{repository}/blob/{self._branch}/{script}"}] - return tool - - @staticmethod - def _workflow(root: Component, tools: list[Component]) -> Workflow: - return { - "bom-ref": "workflow:wheel-build", - "uid": "wheel-build", - "name": "build the wheel and this SBOM", - "taskTypes": ["build"], - "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], - "outputs": [{"type": "artifact", "resource": {"ref": root["bom-ref"]}}], - "inputs": [ - {"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": str(get_reproducible_timestamp())}]}, - ], - } - - @cached_property - def _scope(self) -> str: - declared = ( - "the direct runtime dependencies declared in the wheel's core metadata. The latter carry no version" - " because they are resolved at install time, not at build time." - ) - if directories := sorted({str(PurePosixPath(pattern).parent) for pattern in self._bundled_patterns}): - return f"Components are the wheels bundled as data files under {' and '.join(directories)} and {declared}" - return ( - "Components are the direct runtime dependencies declared in the wheel's core metadata. They carry no" - " version because they are resolved at install time, not at build time." - ) - - -__all__ = [ - "Pep770BuildHook", -] diff --git a/src/hatch_pep770/py.typed b/src/hatch_pep770/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index 19055f8..84d09d9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,35 +4,61 @@ import csv import hashlib import json +import shutil import zipfile from io import StringIO from pathlib import Path -from typing import TYPE_CHECKING, Any, Final, Protocol +from typing import TYPE_CHECKING, Final, Protocol import pytest from hatchling.builders.wheel import WheelBuilder -if TYPE_CHECKING: - from collections.abc import Callable +from hatch_pep770 import Project - from hatch_pep770.cyclonedx import Document +if TYPE_CHECKING: + from hatch_pep770 import Document + +_PROJECT: Final[str] = """\ +[project] +name = "Example" +version = "1.0" +description = "an example" +requires-python = ">=3.11" +license = "MIT" +license-files = ["LICENSE"] +maintainers = [{ name = "Maintainer", email = "maintainer@example.com" }] +keywords = ["sample", "demo"] +classifiers = ["Programming Language :: Python :: 3"] +dependencies = ["packaging>=24", "tomli>=2; python_version<'3.11'"] +optional-dependencies.cli = ["click>=8"] +urls.Changelog = "https://example.com/changelog" +urls.Source = "https://github.com/example/example" +""" +HOOK: Final[str] = "[tool.hatch.build.hooks.pep770]\n" +BUNDLED: Final[str] = 'bundled-wheels = ["src/example/wheels/*.whl"]\n' +_SEEDER: Final[Path] = Path(__file__).parent / "seeder" + + +class WriteProject(Protocol): + def __call__(self, *, hook: str = HOOK, project: str = _PROJECT) -> Path: ... class BundleWheel(Protocol): def __call__(self, files: dict[str, str], metadata: str = "", name: str = "pip") -> Path: ... -class InstallTool(Protocol): - def __call__(self, site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> None: ... +class MakeWheel(Protocol): + def __call__(self, metadata: str) -> Path: ... -@pytest.fixture(autouse=True) -def _clean_environment(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) +class InstallTool(Protocol): + def __call__(self, site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> None: ... @pytest.fixture -def root(tmp_path: Path) -> Path: +def root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + # an exported SOURCE_DATE_EPOCH would move every timestamp the tests expect + monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) (package := tmp_path / "project" / "src" / "example").mkdir(parents=True) (package / "__init__.py").write_text("", encoding="utf-8") (tmp_path / "project" / "LICENSE").write_text("Copyright (c) 2026 Example\n\nMIT terms\n", encoding="utf-8") @@ -40,54 +66,52 @@ def root(tmp_path: Path) -> Path: @pytest.fixture -def config() -> dict[str, Any]: - return { - "project": { - "name": "Example", - "version": "1.0", - "description": "an example", - "requires-python": ">=3.9", - "license": "MIT", - "license-files": ["LICENSE"], - "maintainers": [{"name": "Maintainer", "email": "maintainer@example.com"}], - "keywords": ["sample"], - "classifiers": ["Programming Language :: Python :: 3"], - "dependencies": ["packaging>=24", "tomli>=2; python_version<'3.11'"], - "urls": {"Changelog": "https://example.com/changelog", "Source": "https://github.com/example/example"}, - }, - "tool": { - "hatch": { - "build": { - "targets": {"wheel": {"packages": ["src/example"]}}, - "hooks": {"pep770": {}}, - } - } - }, - } +def write_project(root: Path) -> WriteProject: + def write(*, hook: str = HOOK, project: str = _PROJECT) -> Path: + targets = '[tool.hatch.build.targets.wheel]\npackages = ["src/example"]\n' + (root / "pyproject.toml").write_text(f"{project}\n{targets}\n{hook}", encoding="utf-8") + return root + + return write @pytest.fixture -def build_sbom(root: Path) -> Callable[[dict[str, Any]], str]: - # drives the hook through hatchling's own wheel builder, also on hatchling releases that predate `sbom_files` - def build(config: dict[str, Any]) -> str: - builder: Final[WheelBuilder] = WheelBuilder(str(root), config=config) - build_data: Final[dict[str, Any]] = {**builder.get_default_build_data(), "sbom_files": []} - (hook,) = builder.get_build_hooks(str(root / "dist")).values() - hook.initialize("standard", build_data) - content = Path(build_data["sbom_files"][0]).read_text(encoding="utf-8") - hook.finalize("standard", build_data, str(root / "dist" / "example.whl")) - return content +def build_sbom(write_project: WriteProject) -> BuildSbom: + def build(*, hook: str = HOOK, project: str = _PROJECT) -> Document: + return build_hook_sbom(write_project(hook=hook, project=project)) return build -@pytest.fixture -def build_wheel(root: Path, tmp_path: Path) -> Callable[[dict[str, Any], str], Path]: - def build(config: dict[str, Any], out: str) -> Path: - (artifact,) = WheelBuilder(str(root), config=config).build(directory=str(tmp_path / out), versions=["standard"]) - return Path(artifact) +class BuildSbom(Protocol): + def __call__(self, *, hook: str = HOOK, project: str = _PROJECT) -> Document: ... - return build + +def build_hook_sbom(root: Path) -> Document: + builder: Final[WheelBuilder] = WheelBuilder(str(root)) + build_data = builder.get_default_build_data() + (hook,) = builder.get_build_hooks(str(root / "dist")).values() + hook.initialize("standard", build_data) + (sbom,) = build_data["sbom_files"] + document: Final[Document] = json.loads(Path(sbom).read_text(encoding="utf-8")) + hook.finalize("standard", build_data, str(root / "dist" / "example.whl")) + return document + + +@pytest.fixture +def seeder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) + # a copy, so the build cannot leave files in the source tree; the build script it loads stays the checked-in one + shutil.copytree(_SEEDER, target := tmp_path / "seeder", ignore=shutil.ignore_patterns("hatch_build.py")) + pyproject = (target / "pyproject.toml").read_text(encoding="utf-8") + script = (_SEEDER / "hatch_build.py").as_posix() + (target / "pyproject.toml").write_text( + pyproject.replace( + "build.hooks.custom.supplier", f'build.hooks.custom.path = "{script}"\nbuild.hooks.custom.supplier' + ), + encoding="utf-8", + ) + return target @pytest.fixture @@ -113,15 +137,46 @@ def bundle(files: dict[str, str], metadata: str = "", name: str = "pip") -> Path return bundle +@pytest.fixture +def make_wheel(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> MakeWheel: + # the layout another build backend, such as meson-python, writes: module first, .dist-info last, RECORD at the end + monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) + + def make(metadata: str) -> Path: + files: Final[dict[str, bytes]] = { + "native/__init__.py": b"", + "native-1.0.dist-info/METADATA": f"Metadata-Version: 2.4\nName: native\nVersion: 1.0\n{metadata}".encode(), + "native-1.0.dist-info/WHEEL": ( + b"Wheel-Version: 1.0\nGenerator: meson\nRoot-Is-Purelib: false\nTag: py3-none-any\n" + ), + } + record = "".join(f"{name},{digest(content)},{len(content)}\n" for name, content in files.items()) + (dist := tmp_path / "dist").mkdir(exist_ok=True) + with zipfile.ZipFile(wheel := dist / "native-1.0-py3-none-any.whl", "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in { + **files, + "native-1.0.dist-info/RECORD": f"{record}native-1.0.dist-info/RECORD,,\n".encode(), + }.items(): + info = zipfile.ZipInfo(name, (2024, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, content) + return wheel + + return make + + +def digest(content: bytes) -> str: + return f"sha256={base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b'=').decode()}" + + @pytest.fixture def install_tool() -> InstallTool: def install(site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> None: record: Final[StringIO] = StringIO(newline="") + csv.writer(record).writerows((path, digest(content), len(content)) for path, content in files.items()) for path, content in files.items(): (target := site / path).parent.mkdir(parents=True, exist_ok=True) target.write_bytes(content) - digest = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() - csv.writer(record).writerow((path, f"sha256={digest}", len(content))) (dist_info := site / f"{name}-1.0.dist-info").mkdir(exist_ok=True) (dist_info / "METADATA").write_text(f"Metadata-Version: 2.4\nName: {name}\nVersion: 1.0\n", encoding="utf-8") (dist_info / "WHEEL").write_text( @@ -133,5 +188,5 @@ def install(site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> No @pytest.fixture -def sbom_document(build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any]) -> Document: - return json.loads(build_sbom(config)) +def plain_project(tmp_path: Path) -> Project: + return Project.from_table({}, root=tmp_path, where="[tool.hatch-pep770]") diff --git a/tests/seeder/LICENSE b/tests/seeder/LICENSE new file mode 100644 index 0000000..b69c8d9 --- /dev/null +++ b/tests/seeder/LICENSE @@ -0,0 +1,3 @@ +Copyright (c) 2026 Example + +MIT terms diff --git a/tests/seeder/hatch_build.py b/tests/seeder/hatch_build.py new file mode 100644 index 0000000..fa93007 --- /dev/null +++ b/tests/seeder/hatch_build.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import ast +from pathlib import Path +from typing import TYPE_CHECKING + +from hatch_pep770 import Pep770BuildHook + +if TYPE_CHECKING: + from hatch_pep770 import Property + + +def get_build_hook() -> type[SeederBuildHook]: + # the module also sees the base class, so hatchling needs to be told which subclass to load + return SeederBuildHook + + +class SeederBuildHook(Pep770BuildHook): + def bundled_wheel_properties(self, wheel: Path) -> list[Property]: + # the table that decides which Python versions receive each wheel, read without importing the package + tree = ast.parse((Path(self.root) / "src" / "seeder" / "embed" / "bundle.py").read_text(encoding="utf-8")) + support: dict[str, dict[str, str]] = ast.literal_eval( + next(node.value for node in tree.body if isinstance(node, ast.AnnAssign) and node.value is not None) + ) + return [ + {"name": "seeder:seeded-for-python", "value": python} + for python, wheels in support.items() + if wheel.name in wheels.values() + ] diff --git a/tests/seeder/pyproject.toml b/tests/seeder/pyproject.toml new file mode 100644 index 0000000..8b063c9 --- /dev/null +++ b/tests/seeder/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +build-backend = "hatchling.build" +requires = [ "hatch-pep770", "hatchling>=1.28" ] + +[project] +name = "seeder" +version = "1.0" +description = "Seeds new environments with the wheels it bundles" +license = "MIT" +license-files = [ "LICENSE" ] +maintainers = [ { name = "Maintainer", email = "maintainer@example.com" } ] +requires-python = ">=3.11" +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", +] +urls.Source = "https://github.com/example/seeder" + +[tool.hatch] +build.hooks.custom.supplier = { name = "Example Org", url = [ "https://example.org" ] } +build.hooks.custom.bundled-wheels = [ "src/seeder/embed/*.whl" ] +build.targets.wheel.packages = [ "src/seeder" ] diff --git a/tests/seeder/src/seeder/__init__.py b/tests/seeder/src/seeder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/seeder/src/seeder/embed/__init__.py b/tests/seeder/src/seeder/embed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/seeder/src/seeder/embed/bundle.py b/tests/seeder/src/seeder/embed/bundle.py new file mode 100644 index 0000000..dbd6a41 --- /dev/null +++ b/tests/seeder/src/seeder/embed/bundle.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from typing import Final + +# the Python versions that receive each bundled wheel, by distribution name; the build hook reads it without importing +_BUNDLE_SUPPORT: Final[dict[str, dict[str, str]]] = { + "3.14": {"pip": "pip-1.0-py3-none-any.whl"}, + "3.15": {"pip": "pip-1.0-py3-none-any.whl"}, +} diff --git a/tests/seeder/src/seeder/embed/pip-1.0-py3-none-any.whl b/tests/seeder/src/seeder/embed/pip-1.0-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..b34215e4fc575672818da0f02df4dee7e7e1359a GIT binary patch literal 1080 zcmWIWW@Zs#U|`??Vy6Hg!N3F}fPf!}3o;Az8!n2&Gq2QKGGT>f}iuqitJFuNWI$Hh##2X38s_m`Qm+-IYKr1u;d} zP|rXwC9}9hH#09SU*Fd?#L>ku#PPGQuBVTuj^`C#Z(Xf(XU=a9GPq>?@RRo$->pF! zCv+Y(aZA6FEcltCw)~ms@>jdUL`0T~IUlLDb>6|$^=Wg^WbYE^=9)DkO`pBHCk2;^ z7d1!f*MhAJmVv^@ABe@Vx-#6u)zydf zpdnX-0T1hiy~|&)Ds@HcDdZ_FDqbQlYBF()?4G^vbJYtjbDf{Ad3;^wVTM+3!{-%U z%iM(y<%DINF$q0h`&Onqyv3ET@BQ`3vzlD?e_%!p@S9~**Z%-o?#sx)Ac55pL9WjJ zK`yyxc1~(CU|_g#FQ0Qcx5q|SCx+NS$0<|WIva$}c{E+W{@%j6By4>(i?E_+L3iY& z4^KEV8dF^V{=1~}Vsg>5bLAc0hg^jVPhC30ZqL8UEwVaA^60rOH+a9vbqIg*%d4DS zy7vSBdAXcDGJD?pZF(lpfBfH6y+^xwtMh;Vc`L+xYx%D=Gj%=7Q^N8!CtN8%=DFuY z`V1q!?a%!_BxFRmW%MZahONB1=Jms})V4`Sy*Pt^YQB1SR+8`Y!8xY4dEK5w`Dji^ zd3ji3r@*>nep3Sj4WGQ&^4M-HV-aa{9hKx9m2p>b_U)aM?bG(Zsxw z^}rMU0B=Sn5oX-!85m?>uml)>7^xdwGkWTVXk}nn!l)0^jFHyS%|K7v2s26{X5dKa y=;olOK7={aK(7F^DVjOpw1{p9dg?(KvH+_gkhB!w&B_K+$qIxaKy~N91q1-cq*8eR literal 0 HcmV?d00001 diff --git a/tests/test___main__.py b/tests/test___main__.py deleted file mode 100644 index 0469465..0000000 --- a/tests/test___main__.py +++ /dev/null @@ -1,62 +0,0 @@ -from __future__ import annotations - -import json -import sys -import zipfile -from typing import TYPE_CHECKING - -import pytest - -from hatch_pep770.__main__ import main -from hatch_pep770.spdx import to_spdx - -if TYPE_CHECKING: - from pathlib import Path - - from hatch_pep770.cyclonedx import Document - - -@pytest.fixture -def wheel(sbom_document: Document, tmp_path: Path) -> Path: - (dist := tmp_path / "dist").mkdir() - with zipfile.ZipFile(path := dist / "example-1.0-py3-none-any.whl", "w") as archive: - archive.writestr("example-1.0.dist-info/sboms/example.cdx.json", json.dumps(sbom_document)) - return path - - -def test_main_spdx(sbom_document: Document, tmp_path: Path) -> None: - (source := tmp_path / "example.cdx.json").write_text(json.dumps(sbom_document), encoding="utf-8") - assert (main(["spdx", str(source), str(target := tmp_path / "example.spdx.json")]), target.read_bytes()) == ( - 0, - f"{json.dumps(to_spdx(sbom_document), indent=2)}\n".encode(), - ) - - -@pytest.mark.parametrize( - "target", [pytest.param("example-1.0-py3-none-any.whl", id="wheel"), pytest.param(".", id="directory")] -) -def test_main_validate(wheel: Path, capsys: pytest.CaptureFixture[str], target: str) -> None: - assert (main(["validate", str(wheel.parent / target), "--allow-license", "MIT"]), capsys.readouterr().out) == ( - 0, - "SBOM in example-1.0-py3-none-any.whl is valid\n", - ) - - -def test_main_validate_problems(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: - archive.writestr("example-1.0.dist-info/METADATA", "Name: example\n") - assert (main(["validate", str(wheel)]), capsys.readouterr().err) == ( - 1, - "SBOM invalid: example-1.0-py3-none-any.whl: no .dist-info/sboms/*.cdx.json found in the wheel\n", - ) - - -def test_main_validate_no_wheel(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - assert (main(["validate", str(tmp_path)]), capsys.readouterr().err) == (1, f"no wheel found in {tmp_path}\n") - - -def test_main_module(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - monkeypatch.setattr(sys, "argv", ["hatch-pep770", "--help"]) - with pytest.raises(SystemExit): - main() - assert "usage: hatch-pep770" in capsys.readouterr().out diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..2c29892 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import json +import zipfile +from typing import TYPE_CHECKING, Final + +import pytest + +from hatch_pep770 import main, to_spdx + +if TYPE_CHECKING: + from pathlib import Path + + from conftest import BuildSbom, MakeWheel + + from hatch_pep770 import Document + + +def test_cli_embed(make_wheel: MakeWheel, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + wheel: Final[Path] = make_wheel("") + (tmp_path / "pyproject.toml").write_text('[tool.hatch-pep770]\nsupplier = { name = "Org" }\n', encoding="utf-8") + main(["embed", str(wheel), "--root", str(tmp_path)]) + with zipfile.ZipFile(wheel) as archive: + document: Final[Document] = json.loads(archive.read("native-1.0.dist-info/sboms/native.cdx.json")) + assert (document["metadata"].get("supplier"), capsys.readouterr().out) == ( + {"name": "Org", "url": []}, + f"wrote native-1.0.dist-info/sboms/native.cdx.json into {wheel.name}\n", + ) + + +def test_cli_embed_without_settings(make_wheel: MakeWheel, tmp_path: Path) -> None: + wheel: Final[Path] = make_wheel("") + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'native'\n", encoding="utf-8") + main(["embed", str(wheel), "--root", str(tmp_path)]) + with zipfile.ZipFile(wheel) as archive: + document: Final[Document] = json.loads(archive.read("native-1.0.dist-info/sboms/native.cdx.json")) + assert "supplier" not in document["metadata"] + + +def test_cli_spdx(build_sbom: BuildSbom, tmp_path: Path) -> None: + document: Final[Document] = build_sbom() + (source := tmp_path / "example.cdx.json").write_text(json.dumps(document), encoding="utf-8") + main(["spdx", str(source), str(target := tmp_path / "example.spdx.json")]) + assert target.read_bytes() == f"{json.dumps(to_spdx(document), indent=2)}\n".encode() + + +@pytest.mark.parametrize( + "target", [pytest.param("native-1.0-py3-none-any.whl", id="wheel"), pytest.param(".", id="dir")] +) +def test_cli_validate(make_wheel: MakeWheel, tmp_path: Path, capsys: pytest.CaptureFixture[str], target: str) -> None: + wheel: Final[Path] = make_wheel("License-Expression: MIT\n") + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'native'\n", encoding="utf-8") + main(["embed", str(wheel), "--root", str(tmp_path)]) + main(["validate", str(wheel.parent / target), "--allow-license", "MIT"]) + assert capsys.readouterr().out.splitlines()[-1] == "SBOM in native-1.0-py3-none-any.whl is valid" + + +def test_cli_validate_problems(make_wheel: MakeWheel, capsys: pytest.CaptureFixture[str]) -> None: + wheel: Final[Path] = make_wheel("") + with pytest.raises(SystemExit) as error: + main(["validate", str(wheel)]) + assert (error.value.code, capsys.readouterr().err) == ( + 1, + "SBOM invalid: native-1.0-py3-none-any.whl: no .dist-info/sboms/*.cdx.json found in the wheel\n", + ) + + +def test_cli_validate_no_wheel(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as error: + main(["validate", str(tmp_path)]) + assert (error.value.code, capsys.readouterr().err) == (1, f"no wheel found in {tmp_path}\n") diff --git a/tests/test_embed.py b/tests/test_embed.py new file mode 100644 index 0000000..9320fe4 --- /dev/null +++ b/tests/test_embed.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +import shutil +import sys +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from conftest import digest +from installer import install +from installer.destinations import SchemeDictionaryDestination +from installer.sources import WheelFile + +from hatch_pep770 import Project, embed, validate_wheel + +if TYPE_CHECKING: + from conftest import MakeWheel + + from hatch_pep770 import Document + +_METADATA: Final[str] = "License-Expression: MIT\nProject-URL: Source, https://github.com/example/native\n" + + +def test_embed_writes_sbom(make_wheel: MakeWheel, plain_project: Project) -> None: + wheel: Final[Path] = make_wheel(_METADATA) + name: Final[str] = embed(wheel, plain_project) + with zipfile.ZipFile(wheel) as archive: + document: Final[Document] = json.loads(archive.read(name)) + assert (name, document["metadata"]["component"]["purl"]) == ( + "native-1.0.dist-info/sboms/native.cdx.json", + "pkg:pypi/native@1.0", + ) + + +def test_embed_record(make_wheel: MakeWheel, plain_project: Project) -> None: + wheel: Final[Path] = make_wheel(_METADATA) + name: Final[str] = embed(wheel, plain_project) + with zipfile.ZipFile(wheel) as archive: + sbom: Final[bytes] = archive.read(name) + rows: Final[list[str]] = archive.read("native-1.0.dist-info/RECORD").decode("utf-8").splitlines() + last: Final[str] = archive.namelist()[-1] + assert (rows[-2:], last) == ( + [f"{name},{digest(sbom)},{len(sbom)}", "native-1.0.dist-info/RECORD,,"], + "native-1.0.dist-info/RECORD", + ) + + +def test_embed_twice_same_bytes(make_wheel: MakeWheel, plain_project: Project) -> None: + wheel: Final[Path] = make_wheel(_METADATA) + embed(wheel, plain_project) + first: Final[bytes] = wheel.read_bytes() + embed(wheel, plain_project) + assert wheel.read_bytes() == first + + +def test_embed_keeps_entries(make_wheel: MakeWheel, plain_project: Project) -> None: + wheel: Final[Path] = make_wheel(_METADATA) + with zipfile.ZipFile(wheel) as archive: + before: Final[list[tuple[str, tuple[int, ...]]]] = [ + (info.filename, info.date_time) for info in archive.infolist() + ] + embed(wheel, plain_project) + with zipfile.ZipFile(wheel) as archive: + after: Final[list[tuple[str, tuple[int, ...]]]] = [ + (info.filename, info.date_time) for info in archive.infolist() + ] + assert after == [*before[:-1], ("native-1.0.dist-info/sboms/native.cdx.json", (2020, 2, 2, 0, 0, 0)), before[-1]] + + +def test_embed_crlf_record(make_wheel: MakeWheel, plain_project: Project) -> None: + wheel: Final[Path] = make_wheel(_METADATA) + with zipfile.ZipFile(wheel) as archive: + entries: Final[dict[str, bytes]] = {name: archive.read(name) for name in archive.namelist()} + entries["native-1.0.dist-info/RECORD"] = entries["native-1.0.dist-info/RECORD"].replace(b"\n", b"\r\n") + with zipfile.ZipFile(wheel, "w") as archive: + for name, content in entries.items(): + archive.writestr(name, content) + name: Final[str] = embed(wheel, plain_project) + with zipfile.ZipFile(wheel) as archive: + record: Final[bytes] = archive.read("native-1.0.dist-info/RECORD") + assert (record.count(b"\r\n"), record.count(b"\n"), f"{name},".encode() in record) == (5, 5, True) + + +def test_embed_valid(make_wheel: MakeWheel, plain_project: Project) -> None: + wheel: Final[Path] = make_wheel(_METADATA) + embed(wheel, plain_project) + assert validate_wheel(wheel, frozenset({"MIT"})) == [] + + +def test_embed_survives_install(make_wheel: MakeWheel, plain_project: Project, tmp_path: Path) -> None: + wheel: Final[Path] = make_wheel(_METADATA) + name: Final[str] = embed(wheel, plain_project) + scheme: Final[dict[str, str]] = { + key: str(tmp_path / "site" / key) for key in ("purelib", "platlib", "headers", "scripts", "data") + } + with WheelFile.open(wheel) as source: + source.validate_record() + install(source, SchemeDictionaryDestination(scheme, sys.executable, "posix"), {}) + assert (Path(scheme["platlib"]) / name).is_file() + + +def test_embed_uses_project_settings(make_wheel: MakeWheel, tmp_path: Path) -> None: + wheel: Final[Path] = make_wheel(_METADATA) + project: Final[Project] = Project.from_table( + {"supplier": {"name": "Org"}}, root=tmp_path, where="[tool.hatch-pep770]" + ) + name: Final[str] = embed(wheel, project) + with zipfile.ZipFile(wheel) as archive: + document: Final[Document] = json.loads(archive.read(name)) + assert document["metadata"].get("supplier") == {"name": "Org", "url": []} + + +def test_embed_bundled_wheel(make_wheel: MakeWheel, tmp_path: Path) -> None: + wheel: Final[Path] = make_wheel(_METADATA) + seed: Final[Path] = Path(__file__).parent / "seeder" / "src" / "seeder" / "embed" / "pip-1.0-py3-none-any.whl" + shutil.copyfile(seed, tmp_path / seed.name) + project: Final[Project] = Project.from_table( + {"bundled-wheels": ["*.whl"]}, root=tmp_path, where="[tool.hatch-pep770]" + ) + name: Final[str] = embed(wheel, project) + with zipfile.ZipFile(wheel) as archive: + document: Final[Document] = json.loads(archive.read(name)) + assert document["components"][0].get("properties") == [ + {"name": "native:bundled-wheel", "value": "pip-1.0-py3-none-any.whl"} + ] diff --git a/tests/test_hook.py b/tests/test_hook.py new file mode 100644 index 0000000..6adbbf4 --- /dev/null +++ b/tests/test_hook.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import hashlib +import json +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING, Final + +import pytest +from conftest import BUNDLED, HOOK, build_hook_sbom +from hatchling.builders.sdist import SdistBuilder +from hatchling.builders.wheel import WheelBuilder + +from hatch_pep770 import validate_wheel + +if TYPE_CHECKING: + from conftest import BundleWheel, WriteProject + + from hatch_pep770 import Document + + +def test_wheel_embeds_sbom(write_project: WriteProject, tmp_path: Path) -> None: + with zipfile.ZipFile(_build_wheel(write_project(), tmp_path / "dist")) as archive: + sbom: Final[bytes] = archive.read("example-1.0.dist-info/sboms/example.cdx.json") + record: Final[str] = archive.read("example-1.0.dist-info/RECORD").decode("utf-8") + assert ("example-1.0.dist-info/sboms/example.cdx.json,sha256=" in record, json.loads(sbom)["bomFormat"]) == ( + True, + "CycloneDX", + ) + + +def _build_wheel(root: Path, out: Path) -> Path: + (artifact,) = WheelBuilder(str(root)).build(directory=str(out), versions=["standard"]) + return Path(artifact) + + +def test_wheel_sbom_valid(write_project: WriteProject, bundle_wheel: BundleWheel, tmp_path: Path) -> None: + bundle_wheel({"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}, "License-Expression: MIT\n") + wheel: Final[Path] = _build_wheel(write_project(hook=f"{HOOK}{BUNDLED}"), tmp_path / "dist") + assert validate_wheel(wheel, frozenset({"MIT"})) == [] + + +def test_wheel_reproducible(write_project: WriteProject, tmp_path: Path) -> None: + root: Final[Path] = write_project() + assert _build_wheel(root, tmp_path / "first").read_bytes() == _build_wheel(root, tmp_path / "second").read_bytes() + + +def test_wheel_sbom_directory_removed(write_project: WriteProject) -> None: + builder: Final[WheelBuilder] = WheelBuilder(str(write_project())) + build_data = builder.get_default_build_data() + (hook,) = builder.get_build_hooks(str(builder.root)).values() + hook.initialize("standard", build_data) + hook.finalize("standard", build_data, "") + assert Path(build_data["sbom_files"][0]).parent.exists() is False + + +def test_sdist_skipped(write_project: WriteProject, tmp_path: Path) -> None: + # the hook runs for every target, and only a wheel build has somewhere to put an SBOM + (artifact,) = SdistBuilder(str(write_project())).build(directory=str(tmp_path / "dist")) + assert Path(artifact).name == "example-1.0.tar.gz" + + +def test_attestation_tag_from_another_hook(write_project: WriteProject) -> None: + builder: Final[WheelBuilder] = WheelBuilder(str(write_project())) + assert _attestations(builder, "tag", "py3-none-linux_x86_64") == [ + "https://pypi.org/integrity/Example/1.0/example-1.0-py3-none-linux_x86_64.whl/provenance" + ] + + +def _attestations(builder: WheelBuilder, key: str, value: str | bool) -> list[str]: + build_data = builder.get_default_build_data() + build_data[key] = value + (hook,) = builder.get_build_hooks(str(builder.root)).values() + hook.initialize("standard", build_data) + document: Final[Document] = json.loads(Path(build_data["sbom_files"][0]).read_text(encoding="utf-8")) + references = document["metadata"]["component"].get("externalReferences", []) + return [reference["url"] for reference in references if reference["type"] == "attestation"] + + +def test_attestation_tag_inferred(write_project: WriteProject) -> None: + builder: Final[WheelBuilder] = WheelBuilder(str(write_project())) + assert _attestations(builder, "infer_tag", True) == [ + f"https://pypi.org/integrity/Example/1.0/example-1.0-{builder.get_best_matching_tag()}.whl/provenance" + ] + + +@pytest.mark.parametrize( + ("urls", "references"), + [ + pytest.param( + 'urls.Source = "https://github.com/example/example"', + {"externalReferences": [{"type": "vcs", "url": "https://github.com/example/example/blob/main/build.py"}]}, + id="repository", + ), + pytest.param("", {}, id="no-repository"), + ], +) +def test_custom_build_script_recorded( + root: Path, write_project: WriteProject, urls: str, references: dict[str, list[dict[str, str]]] +) -> None: + (script := root / "build.py").write_text( + "from __future__ import annotations\n\nfrom hatch_pep770 import Pep770BuildHook\n\n\n" + "def get_build_hook() -> type[Pep770BuildHook]:\n return Pep770BuildHook\n", + encoding="utf-8", + ) + project = f'[project]\nname = "example"\nversion = "1.0"\n{urls}\n' + document = build_hook_sbom( + write_project(hook='[tool.hatch.build.hooks.custom]\npath = "build.py"\n', project=project) + ) + assert document["metadata"]["tools"]["components"][0] == { + "type": "application", + "bom-ref": "tool:build.py", + "name": "build.py", + "version": "1.0", + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(script.read_bytes()).hexdigest()}], + **references, + } + + +def test_seeder_subclass_adds_properties(seeder: Path, tmp_path: Path) -> None: + with zipfile.ZipFile(_build_wheel(seeder, tmp_path / "dist")) as archive: + document: Final[Document] = json.loads(archive.read("seeder-1.0.dist-info/sboms/seeder.cdx.json")) + assert document["components"][0].get("properties", [])[:3] == [ + {"name": "seeder:bundled-wheel", "value": "src/seeder/embed/pip-1.0-py3-none-any.whl"}, + {"name": "seeder:seeded-for-python", "value": "3.14"}, + {"name": "seeder:seeded-for-python", "value": "3.15"}, + ] + + +def test_seeder_supplier_and_vendored(seeder: Path, tmp_path: Path) -> None: + with zipfile.ZipFile(_build_wheel(seeder, tmp_path / "dist")) as archive: + document: Final[Document] = json.loads(archive.read("seeder-1.0.dist-info/sboms/seeder.cdx.json")) + pip = document["components"][0] + assert ( + pip.get("supplier"), + [child["name"] for child in pip.get("components", []) if child["type"] == "library"], + ) == ( + {"name": "Example Org", "url": ["https://example.org"]}, + ["urllib3"], + ) + + +def test_seeder_wheel_valid(seeder: Path, tmp_path: Path) -> None: + assert validate_wheel(_build_wheel(seeder, tmp_path / "dist"), frozenset({"MIT"})) == [] diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..8b1a328 --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pytest + + +def test_main(capfd: pytest.CaptureFixture[str]) -> None: + subprocess.check_call([sys.executable, "-m", "hatch_pep770", "-h"]) + out, _err = capfd.readouterr() + assert out.startswith("usage: hatch-pep770 ") + + +def test_script(capfd: pytest.CaptureFixture[str]) -> None: + subprocess.check_call([Path(sys.executable).parent / "hatch-pep770", "-h"]) + out, _err = capfd.readouterr() + assert out.startswith("usage: hatch-pep770 ") diff --git a/tests/test_plugin.py b/tests/test_plugin.py deleted file mode 100644 index 52f91d4..0000000 --- a/tests/test_plugin.py +++ /dev/null @@ -1,829 +0,0 @@ -from __future__ import annotations - -import csv -import hashlib -import json -import os -import subprocess -import sys -import zipfile -from io import StringIO -from pathlib import Path -from typing import TYPE_CHECKING, Any, Final - -import pytest -from hatchling.builders.sdist import SdistBuilder -from hatchling.builders.wheel import WheelBuilder - -from hatch_pep770.validate import wheel_problems - -if TYPE_CHECKING: - from collections.abc import Callable - - from pytest_mock import MockerFixture - - from hatch_pep770.cyclonedx import Component, Document, Workflow - from tests.conftest import BundleWheel, InstallTool - -_BUNDLED: Final[str] = "src/example/wheels/*.whl" -_URLS: Final[dict[str, str]] = {"Source": "https://github.com/example/example"} - - -@pytest.mark.skipif(sys.version_info < (3, 10), reason="hatchling 1.28 needs Python 3.10") -def test_wheel_embeds_sbom( - build_wheel: Callable[[dict[str, Any], str], Path], config: dict[str, Any] -) -> None: # pragma: >=3.10 cover - with zipfile.ZipFile(build_wheel(config, "dist")) as archive: - names: Final[list[str]] = archive.namelist() - record: Final[str] = archive.read("example-1.0.dist-info/RECORD").decode("utf-8") - assert ("example-1.0.dist-info/sboms/example.cdx.json" in names, "sboms/example.cdx.json," in record) == ( - True, - True, - ) - - -@pytest.mark.skipif(sys.version_info < (3, 10), reason="hatchling 1.28 needs Python 3.10") -def test_wheel_sbom_valid( - build_wheel: Callable[[dict[str, Any], str], Path], - bundle_wheel: BundleWheel, - config: dict[str, Any], -) -> None: # pragma: >=3.10 cover - bundle_wheel({"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}, "License-Expression: MIT\n") - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - assert wheel_problems(build_wheel(config, "dist"), frozenset({"MIT"})) == [] - - -@pytest.mark.skipif(sys.version_info < (3, 10), reason="hatchling 1.28 needs Python 3.10") -def test_wheel_reproducible( - build_wheel: Callable[[dict[str, Any], str], Path], config: dict[str, Any] -) -> None: # pragma: >=3.10 cover - assert build_wheel(config, "first").read_bytes() == build_wheel(config, "second").read_bytes() - - -@pytest.mark.skipif(sys.version_info >= (3, 10), reason="Python 3.10 and later get hatchling 1.28") -def test_wheel_old_hatchling( - build_wheel: Callable[[dict[str, Any], str], Path], config: dict[str, Any], capsys: pytest.CaptureFixture[str] -) -> None: # pragma: <3.10 cover - with zipfile.ZipFile(build_wheel(config, "dist")) as archive: - sboms: Final[list[str]] = [name for name in archive.namelist() if "/sboms/" in name] - assert (sboms, "cannot embed SBOMs" in capsys.readouterr().err) == ([], True) - - -def test_sdist_target_skipped(root: Path, config: dict[str, Any], capsys: pytest.CaptureFixture[str]) -> None: - (artifact,) = SdistBuilder(str(root), config=config).build(directory=str(root / "dist")) - assert (Path(artifact).name, capsys.readouterr().err) == ("example-1.0.tar.gz", "") - - -def test_root_component(sbom_document: Document) -> None: - attestation: Final[str] = "https://pypi.org/integrity/example/1.0/example-1.0-py3-none-any.whl/provenance" - assert sbom_document["metadata"]["component"] == { - "type": "application", - "bom-ref": "pkg:pypi/example@1.0", - "authors": [{"name": "Maintainer", "email": "maintainer@example.com"}], - "name": "example", - "version": "1.0", - "description": "an example", - "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], - "copyright": "Copyright (c) 2026 Example", - "purl": "pkg:pypi/example@1.0", - "externalReferences": [ - {"type": "release-notes", "url": "https://example.com/changelog", "comment": "Project-URL: Changelog"}, - {"type": "vcs", "url": "https://github.com/example/example", "comment": "Project-URL: Source"}, - {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, - {"type": "attestation", "url": attestation}, - {"type": "security-contact", "url": "https://github.com/example/example/security/policy"}, - {"type": "advisories", "url": "https://github.com/example/example/security/advisories"}, - {"type": "license", "url": "https://github.com/example/example/blob/main/LICENSE"}, - ], - "properties": [ - {"name": "example:requires-python", "value": ">=3.9"}, - {"name": "python:classifier", "value": "Programming Language :: Python :: 3"}, - {"name": "python:keyword", "value": "sample"}, - ], - } - - -def test_root_component_minimal_metadata( - root: Path, build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any] -) -> None: - (root / "LICENSE").write_text("MIT terms\n", encoding="utf-8") - config["project"] = { - "name": "example", - "version": "1.0", - "license-files": ["LICENSE"], - "authors": [{"name": "Author"}], - } - assert json.loads(build_sbom(config))["metadata"]["component"] == { - "type": "application", - "bom-ref": "pkg:pypi/example@1.0", - "authors": [{"name": "Author"}], - "name": "example", - "version": "1.0", - "purl": "pkg:pypi/example@1.0", - "externalReferences": [ - {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, - { - "type": "attestation", - "url": "https://pypi.org/integrity/example/1.0/example-1.0-py2.py3-none-any.whl/provenance", - }, - ], - "properties": [], - } - - -def test_document_metadata(sbom_document: Document) -> None: - assert {key: value for key, value in sbom_document["metadata"].items() if key not in {"tools", "component"}} == { - "timestamp": "2020-02-02T00:00:00+00:00", - "lifecycles": [{"phase": "build"}], - "authors": [{"name": "Maintainer", "email": "maintainer@example.com"}], - "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], - "properties": [ - { - "name": "example:sbom:scope", - "value": "Components are the direct runtime dependencies declared in the wheel's core metadata. They" - " carry no version because they are resolved at install time, not at build time.", - } - ], - } - - -def test_supplier(build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any]) -> None: - supplier: Final[dict[str, Any]] = {"name": "Example Org", "url": ["https://example.org"]} - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"supplier": supplier} - document: Final[dict[str, Any]] = json.loads(build_sbom(config)) - assert [ - document["metadata"]["manufacturer"], - document["metadata"]["supplier"], - document["metadata"]["component"]["supplier"], - ] == [supplier] * 3 - - -def test_declared_dependencies(sbom_document: Document) -> None: - assert (sbom_document["components"], sbom_document["dependencies"][0]) == ( - [ - { - "type": "library", - "bom-ref": "requires-dist:packaging>=24", - "name": "packaging", - "purl": "pkg:pypi/packaging", - "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/packaging/"}], - "properties": [{"name": "example:requires-dist", "value": "packaging>=24"}], - }, - { - "type": "library", - "bom-ref": "requires-dist:tomli>=2; python_version < '3.11'", - "name": "tomli", - "purl": "pkg:pypi/tomli", - "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/tomli/"}], - "properties": [ - {"name": "example:requires-dist", "value": "tomli>=2; python_version < '3.11'"}, - {"name": "example:environment-marker", "value": 'python_version < "3.11"'}, - ], - }, - ], - { - "ref": "pkg:pypi/example@1.0", - "dependsOn": ["requires-dist:packaging>=24", "requires-dist:tomli>=2; python_version < '3.11'"], - }, - ) - - -def test_composition_incomplete(sbom_document: Document) -> None: - assert sbom_document["compositions"] == [{"aggregate": "incomplete", "dependencies": ["pkg:pypi/example@1.0"]}] - - -def test_build_workflow(sbom_document: Document) -> None: - workflow: Final[Workflow] = sbom_document["formulation"][0]["workflows"][0] - assert {**workflow, "resourceReferences": len(workflow["resourceReferences"])} == { - "bom-ref": "workflow:wheel-build", - "uid": "wheel-build", - "name": "build the wheel and this SBOM", - "taskTypes": ["build"], - "resourceReferences": len(sbom_document["metadata"]["tools"]["components"]), - "outputs": [{"type": "artifact", "resource": {"ref": "pkg:pypi/example@1.0"}}], - "inputs": [{"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": "1580601600"}]}], - } - - -def test_build_tools_name_interpreter_and_hook(sbom_document: Document) -> None: - tools: Final[list[Component]] = sbom_document["metadata"]["tools"]["components"] - assert (tools[0]["type"], any(tool["name"] == "hatch-pep770" for tool in tools)) == ("platform", True) - - -def test_bundled_wheel( - build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] -) -> None: - wheel: Final[Path] = bundle_wheel({}, "License-Expression: MIT\n") - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - sha256: Final[str] = hashlib.sha256(wheel.read_bytes()).hexdigest() - assert json.loads(build_sbom(config))["components"][0] == { - "type": "library", - "bom-ref": "pkg:pypi/pip@1.0", - "name": "pip", - "version": "1.0", - "purl": "pkg:pypi/pip@1.0", - "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], - "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/pip/1.0/"}], - "properties": [{"name": "example:bundled-wheel", "value": "src/example/wheels/pip-1.0-py3-none-any.whl"}], - "hashes": [{"alg": "SHA-256", "content": sha256}], - "evidence": { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [ - {"technique": "manifest-analysis", "confidence": 1, "value": "pip-1.0.dist-info/METADATA"} - ], - }, - { - "field": "hash", - "confidence": 1, - "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], - }, - ] - }, - "components": [ - { - "type": "file", - "bom-ref": "pkg:pypi/pip@1.0#pip/example.py", - "name": "pip/example.py", - "hashes": [ - {"alg": "SHA-256", "content": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} - ], - "properties": [{"name": "size", "value": "3"}], - } - ], - } - - -def test_bundled_wheel_scope( - build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] -) -> None: - bundle_wheel({}) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - assert json.loads(build_sbom(config))["metadata"]["properties"] == [ - { - "name": "example:sbom:scope", - "value": "Components are the wheels bundled as data files under src/example/wheels and the direct runtime" - " dependencies declared in the wheel's core metadata. The latter carry no version because they are" - " resolved at install time, not at build time.", - } - ] - - -def test_bundled_wheels_sorted_and_first( - build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] -) -> None: - bundle_wheel({}, name="setuptools") - bundle_wheel({}, name="pip") - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED, "src/example/wheels/pip-*"]} - document: Final[dict[str, Any]] = json.loads(build_sbom(config)) - assert document["dependencies"][0]["dependsOn"] == [ - "pkg:pypi/pip@1.0", - "pkg:pypi/setuptools@1.0", - "requires-dist:packaging>=24", - "requires-dist:tomli>=2; python_version < '3.11'", - ] - - -@pytest.mark.parametrize( - ("source", "urls", "expected"), - [ - pytest.param("https://github.com/example/pip", _URLS, True, id="same-account"), - pytest.param("https://github.com/other/pip", _URLS, False, id="other-account"), - pytest.param("https://github.com/example/pip", {}, False, id="no-repository"), - ], -) -def test_bundled_wheel_supplier( - build_sbom: Callable[[dict[str, Any]], str], - bundle_wheel: BundleWheel, - config: dict[str, Any], - source: str, - urls: dict[str, str], - expected: bool, -) -> None: - bundle_wheel({}, f"Project-URL: Source, {source}\n") - config["project"]["urls"] = urls - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = { - "bundled-wheels": [_BUNDLED], - "supplier": {"name": "Example Org"}, - } - assert ("supplier" in json.loads(build_sbom(config))["components"][0]) is expected - - -@pytest.mark.parametrize( - "filename", - [ - pytest.param("pip/example.py", id="plain"), - pytest.param("pip/a,b.py", id="comma"), - pytest.param('pip/a"b.py', id="quote"), - pytest.param("pip/a\nb.py", id="newline"), - ], -) -def test_bundled_wheel_record_paths( - root: Path, build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], filename: str -) -> None: - record: Final[StringIO] = StringIO(newline="") - csv.writer(record).writerows([ - (filename, "sha256=ungWv48Bz-pBQUDeXa4iI7ADYaOWF3qctBD_YfIAFa0", "3"), - ("pip-1.0.dist-info/RECORD", "", ""), - ]) - (wheels := root / "src" / "example" / "wheels").mkdir(parents=True) - with zipfile.ZipFile(wheels / "pip-1.0-py3-none-any.whl", "w") as archive: - archive.writestr("pip-1.0.dist-info/METADATA", "Metadata-Version: 2.4\nName: pip\nVersion: 1.0\n") - archive.writestr("pip-1.0.dist-info/RECORD", record.getvalue()) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - assert json.loads(build_sbom(config))["components"][0]["components"] == [ - { - "type": "file", - "bom-ref": f"pkg:pypi/pip@1.0#{filename}", - "name": filename, - "hashes": [ - {"alg": "SHA-256", "content": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} - ], - "properties": [{"name": "size", "value": "3"}], - }, - ] - - -@pytest.mark.parametrize( - ("source", "content"), - [ - pytest.param( - "pip/_vendor/vendor.txt", "# bundled\n\n urllib3==1.26.4 # patched upstream\n", id="pip-manifest" - ), - pytest.param( - "pip/_vendor/urllib3-1.26.4.dist-info/METADATA", "Name: urllib3\nVersion: 1.26.4\n", id="nested-metadata" - ), - ], -) -def test_vendored_identity( - build_sbom: Callable[[dict[str, Any]], str], - bundle_wheel: BundleWheel, - config: dict[str, Any], - source: str, - content: str, -) -> None: - bundle_wheel({source: content}) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - document: Final[dict[str, Any]] = json.loads(build_sbom(config)) - assert [child for child in document["components"][0]["components"] if child["type"] == "library"] == [ - { - "type": "library", - "bom-ref": "pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4", - "name": "urllib3", - "version": "1.26.4", - "purl": "pkg:pypi/urllib3@1.26.4", - "externalReferences": [], - "properties": [{"name": "example:vendored-manifest", "value": source}], - "evidence": { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], - } - ] - }, - } - ] - - -def test_vendored_relationship( - build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] -) -> None: - bundle_wheel({"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - assert json.loads(build_sbom(config))["dependencies"][1] == { - "ref": "pkg:pypi/pip@1.0", - "dependsOn": ["pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4"], - } - - -def test_vendored_metadata_precedence( - build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] -) -> None: - bundle_wheel({ - "pip/_vendor/vendor.txt": "urllib3==1.26.4\n", - "pip/_vendor/urllib3-1.26.4.dist-info/METADATA": "Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n", - }) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - document: Final[dict[str, Any]] = json.loads(build_sbom(config)) - assert [child["licenses"] for child in document["components"][0]["components"] if child["type"] == "library"] == [ - [{"expression": "MIT", "acknowledgement": "declared"}], - ] - - -@pytest.mark.parametrize( - "requirement", - [ - pytest.param("urllib3", id="missing"), - pytest.param("urllib3>=1", id="range"), - pytest.param("urllib3==1.*", id="wildcard"), - pytest.param("urllib3 @ https://example.com/package.whl", id="url"), - ], -) -def test_vendored_unpinned_rejected( - build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any], requirement: str -) -> None: - bundle_wheel({"pip/_vendor/vendor.txt": requirement}) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - with pytest.raises(ValueError, match="Unpinned vendored dependency"): - build_sbom(config) - - -@pytest.mark.parametrize( - ("metadata", "expected"), - [ - pytest.param( - "Classifier: License :: OSI Approved :: MIT License\n", - [ - {"license": {"name": "MIT License", "acknowledgement": "declared"}}, - {"license": {"id": "MIT", "acknowledgement": "concluded"}}, - ], - id="classifier", - ), - pytest.param( - "Classifier: License :: OSI Approved :: Python Software Foundation License\nLicense: PSF-2.0\n", - [ - {"license": {"name": "Python Software Foundation License", "acknowledgement": "declared"}}, - {"license": {"name": "PSF-2.0", "acknowledgement": "declared"}}, - {"license": {"id": "PSF-2.0", "acknowledgement": "concluded"}}, - ], - id="classifier-and-id-agree", - ), - pytest.param( - "License: mit\n", - [ - {"license": {"name": "mit", "acknowledgement": "declared"}}, - {"license": {"id": "MIT", "acknowledgement": "concluded"}}, - ], - id="id-canonical-case", - ), - pytest.param( - "Classifier: License :: OSI Approved :: MIT License\n" - "Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)\n", - [ - {"license": {"name": "MIT License", "acknowledgement": "declared"}}, - {"license": {"name": "GNU Lesser General Public License v3 (LGPLv3)", "acknowledgement": "declared"}}, - ], - id="one-unresolved", - ), - pytest.param( - "License: MIT OR Apache-2.0\n", - [{"license": {"name": "MIT OR Apache-2.0", "acknowledgement": "declared"}}], - id="expression-in-license-field", - ), - pytest.param( - "License: LicenseRef-Proprietary\n", - [{"license": {"name": "LicenseRef-Proprietary", "acknowledgement": "declared"}}], - id="license-ref", - ), - ], -) -def test_license_resolution( - build_sbom: Callable[[dict[str, Any]], str], - bundle_wheel: BundleWheel, - config: dict[str, Any], - metadata: str, - expected: list[dict[str, Any]], -) -> None: - bundle_wheel({}, metadata) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - assert json.loads(build_sbom(config))["components"][0]["licenses"] == expected - - -def test_bundled_wheel_metadata_fields( - build_sbom: Callable[[dict[str, Any]], str], bundle_wheel: BundleWheel, config: dict[str, Any] -) -> None: - bundle_wheel( - {}, - "Summary: installer\nHome-page: https://pip.pypa.io\nAuthor: Someone\nAuthor-email: dev@example.com\n" - "Maintainer-email: Team <>\nProject-URL: Bug Tracker, https://example.com/issues\n", - ) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - component: Final[dict[str, Any]] = json.loads(build_sbom(config))["components"][0] - assert {key: component[key] for key in ("description", "authors", "externalReferences")} == { - "description": "installer", - "authors": [{"name": "Someone"}, {"email": "dev@example.com"}, {"name": "Team"}], - "externalReferences": [ - {"type": "website", "url": "https://pip.pypa.io"}, - {"type": "issue-tracker", "url": "https://example.com/issues", "comment": "Project-URL: Bug Tracker"}, - {"type": "distribution", "url": "https://pypi.org/project/pip/1.0/"}, - ], - } - - -@pytest.mark.parametrize( - ("environment", "expected"), - [ - pytest.param({}, ("2020-02-02T00:00:00+00:00", "1580601600"), id="hatchling-default"), - pytest.param( - {"SOURCE_DATE_EPOCH": "1700000000"}, ("2023-11-14T22:13:20+00:00", "1700000000"), id="source-date-epoch" - ), - ], -) -def test_timestamp( - build_sbom: Callable[[dict[str, Any]], str], - config: dict[str, Any], - mocker: MockerFixture, - environment: dict[str, str], - expected: tuple[str, str], -) -> None: - mocker.patch.dict(os.environ, environment) - document: Final[dict[str, Any]] = json.loads(build_sbom(config)) - workflow: Final[dict[str, Any]] = document["formulation"][0]["workflows"][0] - assert (document["metadata"]["timestamp"], workflow["inputs"][0]["environmentVars"][0]["value"]) == expected - - -def test_build_host_independent( - build_sbom: Callable[[dict[str, Any]], str], - config: dict[str, Any], - install_tool: InstallTool, - mocker: MockerFixture, - tmp_path: Path, -) -> None: - sboms: Final[list[str]] = [] - for host, extension, installer, sys_version in ( - ( - { - "system": "Linux", - "release": "6.17.0-1022-azure", - "version": "#22~24.04.1-Ubuntu SMP", - "machine": "x86_64", - "platform": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", - "python_compiler": "GCC 13.3.0", - "python_build": ("main", "Aug 5 2026 10:00:00"), - }, - "cpython-314-x86_64-linux-gnu.so", - b"uv\n", - "3.14.7 (main, Aug 5 2026, 10:00:00) [GCC 13.3.0]", - ), - ( - { - "system": "Linux", - "release": "7.0.12-linuxkit", - "version": "#1 SMP PREEMPT", - "machine": "aarch64", - "platform": "Linux-7.0.12-linuxkit-aarch64-with-glibc2.36", - "python_compiler": "Clang 20.1.4", - "python_build": ("main", "Sep 1 2026 08:00:00"), - }, - "cpython-314-aarch64-linux-gnu.so", - b"pip\n", - "3.14.7 (main, Sep 1 2026, 08:00:00) [Clang 20.1.4]", - ), - ): - for name, value in host.items(): - mocker.patch(f"platform.{name}", autospec=True, return_value=value) - mocker.patch.object(sys, "version", sys_version) - install_tool(site := tmp_path / extension, "native", False, {f"native/_speedups.{extension}": b"\0"}) - install_tool(site, "pure", True, {"pure-1.0.dist-info/INSTALLER": installer, "pure/__init__.py": b""}) - mocker.patch.object(sys, "path", [str(site), *sys.path]) - sboms.append(build_sbom(config)) - mocker.stopall() - assert sboms[0] == sboms[1] - - -def test_build_tools_pure_files( - build_sbom: Callable[[dict[str, Any]], str], - config: dict[str, Any], - install_tool: InstallTool, - mocker: MockerFixture, - tmp_path: Path, -) -> None: - install_tool( - site := tmp_path / "site", - "pure", - True, - {"pure-1.0.dist-info/INSTALLER": b"uv\n", "pure/__init__.py": b"", "../../bin/pure": b"#!python\n"}, - ) - install_tool(site, "native", False, {"native/_speedups.so": b"\0"}) - mocker.patch.object(sys, "path", [str(site), *sys.path]) - tools: Final[dict[str, dict[str, Any]]] = { - tool["name"]: tool for tool in json.loads(build_sbom(config))["metadata"]["tools"]["components"] - } - assert ([file["name"] for file in tools["pure"]["components"]], "components" in tools["native"]) == ( - ["pure/__init__.py"], - False, - ) - - -def test_ci_rerun( - build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("GITHUB_SHA", "a" * 40) - monkeypatch.setenv("GITHUB_RUN_ID", "100") - first: Final[str] = build_sbom(config) - monkeypatch.setenv("GITHUB_SHA", "b" * 40) - monkeypatch.setenv("GITHUB_RUN_ID", "200") - assert build_sbom(config) == first - - -def test_source_revision( - root: Path, build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], monkeypatch: pytest.MonkeyPatch -) -> None: - for name in ("GIT_AUTHOR_NAME", "GIT_COMMITTER_NAME"): - monkeypatch.setenv(name, "Example") - for name in ("GIT_AUTHOR_EMAIL", "GIT_COMMITTER_EMAIL"): - monkeypatch.setenv(name, "example@example.com") - for command in (["init", "-q"], ["add", "."], ["commit", "-q", "--no-gpg-sign", "-m", "init"]): - subprocess.run(["git", *command], cwd=root, check=True) - commit: Final[str] = subprocess.run( - ["git", "rev-parse", "HEAD"], cwd=root, check=True, capture_output=True, text=True - ).stdout.strip() - component: Final[dict[str, Any]] = json.loads(build_sbom(config))["metadata"]["component"] - assert (component["externalReferences"][-1], component["properties"][-1]) == ( - {"type": "vcs", "url": f"https://github.com/example/example/tree/{commit}", "comment": "exact source revision"}, - {"name": "example:vcs-commit", "value": commit}, - ) - - -def test_source_revision_without_repository( - root: Path, build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], mocker: MockerFixture -) -> None: - (root / ".git").mkdir() - mocker.patch( - "subprocess.run", - autospec=True, - return_value=subprocess.CompletedProcess(["git"], 0, stdout="c0ffee\n", stderr=""), - ) - del config["project"]["urls"] - component: Final[dict[str, Any]] = json.loads(build_sbom(config))["metadata"]["component"] - assert ([ref["type"] for ref in component["externalReferences"]], component["properties"][-1]) == ( - ["distribution", "attestation"], - {"name": "example:vcs-commit", "value": "c0ffee"}, - ) - - -def test_source_revision_unreadable( - root: Path, - build_sbom: Callable[[dict[str, Any]], str], - config: dict[str, Any], - mocker: MockerFixture, - capsys: pytest.CaptureFixture[str], -) -> None: - (root / ".git").mkdir() - mocker.patch( - "subprocess.run", - autospec=True, - side_effect=subprocess.CalledProcessError(128, ["git"], stderr="fatal: detected dubious ownership"), - ) - component: Final[dict[str, Any]] = json.loads(build_sbom(config))["metadata"]["component"] - assert ( - [prop["name"] for prop in component["properties"]], - "names no source revision: fatal: detected dubious ownership" in capsys.readouterr().err, - ) == (["example:requires-python", "python:classifier", "python:keyword"], True) - - -@pytest.mark.parametrize( - ("hook_config", "expected"), - [ - pytest.param( - {"repository": "https://gitlab.com/example/example/"}, - [ - {"type": "release-notes", "url": "https://example.com/changelog", "comment": "Project-URL: Changelog"}, - {"type": "vcs", "url": "https://github.com/example/example", "comment": "Project-URL: Source"}, - {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, - {"type": "license", "url": "https://gitlab.com/example/example/blob/main/LICENSE"}, - ], - id="repository-off-github", - ), - pytest.param( - {"default-branch": "trunk"}, - [ - {"type": "release-notes", "url": "https://example.com/changelog", "comment": "Project-URL: Changelog"}, - {"type": "vcs", "url": "https://github.com/example/example", "comment": "Project-URL: Source"}, - {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, - {"type": "security-contact", "url": "https://github.com/example/example/security/policy"}, - {"type": "advisories", "url": "https://github.com/example/example/security/advisories"}, - {"type": "license", "url": "https://github.com/example/example/blob/trunk/LICENSE"}, - ], - id="default-branch", - ), - ], -) -def test_repository_links( - build_sbom: Callable[[dict[str, Any]], str], - config: dict[str, Any], - hook_config: dict[str, Any], - expected: list[dict[str, str]], -) -> None: - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = hook_config - references: Final[list[dict[str, str]]] = json.loads(build_sbom(config))["metadata"]["component"][ - "externalReferences" - ] - assert [reference for reference in references if reference["type"] != "attestation"] == expected - - -def test_serial_number_per_project(build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any]) -> None: - first: Final[str] = json.loads(build_sbom(config))["serialNumber"] - del config["project"]["urls"] - assert json.loads(build_sbom(config))["serialNumber"] != first - - -@pytest.mark.parametrize( - ("build_data", "tag"), - [ - pytest.param({"tag": "py3-none-linux_x86_64"}, "py3-none-linux_x86_64", id="set-by-another-hook"), - pytest.param({"infer_tag": True}, None, id="inferred"), - ], -) -def test_attestation_wheel_tag(root: Path, config: dict[str, Any], build_data: dict[str, Any], tag: str | None) -> None: - builder: Final[WheelBuilder] = WheelBuilder(str(root), config=config) - data: Final[dict[str, Any]] = {**builder.get_default_build_data(), "sbom_files": [], **build_data} - (hook,) = builder.get_build_hooks(str(root / "dist")).values() - hook.initialize("standard", data) - references: Final[list[dict[str, str]]] = json.loads(Path(data["sbom_files"][0]).read_text(encoding="utf-8"))[ - "metadata" - ]["component"]["externalReferences"] - expected: Final[str] = tag or builder.get_best_matching_tag() - assert [ref["url"] for ref in references if ref["type"] == "attestation"] == [ - f"https://pypi.org/integrity/example/1.0/example-1.0-{expected}.whl/provenance" - ] - - -def test_finalize_removes_sbom(root: Path, config: dict[str, Any]) -> None: - builder: Final[WheelBuilder] = WheelBuilder(str(root), config=config) - build_data: Final[dict[str, Any]] = {**builder.get_default_build_data(), "sbom_files": []} - (hook,) = builder.get_build_hooks(str(root / "dist")).values() - hook.initialize("standard", build_data) - hook.finalize("standard", build_data, "") - assert Path(build_data["sbom_files"][0]).parent.exists() is False - - -@pytest.mark.parametrize( - ("hook_config", "message"), - [ - pytest.param( - {"supplier": "Example"}, "Option `supplier` of build hook `pep770` must be a table", id="supplier" - ), - pytest.param({"supplier": {"name": "Example", "email": "x"}}, "Option `supplier`", id="supplier-unknown-key"), - pytest.param({"repository": 1}, "Option `repository` of build hook `pep770` must be a string", id="repository"), - pytest.param( - {"default-branch": 1}, "Option `default-branch` of build hook `pep770` must be a string", id="branch" - ), - pytest.param( - {"bundled-wheels": "*.whl"}, - "Option `bundled-wheels` of build hook `pep770` must be an array of strings", - id="bundled-wheels", - ), - ], -) -def test_invalid_option( - build_sbom: Callable[[dict[str, Any]], str], config: dict[str, Any], hook_config: dict[str, Any], message: str -) -> None: - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = hook_config - with pytest.raises(TypeError, match=message): - build_sbom(config) - - -@pytest.mark.parametrize( - ("urls", "references"), - [ - pytest.param( - _URLS, - { - "externalReferences": [ - {"type": "vcs", "url": "https://github.com/example/example/blob/main/hatch_build.py"} - ] - }, - id="repository", - ), - pytest.param({}, {}, id="no-repository"), - ], -) -def test_custom_hook_subclass( - root: Path, - build_sbom: Callable[[dict[str, Any]], str], - bundle_wheel: BundleWheel, - config: dict[str, Any], - urls: dict[str, str], - references: dict[str, Any], -) -> None: - bundle_wheel({}) - (script := root / "hatch_build.py").write_text( - "from __future__ import annotations\n\nfrom hatch_pep770 import plugin\n\n\n" - "class Hook(plugin.Pep770BuildHook):\n" - " def bundled_wheel_properties(self, wheel):\n" - ' return [{"name": "example:installed-for-python", "value": wheel.name}]\n\n\n' - "def get_build_hook():\n return Hook\n", - encoding="utf-8", - ) - config["project"]["urls"] = urls - config["tool"]["hatch"]["build"]["hooks"] = {"custom": {"bundled-wheels": [_BUNDLED]}} - document: Final[dict[str, Any]] = json.loads(build_sbom(config)) - assert (document["metadata"]["tools"]["components"][0], document["components"][0]["properties"][:2]) == ( - { - "type": "application", - "bom-ref": "tool:hatch_build.py", - "name": "hatch_build.py", - "version": "1.0", - "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(script.read_bytes()).hexdigest()}], - **references, - }, - [ - {"name": "example:bundled-wheel", "value": "src/example/wheels/pip-1.0-py3-none-any.whl"}, - {"name": "example:installed-for-python", "value": "pip-1.0-py3-none-any.whl"}, - ], - ) diff --git a/tests/test_project.py b/tests/test_project.py new file mode 100644 index 0000000..ce2c4ec --- /dev/null +++ b/tests/test_project.py @@ -0,0 +1,546 @@ +from __future__ import annotations + +import csv +import hashlib +import shutil +import subprocess +import sys +import zipfile +from io import StringIO +from typing import TYPE_CHECKING, Final + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + from conftest import BuildSbom, BundleWheel, InstallTool, WriteProject + from pytest_mock import MockerFixture + + from hatch_pep770 import Component, Document + +from conftest import BUNDLED, HOOK, build_hook_sbom + +_SHA256_ABC: Final[str] = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + + +def test_root_component(build_sbom: BuildSbom) -> None: + assert build_sbom()["metadata"]["component"] == { + "type": "application", + "bom-ref": "pkg:pypi/example@1.0", + "authors": [{"name": "Maintainer", "email": "maintainer@example.com"}], + "name": "Example", + "version": "1.0", + "description": "an example", + "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], + "copyright": "Copyright (c) 2026 Example", + "purl": "pkg:pypi/example@1.0", + "externalReferences": [ + {"type": "release-notes", "url": "https://example.com/changelog", "comment": "Project-URL: Changelog"}, + {"type": "vcs", "url": "https://github.com/example/example", "comment": "Project-URL: Source"}, + {"type": "distribution", "url": "https://pypi.org/project/Example/1.0/"}, + { + "type": "attestation", + "url": "https://pypi.org/integrity/Example/1.0/example-1.0-py3-none-any.whl/provenance", + }, + {"type": "security-contact", "url": "https://github.com/example/example/security/policy"}, + {"type": "advisories", "url": "https://github.com/example/example/security/advisories"}, + {"type": "license", "url": "https://github.com/example/example/blob/main/LICENSE"}, + ], + "properties": [ + {"name": "example:requires-python", "value": ">=3.11"}, + {"name": "python:classifier", "value": "Programming Language :: Python :: 3"}, + {"name": "python:keyword", "value": "demo"}, + {"name": "python:keyword", "value": "sample"}, + ], + } + + +def test_root_component_minimal_metadata(root: Path, build_sbom: BuildSbom) -> None: + (root / "LICENSE").write_text("MIT terms\n", encoding="utf-8") + project = '[project]\nname = "example"\nversion = "1.0"\nlicense-files = ["LICENSE"]\nauthors = [{ name = "A" }]\n' + assert build_sbom(project=project)["metadata"]["component"] == { + "type": "application", + "bom-ref": "pkg:pypi/example@1.0", + "authors": [{"name": "A"}], + "name": "example", + "version": "1.0", + "purl": "pkg:pypi/example@1.0", + "externalReferences": [ + {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, + { + "type": "attestation", + "url": "https://pypi.org/integrity/example/1.0/example-1.0-py2.py3-none-any.whl/provenance", + }, + ], + "properties": [], + } + + +def test_document_metadata(build_sbom: BuildSbom) -> None: + metadata = build_sbom()["metadata"] + assert (metadata["timestamp"], metadata["lifecycles"], metadata.get("authors"), metadata.get("properties")) == ( + "2020-02-02T00:00:00+00:00", + [{"phase": "build"}], + [{"name": "Maintainer", "email": "maintainer@example.com"}], + [ + { + "name": "example:sbom:scope", + "value": "Components are the direct runtime dependencies declared in the wheel's core metadata. They" + " carry no version because they are resolved at install time, not at build time.", + } + ], + ) + + +def test_supplier(build_sbom: BuildSbom) -> None: + metadata = build_sbom(hook=f'{HOOK}supplier = {{ name = "Org", url = ["https://org.example"] }}\n')["metadata"] + assert [metadata.get("manufacturer"), metadata.get("supplier"), metadata["component"].get("supplier")] == [ + {"name": "Org", "url": ["https://org.example"]} + ] * 3 + + +def test_declared_dependencies(build_sbom: BuildSbom) -> None: + document = build_sbom() + assert (document["components"], document["dependencies"][0]) == ( + [ + { + "type": "library", + "bom-ref": "requires-dist:packaging>=24", + "name": "packaging", + "purl": "pkg:pypi/packaging", + "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/packaging/"}], + "properties": [{"name": "example:requires-dist", "value": "packaging>=24"}], + }, + { + "type": "library", + "bom-ref": "requires-dist:tomli>=2; python_version < '3.11'", + "name": "tomli", + "purl": "pkg:pypi/tomli", + "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/tomli/"}], + "properties": [ + {"name": "example:requires-dist", "value": "tomli>=2; python_version < '3.11'"}, + {"name": "example:environment-marker", "value": 'python_version < "3.11"'}, + ], + }, + ], + { + "ref": "pkg:pypi/example@1.0", + "dependsOn": ["requires-dist:packaging>=24", "requires-dist:tomli>=2; python_version < '3.11'"], + }, + ) + + +def test_composition_incomplete(build_sbom: BuildSbom) -> None: + assert build_sbom()["compositions"] == [{"aggregate": "incomplete", "dependencies": ["pkg:pypi/example@1.0"]}] + + +def test_build_workflow(build_sbom: BuildSbom) -> None: + document = build_sbom() + workflow = document["formulation"][0]["workflows"][0] + assert {**workflow, "resourceReferences": len(workflow["resourceReferences"])} == { + "bom-ref": "workflow:wheel-build", + "uid": "wheel-build", + "name": "build the wheel and this SBOM", + "taskTypes": ["build"], + "resourceReferences": len(document["metadata"]["tools"]["components"]), + "outputs": [{"type": "artifact", "resource": {"ref": "pkg:pypi/example@1.0"}}], + "inputs": [{"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": "1580601600"}]}], + } + + +def test_bundled_wheel(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: + sha256: Final[str] = hashlib.sha256(bundle_wheel({}, "License-Expression: MIT\n").read_bytes()).hexdigest() + assert build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0] == { + "type": "library", + "bom-ref": "pkg:pypi/pip@1.0", + "name": "pip", + "version": "1.0", + "purl": "pkg:pypi/pip@1.0", + "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], + "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/pip/1.0/"}], + "properties": [{"name": "example:bundled-wheel", "value": "src/example/wheels/pip-1.0-py3-none-any.whl"}], + "hashes": [{"alg": "SHA-256", "content": sha256}], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [ + {"technique": "manifest-analysis", "confidence": 1, "value": "pip-1.0.dist-info/METADATA"} + ], + }, + { + "field": "hash", + "confidence": 1, + "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], + }, + ] + }, + "components": [ + { + "type": "file", + "bom-ref": "pkg:pypi/pip@1.0#pip/example.py", + "name": "pip/example.py", + "hashes": [{"alg": "SHA-256", "content": _SHA256_ABC}], + "properties": [{"name": "size", "value": "3"}], + } + ], + } + + +def test_bundled_wheel_scope(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: + bundle_wheel({}) + assert build_sbom(hook=f"{HOOK}{BUNDLED}")["metadata"].get("properties") == [ + { + "name": "example:sbom:scope", + "value": "Components are the wheels bundled as data files under src/example/wheels and the direct runtime" + " dependencies declared in the wheel's core metadata. The latter carry no version because they are" + " resolved at install time, not at build time.", + } + ] + + +def test_bundled_wheels_sorted_once(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: + bundle_wheel({}, name="setuptools") + bundle_wheel({}, name="pip") + hook = f'{HOOK}bundled-wheels = ["src/example/wheels/*.whl", "src/example/wheels/pip-*"]\n' + assert build_sbom(hook=hook)["dependencies"][0]["dependsOn"] == [ + "pkg:pypi/pip@1.0", + "pkg:pypi/setuptools@1.0", + "requires-dist:packaging>=24", + "requires-dist:tomli>=2; python_version < '3.11'", + ] + + +@pytest.mark.parametrize( + ("source", "urls", "expected"), + [ + pytest.param("https://github.com/example/pip", "urls.Source = 'https://github.com/example/x'", True, id="same"), + pytest.param("https://github.com/other/pip", "urls.Source = 'https://github.com/example/x'", False, id="other"), + pytest.param("https://github.com/example/pip", "", False, id="no-repository"), + ], +) +def test_bundled_wheel_supplier( + build_sbom: BuildSbom, bundle_wheel: BundleWheel, source: str, urls: str, expected: bool +) -> None: + bundle_wheel({}, f"Project-URL: Source, {source}\n") + project = f'[project]\nname = "example"\nversion = "1.0"\n{urls}\n' + hook = f'{HOOK}{BUNDLED}supplier = {{ name = "Org" }}\n' + assert ("supplier" in build_sbom(hook=hook, project=project)["components"][0]) is expected + + +@pytest.mark.parametrize( + "filename", + [ + pytest.param("pip/example.py", id="plain"), + pytest.param("pip/a,b.py", id="comma"), + pytest.param('pip/a"b.py', id="quote"), + pytest.param("pip/a\nb.py", id="newline"), + ], +) +def test_bundled_wheel_record_paths(root: Path, build_sbom: BuildSbom, filename: str) -> None: + record: Final[StringIO] = StringIO(newline="") + csv.writer(record).writerows([(filename, "sha256=ungWv48Bz-pBQUDeXa4iI7ADYaOWF3qctBD_YfIAFa0", "3")]) + (wheels := root / "src" / "example" / "wheels").mkdir(parents=True) + with zipfile.ZipFile(wheels / "pip-1.0-py3-none-any.whl", "w") as archive: + archive.writestr("pip-1.0.dist-info/METADATA", "Metadata-Version: 2.4\nName: pip\nVersion: 1.0\n") + archive.writestr("pip-1.0.dist-info/RECORD", f"{record.getvalue()}pip-1.0.dist-info/RECORD,,\n") + assert build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0].get("components") == [ + { + "type": "file", + "bom-ref": f"pkg:pypi/pip@1.0#{filename}", + "name": filename, + "hashes": [{"alg": "SHA-256", "content": _SHA256_ABC}], + "properties": [{"name": "size", "value": "3"}], + }, + ] + + +@pytest.mark.parametrize( + ("source", "content"), + [ + pytest.param("pip/_vendor/vendor.txt", "# vendored\n\n urllib3==1.26.4 # patched\n", id="pip-manifest"), + pytest.param( + "pip/_vendor/urllib3-1.26.4.dist-info/METADATA", "Name: urllib3\nVersion: 1.26.4\n", id="nested-metadata" + ), + ], +) +def test_vendored_identity(build_sbom: BuildSbom, bundle_wheel: BundleWheel, source: str, content: str) -> None: + bundle_wheel({source: content}) + children: Final[list[Component]] = build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0].get("components", []) + assert [child for child in children if child["type"] == "library"] == [ + { + "type": "library", + "bom-ref": "pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4", + "name": "urllib3", + "version": "1.26.4", + "purl": "pkg:pypi/urllib3@1.26.4", + "externalReferences": [], + "properties": [{"name": "example:vendored-manifest", "value": source}], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], + } + ] + }, + } + ] + + +def test_vendored_relationship(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: + bundle_wheel({"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}) + assert build_sbom(hook=f"{HOOK}{BUNDLED}")["dependencies"][1] == { + "ref": "pkg:pypi/pip@1.0", + "dependsOn": ["pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4"], + } + + +def test_vendored_metadata_precedence(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: + bundle_wheel({ + "pip/_vendor/vendor.txt": "urllib3==1.26.4\n", + "pip/_vendor/urllib3-1.26.4.dist-info/METADATA": "Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n", + }) + children: Final[list[Component]] = build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0].get("components", []) + assert [child.get("licenses") for child in children if child["type"] == "library"] == [ + [{"expression": "MIT", "acknowledgement": "declared"}], + ] + + +@pytest.mark.parametrize( + "requirement", + [ + pytest.param("urllib3", id="missing"), + pytest.param("urllib3>=1", id="range"), + pytest.param("urllib3==1.*", id="wildcard"), + pytest.param("urllib3 @ https://example.com/package.whl", id="url"), + ], +) +def test_vendored_unpinned_rejected(build_sbom: BuildSbom, bundle_wheel: BundleWheel, requirement: str) -> None: + bundle_wheel({"pip/_vendor/vendor.txt": requirement}) + with pytest.raises(ValueError, match="Unpinned vendored dependency"): + build_sbom(hook=f"{HOOK}{BUNDLED}") + + +@pytest.mark.parametrize( + ("metadata", "expected"), + [ + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\n", + [ + {"license": {"name": "MIT License", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ], + id="classifier", + ), + pytest.param( + "Classifier: License :: OSI Approved :: Python Software Foundation License\nLicense: PSF-2.0\n", + [ + {"license": {"name": "Python Software Foundation License", "acknowledgement": "declared"}}, + {"license": {"name": "PSF-2.0", "acknowledgement": "declared"}}, + {"license": {"id": "PSF-2.0", "acknowledgement": "concluded"}}, + ], + id="classifier-and-id-agree", + ), + pytest.param( + "License: mit\n", + [ + {"license": {"name": "mit", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ], + id="id-canonical-case", + ), + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\n" + "Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)\n", + [ + {"license": {"name": "MIT License", "acknowledgement": "declared"}}, + {"license": {"name": "GNU Lesser General Public License v3 (LGPLv3)", "acknowledgement": "declared"}}, + ], + id="one-unresolved", + ), + pytest.param( + "License: MIT OR Apache-2.0\n", + [{"license": {"name": "MIT OR Apache-2.0", "acknowledgement": "declared"}}], + id="expression-in-license-field", + ), + pytest.param( + "License: LicenseRef-Proprietary\n", + [{"license": {"name": "LicenseRef-Proprietary", "acknowledgement": "declared"}}], + id="license-ref", + ), + ], +) +def test_license_resolution( + build_sbom: BuildSbom, bundle_wheel: BundleWheel, metadata: str, expected: list[dict[str, dict[str, str]]] +) -> None: + bundle_wheel({}, metadata) + assert build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0].get("licenses") == expected + + +def test_bundled_wheel_metadata_fields(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: + bundle_wheel( + {}, + "Summary: installer\nHome-page: https://pip.pypa.io\nAuthor: Someone\nAuthor-email: dev@example.com\n" + "Maintainer-email: Team <>\nProject-URL: Bug Tracker, https://example.com/issues\n", + ) + component = build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0] + assert (component.get("description"), component.get("authors"), component.get("externalReferences")) == ( + "installer", + [{"name": "Someone"}, {"email": "dev@example.com"}, {"name": "Team"}], + [ + {"type": "website", "url": "https://pip.pypa.io"}, + {"type": "issue-tracker", "url": "https://example.com/issues", "comment": "Project-URL: Bug Tracker"}, + {"type": "distribution", "url": "https://pypi.org/project/pip/1.0/"}, + ], + ) + + +@pytest.mark.parametrize( + ("epoch", "expected"), + [ + pytest.param("1580601600", ("2020-02-02T00:00:00+00:00", "1580601600"), id="hatchling-default"), + pytest.param("1700000000", ("2023-11-14T22:13:20+00:00", "1700000000"), id="source-date-epoch"), + ], +) +def test_timestamp( + build_sbom: BuildSbom, monkeypatch: pytest.MonkeyPatch, epoch: str, expected: tuple[str, str] +) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", epoch) + document = build_sbom() + variables = document["formulation"][0]["workflows"][0]["inputs"][0]["environmentVars"] + assert (document["metadata"]["timestamp"], variables[0]["value"]) == expected + + +def test_build_host_independent(write_project: WriteProject, mocker: MockerFixture, install_tool: InstallTool) -> None: + root = write_project() + assert _build_on_host(root, mocker, install_tool, ("x86_64", "GCC 13.3.0", b"uv\n")) == _build_on_host( + root, mocker, install_tool, ("aarch64", "Clang 20.1.4", b"pip\n") + ) + + +def _build_on_host( + root: Path, mocker: MockerFixture, install_tool: InstallTool, host: tuple[str, str, bytes] +) -> Document: + machine, compiler, installer = host + mocker.patch("platform.machine", autospec=True, return_value=machine) + mocker.patch("platform.platform", autospec=True, return_value=f"Linux-{machine}") + mocker.patch("platform.python_compiler", autospec=True, return_value=compiler) + mocker.patch.object(sys, "version", f"3.14.7 (main) [{compiler}]") + install_tool(site := root.parent / machine, "native", False, {f"native/_speedups.{machine}.so": b"\0"}) + install_tool(site, "pure", True, {"pure-1.0.dist-info/INSTALLER": installer, "pure/__init__.py": b""}) + mocker.patch.object(sys, "path", [str(site), *sys.path]) + document = build_hook_sbom(root) + mocker.stopall() + return document + + +def test_build_tools_pure_files( + write_project: WriteProject, mocker: MockerFixture, install_tool: InstallTool, tmp_path: Path +) -> None: + install_tool( + site := tmp_path / "site", + "pure", + True, + {"pure-1.0.dist-info/INSTALLER": b"uv\n", "pure/__init__.py": b"", "../../bin/pure": b"#!python\n"}, + ) + install_tool(site, "native", False, {"native/_speedups.so": b"\0"}) + mocker.patch.object(sys, "path", [str(site), *sys.path]) + tools = {tool["name"]: tool for tool in build_hook_sbom(write_project())["metadata"]["tools"]["components"]} + assert ([file["name"] for file in tools["pure"].get("components", [])], "components" in tools["native"]) == ( + ["pure/__init__.py"], + False, + ) + + +def test_ci_environment_ignored(write_project: WriteProject, monkeypatch: pytest.MonkeyPatch) -> None: + root = write_project() + monkeypatch.setenv("GITHUB_SHA", "a" * 40) + first: Final[Document] = build_hook_sbom(root) + monkeypatch.setenv("GITHUB_SHA", "b" * 40) + assert build_hook_sbom(root) == first + + +def test_source_revision(write_project: WriteProject, monkeypatch: pytest.MonkeyPatch) -> None: + root = write_project() + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(root / "gitconfig")) + (root / "gitconfig").write_text("[user]\n\tname = Example\n\temail = example@example.com\n", encoding="utf-8") + git = shutil.which("git") or "git" + subprocess.run([git, "init", "-q"], cwd=root, check=True) + subprocess.run([git, "commit", "-q", "--allow-empty", "--no-gpg-sign", "-m", "init"], cwd=root, check=True) + commit = subprocess.run([git, "rev-parse", "HEAD"], cwd=root, check=True, capture_output=True, text=True) + component = build_hook_sbom(root)["metadata"]["component"] + assert (component.get("externalReferences", [])[-1], component.get("properties", [])[-1]) == ( + { + "type": "vcs", + "url": f"https://github.com/example/example/tree/{commit.stdout.strip()}", + "comment": "exact source revision", + }, + {"name": "example:vcs-commit", "value": commit.stdout.strip()}, + ) + + +def test_source_revision_without_repository(root: Path, build_sbom: BuildSbom, mocker: MockerFixture) -> None: + (root / ".git").mkdir() + mocker.patch("subprocess.run", autospec=True, return_value=subprocess.CompletedProcess(["git"], 0, "c0ffee\n", "")) + component = build_sbom(project='[project]\nname = "example"\nversion = "1.0"\n')["metadata"]["component"] + assert component.get("properties") == [{"name": "example:vcs-commit", "value": "c0ffee"}] + + +def test_source_revision_unreadable(root: Path, build_sbom: BuildSbom, mocker: MockerFixture) -> None: + (root / ".git").mkdir() + error = subprocess.CalledProcessError(128, ["git"], stderr="fatal: detected dubious ownership") + mocker.patch("subprocess.run", autospec=True, side_effect=error) + with pytest.raises(subprocess.CalledProcessError): + build_sbom() + + +@pytest.mark.parametrize( + ("hook", "expected"), + [ + pytest.param( + f'{HOOK}repository = "https://gitlab.com/example/example/"\n', + ( + ["release-notes", "vcs", "distribution", "attestation", "license"], + "https://gitlab.com/example/example/blob/main/LICENSE", + ), + id="off-github", + ), + pytest.param( + f'{HOOK}default-branch = "trunk"\n', + ( + ["release-notes", "vcs", "distribution", "attestation", "security-contact", "advisories", "license"], + "https://github.com/example/example/blob/trunk/LICENSE", + ), + id="default-branch", + ), + ], +) +def test_repository_links(build_sbom: BuildSbom, hook: str, expected: tuple[list[str], str]) -> None: + references = build_sbom(hook=hook)["metadata"]["component"].get("externalReferences", []) + assert ([reference["type"] for reference in references], references[-1]["url"]) == expected + + +def test_serial_number_per_project(build_sbom: BuildSbom) -> None: + assert ( + build_sbom()["serialNumber"] + != build_sbom(project='[project]\nname = "Example"\nversion = "1.0"\n')["serialNumber"] + ) + + +@pytest.mark.parametrize( + ("hook", "message"), + [ + pytest.param(f'{HOOK}supplier = "Org"\n', "`supplier` in build hook `pep770` must be a table", id="supplier"), + pytest.param(f'{HOOK}supplier = {{ name = "Org", email = "x" }}\n', "`supplier`", id="supplier-unknown-key"), + pytest.param(f"{HOOK}repository = 1\n", "`repository` in build hook `pep770` must be a string", id="repo"), + pytest.param(f"{HOOK}default-branch = 1\n", "`default-branch` in build hook `pep770` must be", id="branch"), + pytest.param(f'{HOOK}bundled-wheels = "*.whl"\n', "`bundled-wheels` in build hook `pep770`", id="bundled"), + ], +) +def test_invalid_option(build_sbom: BuildSbom, hook: str, message: str) -> None: + with pytest.raises(TypeError, match=message): + build_sbom(hook=hook) diff --git a/tests/test_spdx.py b/tests/test_spdx.py index e72aeab..7feb0a3 100644 --- a/tests/test_spdx.py +++ b/tests/test_spdx.py @@ -1,37 +1,32 @@ from __future__ import annotations -import json -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import pytest +from conftest import BUNDLED, HOOK -from hatch_pep770.spdx import to_spdx -from hatch_pep770.version import __version__ +from hatch_pep770 import __version__, to_spdx if TYPE_CHECKING: - from collections.abc import Callable from pathlib import Path - from hatch_pep770.cyclonedx import Document - from tests.conftest import BundleWheel + from conftest import BuildSbom, BundleWheel -_BUNDLED: Final[str] = "src/example/wheels/*.whl" + from hatch_pep770 import Document + +_SUPPLIED: Final[str] = f'{HOOK}{BUNDLED}supplier = {{ name = "Example Org" }}\n' @pytest.fixture -def bundled_config(bundle_wheel: BundleWheel, config: dict[str, Any]) -> dict[str, Any]: +def bundled(bundle_wheel: BundleWheel) -> None: bundle_wheel( {"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}, "Project-URL: Source, https://github.com/example/pip\n" ) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = { - "bundled-wheels": [_BUNDLED], - "supplier": {"name": "Example Org"}, - } - return config -def test_spdx_bundled_wheel(build_sbom: Callable[[dict[str, Any]], str], bundled_config: dict[str, Any]) -> None: - cyclonedx: Final[Document] = json.loads(build_sbom(bundled_config)) +@pytest.mark.usefixtures("bundled") +def test_spdx_bundled_wheel(build_sbom: BuildSbom) -> None: + cyclonedx: Final[Document] = build_sbom(hook=_SUPPLIED) assert to_spdx(cyclonedx)["packages"][1] == { "SPDXID": "SPDXRef-pkg-pypi-pip-1.0", "name": "pip", @@ -42,7 +37,9 @@ def test_spdx_bundled_wheel(build_sbom: Callable[[dict[str, Any]], str], bundled "primaryPackagePurpose": "LIBRARY", "versionInfo": "1.0", "supplier": "Organization: Example Org", - "checksums": [{"algorithm": "SHA256", "checksumValue": cyclonedx["components"][0]["hashes"][0]["content"]}], + "checksums": [ + {"algorithm": "SHA256", "checksumValue": cyclonedx["components"][0].get("hashes", [])[0]["content"]} + ], "comment": "example:bundled-wheel: src/example/wheels/pip-1.0-py3-none-any.whl", "externalRefs": [ {"referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", "referenceLocator": "pkg:pypi/pip@1.0"} @@ -50,10 +47,10 @@ def test_spdx_bundled_wheel(build_sbom: Callable[[dict[str, Any]], str], bundled } -def test_spdx_root_package(sbom_document: Document) -> None: - assert to_spdx(sbom_document)["packages"][0] == { +def test_spdx_root_package(build_sbom: BuildSbom) -> None: + assert to_spdx(build_sbom())["packages"][0] == { "SPDXID": "SPDXRef-pkg-pypi-example-1.0", - "name": "example", + "name": "Example", "downloadLocation": "NOASSERTION", "filesAnalyzed": False, "licenseConcluded": "NOASSERTION", @@ -62,7 +59,7 @@ def test_spdx_root_package(sbom_document: Document) -> None: "versionInfo": "1.0", "copyrightText": "Copyright (c) 2026 Example", "summary": "an example", - "comment": "example:requires-python: >=3.9", + "comment": "example:requires-python: >=3.11", "externalRefs": [ { "referenceCategory": "PACKAGE-MANAGER", @@ -73,10 +70,11 @@ def test_spdx_root_package(sbom_document: Document) -> None: } -def test_spdx_relationships(build_sbom: Callable[[dict[str, Any]], str], bundled_config: dict[str, Any]) -> None: +@pytest.mark.usefixtures("bundled") +def test_spdx_relationships(build_sbom: BuildSbom) -> None: assert [ relationship - for relationship in to_spdx(_load(build_sbom(bundled_config)))["relationships"] + for relationship in to_spdx(build_sbom(hook=_SUPPLIED))["relationships"] if not relationship["spdxElementId"].startswith("SPDXRef-tool-") ] == [ { @@ -112,10 +110,10 @@ def test_spdx_relationships(build_sbom: Callable[[dict[str, Any]], str], bundled ] -def test_spdx_build_tools(sbom_document: Document) -> None: +def test_spdx_build_tools(build_sbom: BuildSbom) -> None: assert { relationship["relatedSpdxElement"] - for relationship in to_spdx(sbom_document)["relationships"] + for relationship in to_spdx(build_sbom())["relationships"] if relationship["relationshipType"] == "BUILD_TOOL_OF" } == {"SPDXRef-pkg-pypi-example-1.0"} @@ -138,60 +136,40 @@ def test_spdx_build_tools(sbom_document: Document) -> None: ], ) def test_spdx_declared_license( - build_sbom: Callable[[dict[str, Any]], str], - bundle_wheel: BundleWheel, - config: dict[str, Any], - metadata: str, - expected: tuple[str, str, str | None], + build_sbom: BuildSbom, bundle_wheel: BundleWheel, metadata: str, expected: tuple[str, str, str | None] ) -> None: bundle_wheel({}, metadata) - config["tool"]["hatch"]["build"]["hooks"]["pep770"] = {"bundled-wheels": [_BUNDLED]} - package: Final[dict[str, Any]] = dict(to_spdx(_load(build_sbom(config)))["packages"][1]) + package = to_spdx(build_sbom(hook=f"{HOOK}{BUNDLED}"))["packages"][1] assert (package["licenseDeclared"], package["licenseConcluded"], package.get("licenseComments")) == expected @pytest.mark.parametrize( - ("urls", "hooks", "expected"), + ("hook", "expected"), [ pytest.param( - {"Source": "https://github.com/example/example"}, - {"pep770": {"supplier": {"name": "Example Org"}}}, - ("https://github.com/example/example/sboms/example-1.0-", ["Organization: Example Org"]), + f'{HOOK}supplier = {{ name = "Example Org" }}\n', + ("https://github.com/example/example/sboms/Example-1.0-", ["Organization: Example Org"]), id="repository-and-supplier", ), - pytest.param({}, {"pep770": {}}, ("https://spdx.org/spdxdocs/sboms/example-1.0-", []), id="neither"), pytest.param( - {}, - {"custom": {}}, - ("https://spdx.org/spdxdocs/sboms/example-1.0-", ["Tool: hatch_build.py-1.0"]), + '[tool.hatch.build.hooks.custom]\npath = "build.py"\n', + ("https://github.com/example/example/sboms/Example-1.0-", ["Tool: build.py-1.0"]), id="custom-build-script", ), ], ) def test_spdx_creation_info( - root: Path, - build_sbom: Callable[[dict[str, Any]], str], - config: dict[str, Any], - monkeypatch: pytest.MonkeyPatch, - urls: dict[str, str], - hooks: dict[str, Any], - expected: tuple[str, list[str]], + root: Path, build_sbom: BuildSbom, monkeypatch: pytest.MonkeyPatch, hook: str, expected: tuple[str, list[str]] ) -> None: monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") - (root / "hatch_build.py").write_text( - "from hatch_pep770 import plugin\n\n\ndef get_build_hook():\n return plugin.Pep770BuildHook\n", + (root / "build.py").write_text( + "from hatch_pep770 import Pep770BuildHook\n\n\ndef get_build_hook():\n return Pep770BuildHook\n", encoding="utf-8", ) - config["project"]["urls"] = urls - config["tool"]["hatch"]["build"]["hooks"] = hooks - cyclonedx: Final[Document] = json.loads(build_sbom(config)) - document: Final[dict[str, Any]] = dict(to_spdx(cyclonedx)) + cyclonedx: Final[Document] = build_sbom(hook=hook) + spdx = to_spdx(cyclonedx) namespace, creators = expected - assert ( - document["documentNamespace"], - document["creationInfo"], - document["comment"], - ) == ( + assert (spdx["documentNamespace"], spdx["creationInfo"], spdx["comment"]) == ( f"{namespace}{cyclonedx['serialNumber'].removeprefix('urn:uuid:')}", {"created": "2023-11-14T22:13:20Z", "creators": [*creators, f"Tool: hatch-pep770-{__version__}"]}, ( @@ -201,5 +179,8 @@ def test_spdx_creation_info( ) -def _load(content: str) -> Document: - return json.loads(content) +def test_spdx_namespace_without_repository(build_sbom: BuildSbom) -> None: + cyclonedx: Final[Document] = build_sbom(project='[project]\nname = "Example"\nversion = "1.0"\n') + assert to_spdx(cyclonedx)["documentNamespace"] == ( + f"https://spdx.org/spdxdocs/sboms/Example-1.0-{cyclonedx['serialNumber'].removeprefix('urn:uuid:')}" + ) diff --git a/tests/test_validate.py b/tests/test_validate.py index d526e3b..3b18969 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -6,12 +6,16 @@ import pytest -from hatch_pep770.validate import document_problems, license_problems, wheel_problems +from hatch_pep770 import validate, validate_wheel if TYPE_CHECKING: from pathlib import Path - from hatch_pep770.cyclonedx import Component, Document, License + from conftest import BuildSbom + from pytest_mock import MockerFixture + + from hatch_pep770 import Component, Document + from hatch_pep770._cyclonedx import License _ALLOWED: Final[frozenset[str]] = frozenset({"Apache-2.0", "BSD-3-Clause", "MIT"}) @@ -28,16 +32,29 @@ def _concluded(identifier: str) -> License: return {"license": {"id": identifier, "acknowledgement": "concluded"}} -def _denied(expression: str, bom_ref: str = "pkg:pypi/example@1.0") -> str: +def _denied(expression: str, bom_ref: str = "pkg:pypi/bundled@1.0") -> str: return f"{bom_ref}: license {expression} is outside the allowed licenses ['Apache-2.0', 'BSD-3-Clause', 'MIT']" +@pytest.fixture +def sbom_document(build_sbom: BuildSbom, mocker: MockerFixture) -> Document: + # schema validation time grows with the document, and the test environment's packages are irrelevant here + mocker.patch("hatch_pep770._cyclonedx.distributions", autospec=True, return_value=[]) + return build_sbom() + + +def _with_component(document: Document, component: Component) -> str: + document["components"].append(component) + document["dependencies"][0]["dependsOn"].append(component["bom-ref"]) + return json.dumps(document) + + def test_document_valid(sbom_document: Document) -> None: - assert document_problems(json.dumps(sbom_document), _ALLOWED) == [] + assert validate(json.dumps(sbom_document), _ALLOWED) == [] def test_document_schema_violation() -> None: - problems: Final[list[str]] = document_problems( + problems: Final[list[str]] = validate( json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "1"}), None ) assert [problem.splitlines()[0] for problem in problems] == [ @@ -46,26 +63,27 @@ def test_document_schema_violation() -> None: def test_document_serial_number_missing(sbom_document: Document) -> None: - unnumbered: Final[dict[str, object]] = {key: value for key, value in sbom_document.items() if key != "serialNumber"} - assert document_problems(json.dumps(unnumbered), None) == ["serialNumber must be a urn:uuid: RFC 4122 UUID, got ''"] + unnumbered = json.loads(json.dumps(sbom_document)) + del unnumbered["serialNumber"] + assert validate(json.dumps(unnumbered), None) == ["serialNumber must be a urn:uuid: RFC 4122 UUID, got ''"] def test_document_duplicate_bom_ref(sbom_document: Document) -> None: sbom_document["components"][1]["bom-ref"] = sbom_document["components"][0]["bom-ref"] - assert document_problems(json.dumps(sbom_document), None)[0] == ( + assert validate(json.dumps(sbom_document), None)[0] == ( "bom-ref values are not unique: ['requires-dist:packaging>=24']" ) def test_document_dangling_dependency(sbom_document: Document) -> None: sbom_document["dependencies"].append({"ref": "pkg:pypi/example@1.0", "dependsOn": ["pkg:pypi/missing"]}) - problems: Final[list[str]] = document_problems(json.dumps(sbom_document), None) + problems: Final[list[str]] = validate(json.dumps(sbom_document), None) assert problems[0] == "dependencies reference bom-refs that do not exist: ['pkg:pypi/missing']" def test_document_root_dependencies(sbom_document: Document) -> None: sbom_document["dependencies"][0]["dependsOn"] = ["requires-dist:packaging>=24"] - assert document_problems(json.dumps(sbom_document), None) == [ + assert validate(json.dumps(sbom_document), None) == [ ( "root dependsOn ['requires-dist:packaging>=24'] != components" " ['requires-dist:packaging>=24', \"requires-dist:tomli>=2; python_version < '3.11'\"]" @@ -75,32 +93,33 @@ def test_document_root_dependencies(sbom_document: Document) -> None: def test_document_workflow_unknown_tool(sbom_document: Document) -> None: sbom_document["formulation"][0]["workflows"][0]["resourceReferences"].append({"ref": "tool:missing"}) - assert document_problems(json.dumps(sbom_document), None) == [ + assert validate(json.dumps(sbom_document), None) == [ "workflow wheel-build references unknown tools: ['tool:missing']" ] def test_document_spdx_unparsable(sbom_document: Document) -> None: sbom_document["metadata"]["component"]["hashes"] = [{"alg": "BLAKE2b-256", "content": "0" * 64}] - assert document_problems(json.dumps(sbom_document), None)[0].startswith("SPDX rendering does not parse:") + assert validate(json.dumps(sbom_document), None)[0].startswith("SPDX rendering does not parse:") def test_document_spdx_invalid(sbom_document: Document) -> None: sbom_document["metadata"]["component"]["purl"] = "not a purl" - assert document_problems(json.dumps(sbom_document), None)[0].startswith("SPDX rendering invalid:") + assert validate(json.dumps(sbom_document), None)[0].startswith("SPDX rendering invalid:") def test_wheel_without_sbom(tmp_path: Path) -> None: with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: archive.writestr("example-1.0.dist-info/METADATA", "Name: example\n") - assert wheel_problems(wheel, None) == ["no .dist-info/sboms/*.cdx.json found in the wheel"] + assert validate_wheel(wheel, None) == ["no .dist-info/sboms/*.cdx.json found in the wheel"] def test_wheel_names_each_sbom(sbom_document: Document, tmp_path: Path) -> None: - unnumbered: Final[dict[str, object]] = {key: value for key, value in sbom_document.items() if key != "serialNumber"} + unnumbered = json.loads(json.dumps(sbom_document)) + del unnumbered["serialNumber"] with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: archive.writestr("example-1.0.dist-info/sboms/example.cdx.json", json.dumps(unnumbered)) - assert wheel_problems(wheel, None) == [ + assert validate_wheel(wheel, None) == [ "example-1.0.dist-info/sboms/example.cdx.json: serialNumber must be a urn:uuid: RFC 4122 UUID, got ''" ] @@ -136,21 +155,21 @@ def test_wheel_names_each_sbom(sbom_document: Document, tmp_path: Path) -> None: ), pytest.param( [_named("MIT License")], - ["pkg:pypi/example@1.0: declares license names ['MIT License'] with no SPDX id"], + ["pkg:pypi/bundled@1.0: declares license names ['MIT License'] with no SPDX id"], id="name-only", ), - pytest.param([], ["pkg:pypi/example@1.0: declares no license"], id="undeclared"), + pytest.param([], ["pkg:pypi/bundled@1.0: declares no license"], id="undeclared"), ], ) -def test_license_policy(licenses: list[License], expected: list[str]) -> None: +def test_license_policy(sbom_document: Document, licenses: list[License], expected: list[str]) -> None: component: Final[Component] = { "type": "library", - "bom-ref": "pkg:pypi/example@1.0", + "bom-ref": "pkg:pypi/bundled@1.0", "name": "example", "hashes": [{"alg": "SHA-256", "content": "0" * 64}], "licenses": licenses, } - assert license_problems([component], _ALLOWED) == expected + assert validate(_with_component(sbom_document, component), _ALLOWED) == expected @pytest.mark.parametrize( @@ -165,17 +184,18 @@ def test_license_policy(licenses: list[License], expected: list[str]) -> None: pytest.param("Apache-2.0 WITH OR", "unexpected 'OR'", id="with-operator"), ], ) -def test_license_policy_malformed(expression: str, reason: str) -> None: +def test_license_policy_malformed(sbom_document: Document, expression: str, reason: str) -> None: component: Final[Component] = { "type": "library", - "bom-ref": "pkg:pypi/example@1.0", + "bom-ref": "pkg:pypi/bundled@1.0", "name": "example", "hashes": [{"alg": "SHA-256", "content": "0" * 64}], "licenses": [_expression(expression)], } - assert license_problems([component], _ALLOWED) == [ - f"pkg:pypi/example@1.0: license expression {expression!r} is malformed: {reason}" - ] + # an expression the policy cannot parse does not parse as SPDX either, which the check reports first + assert validate(_with_component(sbom_document, component), _ALLOWED)[-1] == ( + f"pkg:pypi/bundled@1.0: license expression {expression!r} is malformed: {reason}" + ) @pytest.mark.parametrize( @@ -197,22 +217,13 @@ def test_license_policy_malformed(expression: str, reason: str) -> None: }, id="vendored-by-bundled-wheel", ), - pytest.param( - { - "type": "file", - "bom-ref": "loader", - "name": "loader", - "hashes": [{"alg": "SHA-256", "content": "0" * 64}], - }, - id="file", - ), ], ) -def test_license_policy_skips_unshipped(component: Component) -> None: - assert license_problems([component], _ALLOWED) == [] +def test_license_policy_skips_unshipped(sbom_document: Document, component: Component) -> None: + assert validate(_with_component(sbom_document, component), _ALLOWED) == [] -def test_license_policy_judges_files_owner() -> None: +def test_license_policy_judges_files_owner(sbom_document: Document) -> None: component: Final[Component] = { "type": "library", "bom-ref": "pkg:pypi/distlib@0.4", @@ -220,4 +231,6 @@ def test_license_policy_judges_files_owner() -> None: "licenses": [_concluded("LGPL-3.0-only")], "components": [{"type": "file", "bom-ref": "pkg:pypi/distlib@0.4#distlib/__init__.py", "name": "x"}], } - assert license_problems([component], _ALLOWED) == [_denied("LGPL-3.0-only", "pkg:pypi/distlib@0.4")] + assert validate(_with_component(sbom_document, component), _ALLOWED) == [ + _denied("LGPL-3.0-only", "pkg:pypi/distlib@0.4") + ] diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 0000000..1436630 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from packaging.version import Version + +from hatch_pep770 import __version__ + + +def test_version() -> None: + assert Version(__version__).release From 6b77aea77d7f80b63feb576cc044017626e1612a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 19:54:34 -0800 Subject: [PATCH 006/115] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20the=20Read?= =?UTF-8?q?=20the=20Docs=20site?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/CONTRIBUTING.md | 18 +++ .github/SECURITY.md | 25 ++-- .github/THREAT_MODEL.md | 72 ++++++++++ .readthedocs.yaml | 8 ++ CHANGELOG.md | 3 - CONTRIBUTING.md | 43 ------ README.md | 110 ++------------- changelog.d/+pep770-hook.feature.md | 2 - docs/__init__.py | 1 + docs/_static/changelog-toc.css | 12 ++ docs/changelog.d/+pep770-hook.feature.rst | 3 + {changelog.d => docs/changelog.d}/.gitkeep | 0 docs/changelog.rst | 4 + docs/cli.rst | 5 + docs/conf.py | 128 +++++++++++++++++ docs/contributing.rst | 28 ++++ docs/explanation.rst | 45 ++++++ docs/extend.rst | 152 +++++++++++++++++++++ docs/index.rst | 43 ++++++ docs/py_api.rst | 80 +++++++++++ docs/usage.rst | 117 ++++++++++++++++ 21 files changed, 742 insertions(+), 157 deletions(-) create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/THREAT_MODEL.md create mode 100644 .readthedocs.yaml delete mode 100644 CHANGELOG.md delete mode 100644 changelog.d/+pep770-hook.feature.md create mode 100644 docs/__init__.py create mode 100644 docs/_static/changelog-toc.css create mode 100644 docs/changelog.d/+pep770-hook.feature.rst rename {changelog.d => docs/changelog.d}/.gitkeep (100%) create mode 100644 docs/changelog.rst create mode 100644 docs/cli.rst create mode 100644 docs/conf.py create mode 100644 docs/contributing.rst create mode 100644 docs/explanation.rst create mode 100644 docs/extend.rst create mode 100644 docs/index.rst create mode 100644 docs/py_api.rst create mode 100644 docs/usage.rst diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..8090cea --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,18 @@ +# Contributing + +Add a `docs/changelog.d/+short-name..rst` fragment for a change that affects package users. Choose a unique short +name; the release workflow links the fragment to its merged squash pull request and credits the human author. A pull +request number is not needed in the filename. + +Use `feature` for Added, `change` for Changes, `bugfix` for Fixed, and `doc` for Documentation. Use `breaking`, +`security`, `performance`, `deprecation`, `removal`, or `packaging` only when the change affects users in that way. A +supported Python version added or removed is a `feature` and results in a minor release. + +Use `skip news` for internal maintenance without a user-facing change. The label may be added after opening the pull +request. Dependabot and maintenance bot pull requests without a fragment receive it automatically; a bot pull request +with a fragment still needs the news check to pass. For a change written by more than one person, put all human authors +in a `by` credit. Do not list bots or reviewers. + +The documentation preview shows fragments under Unreleased. The versioned `docs/changelog.rst` changes only during +release preparation. The release workflow chooses a version from the fragment types, runs `prek` on the assembled +changelog, and publishes GitHub notes from the tagged changelog after PyPI succeeds. diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 5447e64..0804a5f 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -1,14 +1,14 @@ # Security policy -hatch-pep770 is a hatchling build hook that writes a CycloneDX SBOM into wheels, plus a command line tool that renders -that SBOM as SPDX and validates it. This policy says how to report a security problem and how we decide whether a report -is one. +hatch-pep770 writes a CycloneDX SBOM into wheels, from a hatchling build hook or from a command that rewrites a wheel +another build backend built, and renders or validates that SBOM. This policy says how to report a security problem and +how we decide whether a report is one. ## Reporting a vulnerability **Do not open a public issue for a security problem.** Use -[GitHub's private vulnerability reporting](https://github.com/tox-dev/hatch-pep770/security/advisories/new). If you do -not have a GitHub account, email [gaborjbernat@gmail.com](mailto:gaborjbernat@gmail.com). +[GitHub's private vulnerability reporting](https://github.com/tox-dev/hatch-pep770/security/advisories/new). If neither +works for you, email [gaborjbernat@gmail.com](mailto:gaborjbernat@gmail.com). Include what the attacker controls and what they gain, the steps or a proof of concept, and the versions and platform you tested. Write the report yourself; unreviewed tool or model output takes longer to triage. @@ -27,19 +27,20 @@ Only the latest release receives fixes. Confirm the problem reproduces on it bef ## Vulnerability or bug -The build hook runs inside a project's own build, with the trust that build already has: the project source, its -`pyproject.toml`, its build requirements and the build environment are trusted. A report that needs control of one of -those is a bug: we fix it in the open and publish no advisory. +A report is a vulnerability when an attacker who controls only an input that the [threat model](THREAT_MODEL.md) lists +as untrusted breaks a guarantee hatch-pep770 makes. A report that first needs control of something the threat model +trusts, such as the project source, its `pyproject.toml` or the build environment, is a bug: we fix it in the open and +publish no advisory. Classify by who supplies the dangerous input, not by how bad the outcome sounds. **In scope** -- A wheel bundled as a data file, or an SBOM given to `hatch-pep770 spdx` or `validate`, that makes the tool write - outside its output file or the wheel's `.dist-info/sboms/` directory, or run code. -- An SBOM that misstates what the wheel contains in a way the project's own inputs cannot explain, such as a wrong hash. +- A bundled wheel, or an SBOM given to `hatch-pep770 spdx` or `validate`, that makes hatch-pep770 write outside its + output file or the wheel it rewrites, or run code. +- An SBOM that misstates what a wheel holds in a way its inputs cannot explain, such as a wrong hash. **Out of scope** - Anything hatchling's own build trusts. -- Vulnerabilities in the packages the SBOM lists; report those to their projects. +- Vulnerabilities in the packages an SBOM lists; report those to their projects. If you cannot tell which side a finding falls on, report it privately and we will decide. diff --git a/.github/THREAT_MODEL.md b/.github/THREAT_MODEL.md new file mode 100644 index 0000000..e84cf8b --- /dev/null +++ b/.github/THREAT_MODEL.md @@ -0,0 +1,72 @@ +# Threat model + +This model says what hatch-pep770 trusts, where untrusted input enters, and which guarantees hold at each boundary, so a +report can be classified as a vulnerability or a bug. [SECURITY.md](SECURITY.md) covers reporting; if the two disagree, +SECURITY.md wins and we fix this file. + +## How hatch-pep770 works + +The build hook runs inside a hatchling wheel build. It reads the wheel's core metadata, the project's license files, the +wheels the project bundles and the installed distributions of the build environment, and writes a CycloneDX document +that hatchling adds to the wheel. `hatch-pep770 embed` does the same for a wheel another backend built and rewrites that +wheel. `hatch-pep770 spdx` and `validate` read a CycloneDX document and write or check its SPDX form. + +```mermaid +flowchart LR + project[("Project source, pyproject.toml
(trusted)")] --> hook["hatch-pep770"] + env[("Build environment
(trusted)")] --> hook + bundled[("Bundled wheels
(third-party content)")] -->|"metadata, RECORD, vendor list"| hook + sbom[("SBOM files given to spdx, validate")] -->|"untrusted JSON"| hook + hook --> wheel[("Wheel")] + classDef trusted fill:#cfe4ff,stroke:#1f6feb,color:#0b1f3a; + classDef boundary fill:#ffe3a3,stroke:#d29200,color:#3a2c00; + class project,env,hook,wheel trusted; + class bundled,sbom boundary; +``` + +## Trusted + +An attacker who controls any of these already has what hatch-pep770 could give them: + +- The project source, its `pyproject.toml` and the build hook script. +- The build environment and the build backend. +- The wheel handed to `hatch-pep770 embed`, which the project's own build produced. + +## Untrusted inputs + +| Input | Comes from | What hatch-pep770 guarantees | +| --------------------------------------- | ------------------------- | ----------------------------------------------------------------------------------------- | +| Metadata and `RECORD` of bundled wheels | Upstream projects | Read as text, copied into the document as JSON strings; nothing is extracted or executed. | +| `_vendor/vendor.txt` of bundled wheels | Upstream projects | Each line must pin one exact version, or the build fails. | +| SBOM files given to `spdx`, `validate` | Whoever produced the file | Parsed as JSON only; `spdx` writes the one file named on the command line. | + +## Threats and controls + +| STRIDE | Threat | Control | +| ---------------------- | ---------------------------------------------------------- | ------------------------------------------------------------- | +| Tampering | A bundled wheel's metadata injects JSON into the document. | The document is serialized with `json.dumps`. | +| Tampering | A bundled wheel changes and the SBOM still lists it. | Each bundled wheel is hashed from its bytes at build time. | +| Elevation of privilege | A bundled wheel runs code during the build. | Wheels are read as zip archives; nothing in them is imported. | + +## Release chain + +Releases run on the shared [tox-dev/github-workflows](https://github.com/tox-dev/github-workflows) pipeline. A release +App, not a person, pushes the release commit and tag; the build installs its tools from a hash-checked `pylock.toml` +resolved with a seven-day upload cooldown by default; PyPI receives the files through +[trusted publishing](https://docs.pypi.org/trusted-publishers/) with attestations, and the GitHub Release is immutable. +Every workflow pins third-party actions to a commit SHA. + +## Severity + +- **High:** a bundled wheel or SBOM file that makes hatch-pep770 run code or write outside its output. +- **Medium:** an SBOM that misstates a hash, a license or the contents of a wheel. +- **Low:** a crash on a malformed bundled wheel or SBOM file instead of a clean error. + +## Accepted risks + +- The SBOM reports what the build environment holds; it does not check that environment against a lock. + +## Keeping this current + +Revisit this model when hatch-pep770 gains a new input, a new way to run code, or a new place it writes to, and whenever +an advisory is published. diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..a173db1 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,8 @@ +version: 2 +build: + os: ubuntu-lts-latest + tools: {} + commands: + - curl -LsSf https://astral.sh/uv/0.13.0/install.sh | sh + - ~/.local/bin/uv tool install tox==4.64.7 --with tox-uv==1.36.1 -p 3.15 --managed-python + - ~/.local/bin/tox run -e docs -- diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index b132379..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,3 +0,0 @@ -# Changelog - - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1095ef..9739d57 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,48 +1,5 @@ # Contributing -Add a `changelog.d/+short-name..md` fragment for each user-visible change. Use a descriptive name instead of a -pull request number. After a squash merge, the release workflow links the fragment to the pull request. A pull request -with two separate user-visible changes needs two fragments. - -Do not write credits in fragments. The release credits come from the commit authors and `Co-authored-by` trailers, and -the prepare workflow runs `release-creator contributors` to add first-time contributors to the changelog. - -Use these categories: - -| Fragment suffix | Changelog heading | Use for | -| --------------- | ----------------- | --------------------------------------------------------- | -| `feature` | Added | New capabilities and Python support additions or removals | -| `change` | Changes | Changes to existing behavior | -| `bugfix` | Fixed | User-visible fixes | -| `doc` | Documentation | User documentation changes | -| `breaking` | Breaking changes | Incompatible changes to the public interface | -| `security` | Security | Security fixes users should know about | -| `performance` | Performance | User-visible speed or resource improvements | -| `deprecation` | Deprecated | Public features scheduled for removal | -| `removal` | Removed | Removed public features other than Python support | -| `packaging` | Packaging | Distribution or installation changes | - -For example, `changelog.d/+vendored-licenses.feature.md` could contain: - -```markdown -Record the licenses of packages that bundled wheels vendor. -``` - -Internal maintenance needs no fragment. Apply the `skip news` label to a human-authored maintenance pull request; the -bot labeling workflow handles supported automation. A bot-supplied fragment is kept. - -To preview pending news locally, install the locked release tools and let the fragments select the next version: - -```bash -uv venv --python 3.15 .tox/release-tools -uv pip sync --preview-features pylock --python .tox/release-tools/bin/python --group release-news pylock.toml -version=$(git tag --list | .tox/release-tools/bin/release-creator tags --next auto --directory changelog.d) -.tox/release-tools/bin/towncrier build --draft --version "$version" -``` - -The draft shows named fragments before merge. The final changelog gains pull request links after the squash commit is on -`main`, and the prepare workflow adds the credits. - ## Tests Add a test for every new feature and bug fix. A change without a test needs a reason in the pull request description. diff --git a/README.md b/README.md index a8171e4..341f9b7 100644 --- a/README.md +++ b/README.md @@ -3,27 +3,17 @@ [![PyPI](https://img.shields.io/pypi/v/hatch-pep770?style=flat-square)](https://pypi.org/project/hatch-pep770/) [![Supported Python versions](https://img.shields.io/pypi/pyversions/hatch-pep770.svg)](https://pypi.org/project/hatch-pep770/) [![check](https://github.com/tox-dev/hatch-pep770/actions/workflows/check.yaml/badge.svg)](https://github.com/tox-dev/hatch-pep770/actions/workflows/check.yaml) +[![Documentation Status](https://readthedocs.org/projects/hatch-pep770/badge/?version=latest)](https://hatch-pep770.readthedocs.io/en/latest/?badge=latest) +[![Downloads](https://static.pepy.tech/badge/hatch-pep770/month)](https://pepy.tech/project/hatch-pep770) -A [hatchling](https://hatch.pypa.io/latest/) build hook that writes a [CycloneDX](https://cyclonedx.org/) 1.6 SBOM into -every wheel at `-.dist-info/sboms/.cdx.json`, where [PEP 770](https://peps.python.org/pep-0770/) -says installers and scanners look for it. +Write a [CycloneDX](https://cyclonedx.org/) SBOM into every wheel you build, where +[PEP 770](https://peps.python.org/pep-0770/) tells installers to keep it. The SBOM records your project, the runtime +dependencies it declares, any wheels it bundles, and the build toolchain, and a rebuild with the same inputs reproduces +it byte for byte. -General-purpose SBOM scanners read installed environments or declared dependencies. They cannot see a wheel your project -bundles as a data file, and they say nothing about how a wheel was built. The SBOM this hook writes records: +## Use -- the project, from its core metadata: version, license, maintainers, project URLs, classifiers, and the source revision - when the build runs in a git checkout; -- the runtime dependencies the wheel declares, unversioned, since the installer resolves them; -- every wheel the project bundles as a data file, with its SHA-256, license, files, and the packages it vendors; -- the build toolchain: the Python version, and every distribution in the build environment with its dependencies and, - for pure-Python ones, the hash of each file. - -It records nothing about the build machine and takes its timestamp from `SOURCE_DATE_EPOCH`, so rebuilding the same -source with the same toolchain and epoch reproduces the wheel byte for byte on any operating system or architecture. - -## Use it - -Add the hook to the build requirements and enable it: +With [hatchling](https://hatch.pypa.io/latest/), add the hook to the build requirements and enable it: ```toml [build-system] @@ -33,85 +23,11 @@ requires = ["hatch-pep770>=0.1", "hatchling>=1.28"] [tool.hatch.build.hooks.pep770] ``` -hatchling 1.28 added SBOM support to wheels. With an older hatchling the hook prints a warning and the wheel carries no -SBOM, so a project that still builds on Python 3.9 (where hatchling stops at 1.27) can keep the hook enabled. - -The hook only acts on the wheel target. Put it after any hook that sets the wheel tag, since the SBOM names the wheel -file in its attestation link. - -## Options - -All options are optional. - -| Option | Default | Meaning | -| ---------------- | -------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `supplier` | none | `{ name = "...", url = ["..."] }`, the organization that supplies the project; it becomes the SBOM's supplier and manufacturer and the project's supplier. | -| `repository` | the first `Source`, `Repository` or `GitHub` project URL | Repository URL for the source revision, license and security policy links. | -| `default-branch` | `main` | Branch used in links to files in the repository. | -| `bundled-wheels` | `[]` | Glob patterns, relative to the project root, of wheels the project ships as data files. | - -For example, a project that ships the wheels it installs into new environments: - -```toml -[tool.hatch.build.hooks.pep770] -supplier = { name = "Python Packaging Authority", url = [ - "https://www.pypa.io", -] } -bundled-wheels = ["src/virtualenv/seed/wheels/embed/*.whl"] -``` - -A bundled wheel whose project URLs point to the same forge account as `repository` gets the same `supplier`. Its -vendored packages come from `_vendor/vendor.txt` (pip's manifest, whose entries must pin one exact version) and from any -`.dist-info/METADATA` under `_vendor/`. - -Property names the hook writes start with the project name, for example `virtualenv:bundled-wheel`. - -## Add project-specific detail - -To say more about each bundled wheel, subclass the hook in a -[custom build hook](https://hatch.pypa.io/latest/plugins/build-hook/custom/) and override `bundled_wheel_properties`. -Its options then live under `[tool.hatch.build.hooks.custom]`, and the SBOM lists the script among the build tools: - -```python -# hatch_build.py -from __future__ import annotations - -from pathlib import Path - -from hatch_pep770 import cyclonedx, plugin - - -class SbomBuildHook(plugin.Pep770BuildHook): - def bundled_wheel_properties(self, wheel: Path) -> list[cyclonedx.Property]: - return [{"name": "example:installed-for-python", "value": "3.14"}] - - -def get_build_hook() -> type[SbomBuildHook]: - return SbomBuildHook -``` - -```toml -[tool.hatch.build.hooks.custom] -path = "hatch_build.py" -bundled-wheels = ["src/example/wheels/*.whl"] -``` +With another build backend, such as meson-python, embed the SBOM after the build: -The `get_build_hook` function tells hatchling which class to load, since the module also sees the base class. -`hatch_pep770.cyclonedx` holds the pieces the hook is built from (`build_tools`, `component_from_metadata`, `document`) -for describing other artifacts, such as a zipapp, the same way. - -## Command line - -```shell -hatch-pep770 spdx dist/example.cdx.json example.spdx.json -hatch-pep770 validate dist/ --allow-license MIT --allow-license Apache-2.0 +```console +hatch-pep770 embed dist/*.whl ``` -`spdx` renders an SBOM as [SPDX](https://spdx.dev/) 2.3 JSON for tools that read only SPDX. It keeps packages, hashes, -licenses, purls and the relationships, and leaves out per-file hashes (SPDX 2.3 wants a SHA-1 per file, wheel `RECORD` -files hold SHA-256) and the build formulation, which SPDX 2.3 has no field for. - -`validate` needs the `validate` extra (`pip install hatch-pep770[validate]`). For each wheel it checks that an SBOM -exists, conforms to the CycloneDX 1.6 schema, carries the `urn:uuid` serial number `actions/attest` needs to recognize -it, has unique and resolvable references, and renders as valid SPDX. With `--allow-license`, every bundled component -must also carry a license the given SPDX ids allow. +The [documentation](https://hatch-pep770.readthedocs.io) covers the options, extending the SBOM from a custom build +hook, rendering it as SPDX and validating it. diff --git a/changelog.d/+pep770-hook.feature.md b/changelog.d/+pep770-hook.feature.md deleted file mode 100644 index 708aa1e..0000000 --- a/changelog.d/+pep770-hook.feature.md +++ /dev/null @@ -1,2 +0,0 @@ -Add the `pep770` hatchling build hook, which writes a CycloneDX 1.6 SBOM into the wheel at -`.dist-info/sboms/.cdx.json`, and the `hatch-pep770 spdx` and `hatch-pep770 validate` commands. diff --git a/docs/__init__.py b/docs/__init__.py new file mode 100644 index 0000000..8e33bc6 --- /dev/null +++ b/docs/__init__.py @@ -0,0 +1 @@ +"""Sphinx configuration, importable for static analysis.""" diff --git a/docs/_static/changelog-toc.css b/docs/_static/changelog-toc.css new file mode 100644 index 0000000..add3823 --- /dev/null +++ b/docs/_static/changelog-toc.css @@ -0,0 +1,12 @@ +@media (min-width: 64em) and (max-width: 82em) { + .toc-content-icon { + position: fixed; + top: 1rem; + right: 1rem; + z-index: 10; + padding: 0.5rem; + background: var(--color-background-primary); + border: 1px solid var(--color-foreground-border); + border-radius: 0.25rem; + } +} diff --git a/docs/changelog.d/+pep770-hook.feature.rst b/docs/changelog.d/+pep770-hook.feature.rst new file mode 100644 index 0000000..d1db1f1 --- /dev/null +++ b/docs/changelog.d/+pep770-hook.feature.rst @@ -0,0 +1,3 @@ +Add the ``pep770`` hatchling build hook, which writes a CycloneDX 1.6 SBOM into each wheel, the ``hatch-pep770`` +command that embeds the same SBOM into wheels other build backends build and renders or validates it, and the Python API +behind both. diff --git a/changelog.d/.gitkeep b/docs/changelog.d/.gitkeep similarity index 100% rename from changelog.d/.gitkeep rename to docs/changelog.d/.gitkeep diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..9e3a308 --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1,4 @@ +Changelog +========= + +.. towncrier release notes start diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 0000000..d28756c --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,5 @@ +.. sphinx_argparse_cli:: + :module: hatch_pep770._cli + :func: _build_parser + :title: Command line interface + :group_title_prefix: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..c49d3ef --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,128 @@ +"""Documentation generation configuration.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from tomllib import loads +from typing import TYPE_CHECKING, Final + +import hatch_pep770 +from hatch_pep770 import __version__ + +if TYPE_CHECKING: + from docutils.nodes import Element, reference + from sphinx.addnodes import pending_xref + from sphinx.application import Sphinx + from sphinx.environment import BuildEnvironment + + +def _inheritance_aliases() -> dict[str, str]: + aliases: dict[str, str] = {} + for entry in hatch_pep770.__all__: + to_module = getattr(getattr(hatch_pep770, entry), "__module__", "") + # first instance wins + aliases.setdefault(f"{to_module}{'.' if to_module else ''}{entry}", f"{hatch_pep770.__name__}.{entry}") + return aliases + + +project: Final[str] = "hatch-pep770" +_NOW: Final[datetime] = datetime.now(tz=UTC) +globals()["copyright"] = f"2026-{_NOW.year}" +version: Final[str] = __version__ +release: Final[str] = __version__.split("+")[0] + +extensions: Final[list[str]] = [ + "sphinx.ext.autosectionlabel", + "sphinx.ext.autodoc", + "sphinx.ext.extlinks", + "sphinx.ext.intersphinx", + "sphinx.ext.inheritance_diagram", + "sphinx_argparse_cli", + "sphinx_autodoc_typehints", +] + +master_doc: Final[str] = "index" +source_suffix: Final[str] = ".rst" +exclude_patterns: Final[list[str]] = ["changelog.d/*"] +autosectionlabel_maxdepth: Final[int] = 2 + +html_theme: Final[str] = "furo" +html_title: Final[str] = "hatch-pep770" +html_last_updated_fmt: Final[str] = _NOW.isoformat() +html_static_path: Final[list[str]] = ["_static"] +html_css_files: Final[list[str]] = ["changelog-toc.css"] +pygments_style: Final[str] = "sphinx" +pygments_dark_style: Final[str] = "monokai" + +autoclass_content: Final[str] = "both" +inheritance_alias: Final[dict[str, str]] = _inheritance_aliases() +inheritance_graph_attrs: Final[dict[str, str]] = {"rankdir": "TB"} +autodoc_default_options: Final[dict[str, bool | str]] = { + "members": True, + "member-order": "bysource", + "undoc-members": True, + "show-inheritance": True, +} + +intersphinx_mapping: Final[dict[str, tuple[str, None]]] = {"python": ("https://docs.python.org/3.11", None)} +extlinks: Final[dict[str, tuple[str, str]]] = { + "pr": ("https://github.com/tox-dev/hatch-pep770/pull/%s", "#%s"), + "user": ("https://github.com/%s", "@%s"), +} +nitpicky: Final[bool] = True +nitpick_ignore: Final[list[tuple[str, str]]] = [ + # hatchling publishes no Sphinx inventory + ("py:class", "hatchling.builders.hooks.plugin.interface.BuildHookInterface"), + # the Python documentation describes the protocol without a reference target + ("py:class", "importlib.metadata._meta.PackageMetadata"), +] + + +def setup(app: Sphinx) -> None: + # without this hook Sphinx resolves pathlib.Path to an undocumented private alias + app.connect("missing-reference", _resolve_private_path) + app.connect("source-read", _include_unreleased) + + +def _resolve_private_path( + app: Sphinx, + env: BuildEnvironment, + node: pending_xref, + contnode: Element, +) -> reference | None: + if node["reftarget"] != "pathlib._local.Path": + return None + node["reftarget"] = "pathlib.Path" + domain = env.get_domain("py") + return domain.resolve_xref(env, node["refdoc"], app.builder, node["reftype"], node["reftarget"], node, contnode) + + +def _include_unreleased(app: Sphinx, docname: str, source: list[str]) -> None: + if docname != "changelog": + return + root = Path(__file__).parent.parent + fragments = root / "docs" / "changelog.d" + app.env.note_dependency(str(fragments)) + sections: list[str] = [] + for category in loads((root / "pyproject.toml").read_text(encoding="utf-8"))["tool"]["towncrier"]["type"]: + entries = [ + f"- {' '.join(fragment.read_text(encoding='utf-8').split())}" + for fragment in sorted(fragments.glob(f"*.{category['directory']}.rst")) + ] + if entries: + sections.extend(( + f".. _release-unreleased-{category['name'].lower().replace(' ', '-')}:", + "", + category["name"], + "~" * len(category["name"]), + "", + *entries, + "", + )) + if sections: + joined = "\n".join(sections) + source[0] = source[0].replace( + ".. towncrier release notes start", + f".. _release-unreleased:\n\nUnreleased\n----------\n\n{joined}\n.. towncrier release notes start", + ) diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..a71e5a9 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,28 @@ +Contributing +============ + +Add a changelog fragment under ``docs/changelog.d`` when a pull request changes what package users can use or observe. +Name it ``+short-name..rst``, for example ``+vendored-licenses.feature.rst``. The name describes the change; you +do not need a pull request number before opening the pull request. Write one change per fragment, and use several +fragments when a pull request has several user-visible changes. + +Choose the type by its effect on users: + +- ``feature`` (Added): a new capability, including adding or removing support for a Python version. +- ``change`` (Changes): a change in behavior that remains available. +- ``bugfix`` (Fixed): a correction to existing behavior. +- ``doc`` (Documentation): a documentation change users need to know about. +- ``breaking`` (Breaking changes): a change that requires users to change their usage. +- ``security`` (Security): a security fix users need to act on. +- ``performance`` (Performance): a visible speed or resource improvement. +- ``deprecation`` (Deprecated): a supported behavior scheduled for removal. +- ``removal`` (Removed): removal of a user-facing capability. +- ``packaging`` (Packaging): a change to installation or distributions. + +Apply the ``skip news`` label to a pull request with no user-visible change. You can add the label after opening the +pull request. Supplied fragments are still checked when the label is present. Bot pull requests need no fragment unless +they include one. + +The release workflow links each fragment to its merged pull request and credits its human author. If several people +authored a change, put their `GitHub `_ handles in a ``by`` credit in the fragment; omit bots and +reviewers. The changelog uses linked names, while ``GitHub`` Release notes use ``@handle``. diff --git a/docs/explanation.rst b/docs/explanation.rst new file mode 100644 index 0000000..35b97b9 --- /dev/null +++ b/docs/explanation.rst @@ -0,0 +1,45 @@ +How it works +============ + +What the SBOM records +--------------------- + +The document's root component is the project, described from the wheel's ``METADATA``: name, version, summary, +license expression, maintainers, project URLs, classifiers and keywords. Links to the PyPI release, the wheel's +attestation and, with a repository, the license file follow. A build from a git checkout adds the commit and a link to +that exact revision. + +The components below the root are the wheels the project bundles and the runtime dependencies it declares. A bundled +wheel carries its SHA-256, its license, a component per file from its ``RECORD``, and the packages it vendors. A +declared dependency carries no version, because the installer picks it at install time; the document marks the root's +dependencies incomplete for that reason. An extra's dependencies are left out. + +The formulation records how the wheel was built: the interpreter's version and implementation, every distribution of +the build environment with its dependencies, the files and hashes of each pure-Python one, and ``SOURCE_DATE_EPOCH``. +Property names start with the project's normalized name, such as ``virtualenv:bundled-wheel``, or with ``python:`` for +values copied from core metadata. + +Reproducibility +--------------- + +The document holds nothing about the build machine. Its timestamp comes from ``SOURCE_DATE_EPOCH``, or hatchling's +fixed default of 2020-02-02 when that is unset. Its serial number derives from the content, so identical documents +share one and any difference gets a new one. Platform wheels in the build environment appear without their files, +because those differ per operating system and architecture. Installer metadata and console-script launchers are left +out too, since they name the frontend and the environment's path rather than the distribution. + +So a rebuild reproduces the wheel byte for byte on any operating system when these match the original build: the +source tree, as a checkout of the same commit; ``SOURCE_DATE_EPOCH``; the Python version; and the versions of the +build backend and its dependencies. The SBOM records the last three, so it says what to install before rebuilding. + +Limitations +----------- + +- The document describes what the project declares and bundles, not the full tree of runtime dependencies, which + depends on the environment the wheel is installed into. +- The links to the PyPI release and attestation assume the project publishes to PyPI with attestations. +- A build from an sdist has no git checkout, so its SBOM names no commit. +- ``hatch-pep770 embed`` records the environment it runs in. Unless the wheel was built without isolation in that + environment, the build tool record does not match the real build. +- The SPDX rendering leaves out per-file hashes and the build record, which SPDX 2.3 cannot hold. +- A hook that changes the wheel tag after this one runs leaves the attestation link pointing at the wrong file name. diff --git a/docs/extend.rst b/docs/extend.rst new file mode 100644 index 0000000..cdff8ae --- /dev/null +++ b/docs/extend.rst @@ -0,0 +1,152 @@ +Extend it +========= + +The options cover what every project shares. When a project knows more about a bundled wheel than its metadata says, +it subclasses the hook. When it ships an artifact other than a wheel, it builds that artifact's SBOM from the same +pieces the hook uses. + +Add properties to bundled wheels +-------------------------------- + +:class:`~hatch_pep770.Pep770BuildHook` calls :meth:`~hatch_pep770.Pep770BuildHook.bundled_wheel_properties` once per +bundled wheel. Override it in a hatchling `custom build hook `_ +to add properties. They follow ``:bundled-wheel`` and come before those taken from the wheel's metadata. The +options then live in the custom hook's table, and the SBOM lists the script among the build tools, hashed, because the +script shapes the document a rebuild has to reproduce. + +Worked example: virtualenv +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`virtualenv `_ ships the pip and setuptools wheels it installs into new environments, +under ``src/virtualenv/seed/wheels/embed``. A ``BUNDLE_SUPPORT`` table in that package's ``__init__.py`` decides which +Python versions receive each wheel. Its ``pyproject.toml`` names the supplier and the wheels: + +.. code-block:: toml + + [build-system] + build-backend = 'hatchling.build' + requires = ['hatch-pep770>=0.1; python_version>='3.11'', 'hatch-vcs>=0.5', 'hatchling>=1.28'] + + [tool.hatch.build.hooks.custom] + path = 'hatch_build.py' + supplier = { name = 'Python Packaging Authority', url = ['https://www.pypa.io'] } + bundled-wheels = ['src/virtualenv/seed/wheels/embed/*.whl'] + +``hatch_build.py`` turns the table into a ``virtualenv:seeded-for-python`` property per Python version: + +.. code-block:: python + + from __future__ import annotations + + import ast + from pathlib import Path + from typing import TYPE_CHECKING + + from hatch_pep770 import Pep770BuildHook + + if TYPE_CHECKING: + from hatch_pep770 import Property + + + class SbomBuildHook(Pep770BuildHook): + def bundled_wheel_properties(self, wheel: Path) -> list[Property]: + embed = Path(self.root) / 'src' / 'virtualenv' / 'seed' / 'wheels' / 'embed' / '__init__.py' + tree = ast.parse(embed.read_text(encoding='utf-8')) + support: dict[str, dict[str, str]] = ast.literal_eval( + next(node.value for node in tree.body if isinstance(node, ast.Assign)) + ) + return [ + {'name': 'virtualenv:seeded-for-python', 'value': python} + for python, wheels in support.items() + if wheel.name in wheels.values() + ] + + + def get_build_hook() -> type[SbomBuildHook]: + return SbomBuildHook + +The ``get_build_hook`` function tells hatchling which class to load, since the module also sees the base class it +imports. The pip component of the wheel's SBOM then carries these properties, in this order: + +.. list-table:: + :header-rows: 1 + + * - Property + - Value + * - ``virtualenv:bundled-wheel`` + - ``src/virtualenv/seed/wheels/embed/pip-26.1-py3-none-any.whl`` + * - ``virtualenv:seeded-for-python`` + - ``3.14`` + * - ``virtualenv:seeded-for-python`` + - ``3.15`` + * - ``python:requires-python`` + - ``>=3.10`` + +A project that still builds on a Python older than hatch-pep770 supports keeps the requirement's environment marker +and makes the script fall back to hatchling's plain hook there, so those builds produce a wheel without an SBOM: + +.. code-block:: python + + import sys + + if sys.version_info >= (3, 11): + from hatch_pep770 import Pep770BuildHook as BaseHook + else: # hatch-pep770 needs Python 3.11; older builds get no SBOM + from hatchling.builders.hooks.plugin.interface import BuildHookInterface as BaseHook + +Describe another artifact +------------------------- + +virtualenv also publishes a zipapp. Its SBOM describes what the archive holds and records the same build toolchain as +the wheel's. The functions the hook uses are public for that: + +- :func:`~hatch_pep770.component_from_metadata` turns a distribution's ``METADATA`` into a component, with licenses, + project URLs and metadata properties. +- :func:`~hatch_pep770.build_tools` records the interpreter and every distribution of the running environment, with + the files of the pure-Python ones and the dependencies between them. +- :func:`~hatch_pep770.timestamp` gives the time from ``SOURCE_DATE_EPOCH``. +- :func:`~hatch_pep770.document` adds the CycloneDX header, with a serial number derived from the content. + +.. code-block:: python + + import uuid + + from hatch_pep770 import Body, Component, build_tools, document, timestamp + + tools, tool_dependencies = build_tools() + zipapp: Component = {'type': 'application', 'bom-ref': 'pkg:generic/virtualenv.pyz@21.0', 'name': 'virtualenv.pyz'} + body: Body = { + 'metadata': { + 'timestamp': timestamp(), + 'lifecycles': [{'phase': 'build'}], + 'tools': {'components': tools}, + 'component': zipapp, + }, + 'components': [], + 'dependencies': [{'ref': zipapp['bom-ref'], 'dependsOn': []}, *tool_dependencies], + 'formulation': [], + } + sbom = document(body, uuid.uuid5(uuid.NAMESPACE_URL, 'https://github.com/pypa/virtualenv/sboms')) + +:func:`~hatch_pep770.validate` checks the result the way ``hatch-pep770 validate`` checks a wheel's SBOM. + +Use it as a library +------------------- + +:class:`~hatch_pep770.Project` holds the options and writes the SBOM of a wheel from its ``METADATA``, and +:func:`~hatch_pep770.embed` rewrites a built wheel with it. Pass ``bundled_wheel_properties`` to add properties to +bundled wheels without a hatchling hook: + +.. code-block:: python + + from pathlib import Path + + from hatch_pep770 import Project, embed + + project = Project.from_table( + {'bundled-wheels': ['src/project/wheels/*.whl']}, + root=Path.cwd(), + where='my build script', + bundled_wheel_properties=lambda wheel: [{'name': 'project:role', 'value': 'seed'}], + ) + embed(Path('dist/project-1.0-py3-none-any.whl'), project) diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..750ce82 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,43 @@ +hatch-pep770 +============ + +hatch-pep770 writes a `CycloneDX `_ 1.6 SBOM into every wheel you build, at +``-.dist-info/sboms/.cdx.json``. That is where `PEP 770 `_ +tells installers to keep SBOMs, so they land in the environment next to the installed package, where scanners look for +them. + +Why a wheel needs its own SBOM +------------------------------ + +General-purpose scanners read the dependencies a package declares or the packages an environment has installed. They +cannot see a wheel your project ships as a data file, and they cannot say how the wheel was built. The SBOM this package +writes records both: the wheels you bundle, with their hashes, licenses and vendored packages, and the Python version +and packages of the build environment. A third party can use that record to rebuild the wheel and compare the bytes. + +Quick start +----------- + +For a project built with `hatchling `_, add the hook to the build requirements and +enable it in ``pyproject.toml``: + +.. code-block:: toml + + [build-system] + build-backend = 'hatchling.build' + requires = ['hatch-pep770>=0.1', 'hatchling>=1.28'] + + [tool.hatch.build.hooks.pep770] + +Every wheel you build now carries the SBOM. For a project built with another backend, such as meson-python, run +``hatch-pep770 embed dist/*.whl`` after the build; :doc:`usage` shows both. + +.. toctree:: + :maxdepth: 2 + + usage + extend + explanation + cli + py_api + changelog + contributing diff --git a/docs/py_api.rst b/docs/py_api.rst new file mode 100644 index 0000000..631dbef --- /dev/null +++ b/docs/py_api.rst @@ -0,0 +1,80 @@ +Python API +========== + +Everything below is importable from ``hatch_pep770``. + +.. autodata:: hatch_pep770.__version__ + +Build hook +---------- + +.. autoclass:: hatch_pep770.Pep770BuildHook + :members: bundled_wheel_properties + +Wheels from any build backend +----------------------------- + +.. autoclass:: hatch_pep770.Project + :members: from_table, describe + :exclude-members: root, supplier, repository, default_branch, bundled_wheels, build_script, bundled_wheel_properties + +.. autoclass:: hatch_pep770.BundledWheelProperties + :members: __call__ + +.. autofunction:: hatch_pep770.embed + +Building blocks +--------------- + +.. autofunction:: hatch_pep770.component_from_metadata + +.. autofunction:: hatch_pep770.build_tools + +.. autofunction:: hatch_pep770.timestamp + +.. autofunction:: hatch_pep770.document + +SPDX and validation +------------------- + +.. autofunction:: hatch_pep770.to_spdx + +.. autofunction:: hatch_pep770.validate + +.. autofunction:: hatch_pep770.validate_wheel + +Types +----- + +These are :class:`~typing.TypedDict` types of the JSON documents, for type checking code that builds or reads them. + +.. autoclass:: hatch_pep770.Document + :no-members: + +.. autoclass:: hatch_pep770.Body + :no-members: + +.. autoclass:: hatch_pep770.Component + :no-members: + +.. autoclass:: hatch_pep770.Dependency + :no-members: + +.. autoclass:: hatch_pep770.OrganizationalEntity + :no-members: + +.. autoclass:: hatch_pep770.Property + :no-members: + +.. autoclass:: hatch_pep770.Spdx + :no-members: + +.. autodata:: hatch_pep770.Toml + +Command line +------------ + +.. autofunction:: hatch_pep770.main + +.. autoclass:: hatch_pep770.Options + :members: diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000..c1bb4d3 --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,117 @@ +Use it +====== + +With hatchling +-------------- + +Add the hook to ``build-system.requires`` and enable it with an empty table, as the quick start on the :doc:`index` +shows. hatchling 1.28 added SBOM support to wheels, so the hook requires it. The hook acts on wheel builds only; an +sdist carries no SBOM. + +The SBOM links to the wheel's PyPI attestation, so it needs the wheel's tag. List the hook after any hook that changes +the tag. + +Options +~~~~~~~ + +Each option lives in the hook's table and is optional. + +``supplier`` + The organization that supplies the project: a table with a ``name`` string and an optional ``url`` string array. It + becomes the supplier and manufacturer of the document and the supplier of the project. Left out, the document names + no supplier. + + .. code-block:: toml + + [tool.hatch.build.hooks.pep770] + supplier = { name = 'Python Packaging Authority', url = ['https://www.pypa.io'] } + +``repository`` + The repository URL, used for the link to the source revision, the license file and, on GitHub, the security policy + and advisories. It defaults to the first ``Source``, ``Repository`` or ``GitHub`` URL in ``[project.urls]``. + + .. code-block:: toml + + [tool.hatch.build.hooks.pep770] + repository = 'https://gitlab.com/example/project' + +``default-branch`` + The branch in links to files of the repository, ``main`` by default. + + .. code-block:: toml + + [tool.hatch.build.hooks.pep770] + default-branch = 'trunk' + +``bundled-wheels`` + Glob patterns, relative to the project root, of wheels the project ships as data files. Each matching wheel becomes + a component with its SHA-256, license, files and the packages it vendors. + + .. code-block:: toml + + [tool.hatch.build.hooks.pep770] + bundled-wheels = ['src/project/wheels/*.whl'] + +A bundled wheel whose project URLs point to the same forge account as ``repository`` gets the same ``supplier``. Its +vendored packages come from ``_vendor/vendor.txt``, pip's manifest, whose lines must each pin one exact version, and +from any ``.dist-info/METADATA`` under ``_vendor/``. + +With another build backend +-------------------------- + +A backend other than hatchling, such as `meson-python `_, cannot run the hook. +``hatch-pep770 embed`` writes the same document into a wheel that backend built, adds the ``RECORD`` row, and keeps +every other entry of the archive as it was. It reads the options above from a ``[tool.hatch-pep770]`` table: + +.. code-block:: toml + + [tool.hatch-pep770] + supplier = { name = 'tox-dev', url = ['https://github.com/tox-dev'] } + +Build, then embed, from the project root: + +.. code-block:: console + + $ python -m build --wheel + $ hatch-pep770 embed dist/*.whl + wrote turbohtml-1.0.dist-info/sboms/turbohtml.cdx.json into turbohtml-1.0-cp314-cp314-linux_x86_64.whl + +The build tool record describes the environment ``embed`` runs in, not the isolated environment the backend built in. +To record the build toolchain, build without isolation in the same locked environment: + +.. code-block:: console + + $ python -m pip install hatch-pep770 meson-python meson ninja + $ python -m build --wheel --no-isolation + $ hatch-pep770 embed dist/*.whl + +Render it as SPDX +----------------- + +Some tools read only `SPDX `_. ``hatch-pep770 spdx`` renders the CycloneDX document as SPDX 2.3 +JSON: + +.. code-block:: console + + $ unzip -p dist/project-1.0-py3-none-any.whl '*/sboms/project.cdx.json' > project.cdx.json + $ hatch-pep770 spdx project.cdx.json project.spdx.json + +The SPDX document keeps the packages, their hashes, licenses and purls, and the containment, dependency and build tool +relationships. It leaves out the per-file hashes, because SPDX 2.3 wants a SHA-1 for every file and wheel ``RECORD`` +files hold SHA-256, and the build record, which SPDX 2.3 has no field for. Its namespace ends in the CycloneDX serial +number, so both documents name the same build. + +Check it in CI +-------------- + +``hatch-pep770 validate`` needs the ``validate`` extra. It checks each wheel's SBOM against the CycloneDX 1.6 schema, +requires the ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, checks that every reference +resolves, and validates the SPDX rendering: + +.. code-block:: console + + $ python -m pip install 'hatch-pep770[validate]' + $ hatch-pep770 validate dist/ --allow-license MIT --allow-license Apache-2.0 + +Each ``--allow-license`` adds an SPDX id that bundled wheels may use. With any given, a bundled wheel whose license the +ids do not allow fails the check, and so does one that declares a license only by name. From 6092cc249310f7ee3ec3c172fbd19240a0025a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 19:54:34 -0800 Subject: [PATCH 007/115] =?UTF-8?q?=F0=9F=94=A7=20build:=20adopt=20the=20b?= =?UTF-8?q?ump-deps-index=20tooling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/check.yaml | 42 +- .github/workflows/codeql.yaml | 2 +- .github/workflows/news.yaml | 6 +- .github/workflows/prepare-release.yaml | 7 +- .github/workflows/release.yaml | 12 +- .github/workflows/upgrade-release-tools.yaml | 17 + .mdformat.toml | 2 - .pre-commit-config.yaml | 36 +- pylock.toml | 1245 +++++++++++++----- pyproject.toml | 115 +- tox.toml | 95 +- 11 files changed, 1140 insertions(+), 439 deletions(-) create mode 100644 .github/workflows/upgrade-release-tools.yaml delete mode 100644 .mdformat.toml diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 55ebfc3..ada78bd 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -3,44 +3,59 @@ on: workflow_dispatch: push: branches: ["main"] - tags-ignore: ["**"] pull_request: types: [opened, synchronize, reopened] schedule: - cron: "27 8 * * *" concurrency: - group: check-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true permissions: contents: read jobs: test: - name: ๐Ÿงช Test + name: ๐Ÿงช test + if: github.event_name != 'schedule' || github.repository_owner == 'gaborbernat' uses: tox-dev/github-workflows/.github/workflows/check.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 permissions: contents: read with: include: | [ - {"env": "3.15"}, {"env": "3.15t", "python": "3.15t"}, + {"env": "3.15", "python": "3.15"}, + {"env": "3.14t", "python": "3.14t"}, {"env": "3.14", "python": "3.14"}, {"env": "3.13", "python": "3.13"}, {"env": "3.12", "python": "3.12"}, {"env": "3.11", "python": "3.11"}, - {"env": "3.10", "python": "3.10"}, - {"env": "3.9", "python": "3.9"}, - {"env": "type"}, + {"env": "3.15t", "python": "3.15t", "os": "macos-26"}, + {"env": "3.15", "python": "3.15", "os": "macos-26"}, + {"env": "3.14t", "python": "3.14t", "os": "macos-26"}, + {"env": "3.14", "python": "3.14", "os": "macos-26"}, + {"env": "3.13", "python": "3.13", "os": "macos-26"}, + {"env": "3.12", "python": "3.12", "os": "macos-26"}, + {"env": "3.11", "python": "3.11", "os": "macos-26"}, + {"env": "3.15t", "python": "3.15t", "os": "windows-2025"}, + {"env": "3.15", "python": "3.15", "os": "windows-2025"}, + {"env": "3.14t", "python": "3.14t", "os": "windows-2025"}, + {"env": "3.14", "python": "3.14", "os": "windows-2025"}, + {"env": "3.13", "python": "3.13", "os": "windows-2025"}, + {"env": "3.12", "python": "3.12", "os": "windows-2025"}, + {"env": "3.11", "python": "3.11", "os": "windows-2025"}, {"env": "dev"}, + {"env": "dev", "os": "windows-2025"}, + {"env": "docs"}, + {"env": "docs", "os": "windows-2025"}, + {"env": "fix"}, + {"env": "type"}, + {"env": "type", "os": "windows-2025"}, {"env": "pkg_meta"}, - {"env": "3.15", "os": "windows-2025-vs2026"}, - {"env": "3.14", "python": "3.14", "os": "windows-2025-vs2026"}, - {"env": "3.9", "python": "3.9", "os": "windows-2025-vs2026"}, - {"env": "3.15", "os": "macos-26"} + {"env": "pkg_meta", "os": "windows-2025"} ] release: name: ๐Ÿท๏ธ Release dry run - uses: ./.github/workflows/prepare-release.yaml # zizmor: ignore[self-repository] actionlint rejects the $/ form + uses: $/.github/workflows/prepare-release.yaml permissions: contents: read pull-requests: read @@ -56,6 +71,7 @@ jobs: if: ${{ !cancelled() }} # codespell:ignore cancelled needs: [test, release, hooks] runs-on: ubuntu-26.04 + permissions: {} steps: - name: ๐Ÿšฆ Require all needed jobs to pass uses: tox-dev/github-workflows/required-checks@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index de17f24..cec793a 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -7,7 +7,7 @@ on: schedule: - cron: "35 8 * * 2" concurrency: - group: codeql-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: {} jobs: diff --git a/.github/workflows/news.yaml b/.github/workflows/news.yaml index c707bc0..d5a8c28 100644 --- a/.github/workflows/news.yaml +++ b/.github/workflows/news.yaml @@ -1,4 +1,4 @@ -name: ๐Ÿ“ Changelog +name: Changelog on: pull_request: types: [opened, reopened, synchronize, labeled, unlabeled] @@ -10,7 +10,7 @@ jobs: name: ๐Ÿ“ News fragment runs-on: ubuntu-26.04 steps: - - name: ๐Ÿ“ฅ Check out code + - name: ๐Ÿ“ฅ Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 @@ -26,5 +26,5 @@ jobs: with: groups: release-news venv: ${{ runner.temp }}/release-tools - - name: ๐Ÿ” Validate news fragments + - name: ๐Ÿ” Validate changelog fragments uses: tox-dev/github-workflows/validate-news@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 diff --git a/.github/workflows/prepare-release.yaml b/.github/workflows/prepare-release.yaml index 3daaefb..c0d478c 100644 --- a/.github/workflows/prepare-release.yaml +++ b/.github/workflows/prepare-release.yaml @@ -1,4 +1,4 @@ -name: ๐Ÿท๏ธ Prepare release +name: Prepare release on: workflow_dispatch: inputs: @@ -26,8 +26,9 @@ jobs: permissions: contents: read pull-requests: read - secrets: - RELEASE_APP_PRIVATE_KEY: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} with: bump: ${{ inputs.bump || 'auto' }} dry-run: ${{ inputs.dry-run == true }} + release-auth-deployment: false + secrets: + RELEASE_APP_PRIVATE_KEY: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index df5858b..63e208b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,4 +1,4 @@ -name: ๐Ÿš€ Release to PyPI +name: Release to PyPI on: push: tags: ["*"] @@ -20,9 +20,9 @@ jobs: contents: read with: tag: ${{ github.ref_name }} - changelog: CHANGELOG.md + changelog: docs/changelog.rst artifact-url: https://pypi.org/project/hatch-pep770/${{ github.ref_name }}/ - publish-pypi: + publish: name: ๐Ÿš€ Publish to PyPI needs: [build, notes] runs-on: ubuntu-26.04 @@ -36,15 +36,15 @@ jobs: uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: artifact-ids: ${{ needs.build.outputs.distributions-artifact-id }} - path: dist + path: dist/ - name: ๐Ÿš€ Publish to PyPI uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 with: attestations: true packages-dir: dist - github-release: + release: name: ๐Ÿš€ Publish GitHub Release - needs: [build, notes, publish-pypi] + needs: [build, publish, notes] uses: tox-dev/github-workflows/.github/workflows/github-release.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 permissions: contents: write diff --git a/.github/workflows/upgrade-release-tools.yaml b/.github/workflows/upgrade-release-tools.yaml new file mode 100644 index 0000000..a4aabe6 --- /dev/null +++ b/.github/workflows/upgrade-release-tools.yaml @@ -0,0 +1,17 @@ +name: Upgrade release tools +on: + workflow_dispatch: + schedule: + - cron: "51 8 * * 2" +permissions: {} +concurrency: + group: upgrade-release-tools + cancel-in-progress: false +jobs: + upgrade: + name: ๐Ÿ”€ Upgrade release tools + uses: tox-dev/github-workflows/.github/workflows/upgrade-release-tools.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + permissions: + contents: read + secrets: + RELEASE_APP_PRIVATE_KEY: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} diff --git a/.mdformat.toml b/.mdformat.toml deleted file mode 100644 index a571de0..0000000 --- a/.mdformat.toml +++ /dev/null @@ -1,2 +0,0 @@ -wrap = 120 -number = true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a029eb..cecb70b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,15 +13,14 @@ repos: hooks: - id: check-github-workflows args: ["--verbose"] - - repo: https://github.com/rhysd/actionlint - rev: 914e7df21a07ef503a81201c76d2b11c789d3fca # frozen: v1.7.12 - hooks: - - id: actionlint - args: ["-ignore", 'label "ubuntu-26[.]04" is unknown'] # GitHub offers this runner before actionlint lists it. - repo: https://github.com/codespell-project/codespell rev: 57b21406f092110c18776e39b0bda50d37c945c8 # frozen: v2.4.3 hooks: - id: codespell + - repo: https://github.com/tox-dev/tox-toml-fmt + rev: "5125f601c3e134347288c3d42f3b34298b7cf1be" # frozen: v1.11.1 + hooks: + - id: tox-toml-fmt - repo: https://github.com/tox-dev/pyproject-fmt rev: "50e0f3aee9dc5747a5f173f79a7417403c18dda6" # frozen: v2.30.3 hooks: @@ -32,22 +31,35 @@ repos: - id: ruff-format - id: ruff-check args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] - - repo: https://github.com/hukkin/mdformat - rev: "82912cdaea4fb830f751504486a7879c70526547" # frozen: 1.0.0 + - repo: https://github.com/gaborbernat/simplint-pre-commit + rev: c8719dca99c13381570bc69f8955c9448fc271b7 # frozen: v0.7.1 hooks: - - id: mdformat - additional_dependencies: - - mdformat-config==0.2.1 - - mdformat-gfm==1.0.0 - - mdformat-toc==0.5.0 + - id: simplint - repo: https://github.com/google/yamlfmt rev: "21ca5323a9c87ee37a434e0ca908efc0a89daa07" # frozen: v0.21.0 hooks: - id: yamlfmt + - repo: https://github.com/executablebooks/mdformat + rev: 82912cdaea4fb830f751504486a7879c70526547 # frozen: 1.0.0 + hooks: + - id: mdformat + args: ["--wrap", "120"] + additional_dependencies: + - mdformat-gfm==1.0.0 + - mdformat-ruff==0.1.3 + - repo: https://github.com/LilSpazJoekp/docstrfmt + rev: c1813841673cdcd6cf602dde1edb78a5d5e45235 # frozen: v2.2.1 + hooks: + - id: docstrfmt + files: ^docs/(changelog|contributing)[.]rst$ + args: ["-l", "120", "--preserve-adornments"] + additional_dependencies: + - "sphinx==9.2.0" - repo: https://github.com/zizmorcore/zizmor-pre-commit rev: c92afdf2c6ca336aaa8f39e2bd8d48f62dbd8e1f # frozen: v1.31.1 hooks: - id: zizmor + args: ["--min-severity", "high"] - repo: meta hooks: - id: check-hooks-apply diff --git a/pylock.toml b/pylock.toml index 64ef8ba..a7f3b90 100644 --- a/pylock.toml +++ b/pylock.toml @@ -1,18 +1,18 @@ lock-version = "1.0" environments = [ "python_version == \"3.15\" and sys_platform == \"linux\" and platform_machine == \"x86_64\" and implementation_name == \"cpython\" and platform_python_implementation == \"CPython\" and platform_system == \"Linux\"", - "python_version == \"3.15\" and sys_platform == \"darwin\" and platform_machine == \"arm64\" and platform_python_implementation == \"CPython\" and platform_system == \"Darwin\"", - "python_version == \"3.15\" and sys_platform == \"win32\" and platform_machine == \"AMD64\" and platform_python_implementation == \"CPython\" and platform_system == \"Windows\"", ] -requires-python = ">=3.9" +requires-python = ">=3.11" extras = [ "validate", ] dependency-groups = [ "build", + "dev", + "docs", + "lint", "pkg-meta", "project-runtime", - "release", "release-news", "release-notes", "release-prepare", @@ -24,10 +24,63 @@ default-groups = [ ] created-by = "nab" +[[packages]] +name = "accessible-pygments" +version = "0.0.5" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +dependencies = [ + { name = "pygments" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "accessible_pygments-0.0.5.tar.gz" +upload-time = 2024-05-10 11:23:10.216775+00:00 +url = "https://files.pythonhosted.org/packages/bc/c1/bbac6a50d02774f91572938964c582fff4270eee73ab822a4aeea4d8b11b/accessible_pygments-0.0.5.tar.gz" +size = 1377899 + +[packages.sdist.hashes] +sha256 = "40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872" + +[[packages.wheels]] +name = "accessible_pygments-0.0.5-py3-none-any.whl" +upload-time = 2024-05-10 11:23:08.421378+00:00 +url = "https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl" +size = 1395903 + +[packages.wheels.hashes] +sha256 = "88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7" + +[[packages]] +name = "alabaster" +version = "1.0.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "alabaster-1.0.0.tar.gz" +upload-time = 2024-07-26 18:15:03.762569+00:00 +url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz" +size = 24210 + +[packages.sdist.hashes] +sha256 = "c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e" + +[[packages.wheels]] +name = "alabaster-1.0.0-py3-none-any.whl" +upload-time = 2024-07-26 18:15:02.050654+00:00 +url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl" +size = 13929 + +[packages.wheels.hashes] +sha256 = "fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b" + [[packages]] name = "annotated-types" version = "0.8.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -52,7 +105,7 @@ sha256 = "f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0" [[packages]] name = "arrow" version = "1.4.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" dependencies = [ { name = "python-dateutil" }, @@ -81,7 +134,7 @@ sha256 = "749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205" [[packages]] name = "attrs" version = "26.1.0" -marker = "\"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -103,10 +156,35 @@ size = 67548 [packages.wheels.hashes] sha256 = "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" +[[packages]] +name = "babel" +version = "2.18.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.8" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "babel-2.18.0.tar.gz" +upload-time = 2026-02-01 12:30:56.078194+00:00 +url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz" +size = 9959554 + +[packages.sdist.hashes] +sha256 = "b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d" + +[[packages.wheels]] +name = "babel-2.18.0-py3-none-any.whl" +upload-time = 2026-02-01 12:30:53.445172+00:00 +url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl" +size = 10196845 + +[packages.wheels.hashes] +sha256 = "e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35" + [[packages]] name = "beartype" version = "0.23.1" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -128,10 +206,39 @@ size = 1582101 [packages.wheels.hashes] sha256 = "4461b4dc57e3fdd6c8a8464b22cf118002eea107c33e9fef651c69068ab3cce3" +[[packages]] +name = "beautifulsoup4" +version = "4.15.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.7.0" +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "beautifulsoup4-4.15.0.tar.gz" +upload-time = 2026-06-07 16:44:20.453113+00:00 +url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz" +size = 632571 + +[packages.sdist.hashes] +sha256 = "288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7" + +[[packages.wheels]] +name = "beautifulsoup4-4.15.0-py3-none-any.whl" +upload-time = 2026-06-07 16:44:21.566528+00:00 +url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl" +size = 109924 + +[packages.wheels.hashes] +sha256 = "d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9" + [[packages]] name = "boolean-py" version = "5.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" index = "https://pypi.org/simple/" [packages.sdist] @@ -155,7 +262,7 @@ sha256 = "ef28a70bd43115208441b53a045d1549e2f0ec6e3d08a9d142cbc41c1938e8d9" [[packages]] name = "certifi" version = "2026.10.23" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.7" index = "https://pypi.org/simple/" @@ -180,7 +287,7 @@ sha256 = "e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1" [[packages]] name = "cffi" version = "2.1.1" -marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "pycparser" }, @@ -205,10 +312,35 @@ size = 221630 [packages.wheels.hashes] sha256 = "aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399" +[[packages]] +name = "cfgv" +version = "3.5.0" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "cfgv-3.5.0.tar.gz" +upload-time = 2025-11-19 20:55:51.612035+00:00 +url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz" +size = 7334 + +[packages.sdist.hashes] +sha256 = "d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132" + +[[packages.wheels]] +name = "cfgv-3.5.0-py2.py3-none-any.whl" +upload-time = 2025-11-19 20:55:50.744331+00:00 +url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl" +size = 7445 + +[packages.wheels.hashes] +sha256 = "a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0" + [[packages]] name = "charset-normalizer" version = "3.5.2" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.7" index = "https://pypi.org/simple/" @@ -221,15 +353,6 @@ size = 172659 [packages.sdist.hashes] sha256 = "39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef" -[[packages.wheels]] -name = "charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl" -upload-time = 2026-09-30 04:37:28.329851+00:00 -url = "https://files.pythonhosted.org/packages/ca/e9/34e597dee616d0b8ee4b34d29399e85c2204ade174157a48505d42baa4ff/charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl" -size = 367676 - -[packages.wheels.hashes] -sha256 = "87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26" - [[packages.wheels]] name = "charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" upload-time = 2026-09-30 04:37:37.397915+00:00 @@ -239,24 +362,6 @@ size = 257336 [packages.wheels.hashes] sha256 = "3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229" -[[packages.wheels]] -name = "charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl" -upload-time = 2026-09-30 04:37:53.427053+00:00 -url = "https://files.pythonhosted.org/packages/98/e8/55d5fd3935b4bce6da4fe0df61898e8c82653e317e677bd58aceb9c60f13/charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl" -size = 210858 - -[packages.wheels.hashes] -sha256 = "89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8" - -[[packages.wheels]] -name = "charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl" -upload-time = 2026-09-30 04:38:26.216534+00:00 -url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl" -size = 331815 - -[packages.wheels.hashes] -sha256 = "4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd" - [[packages.wheels]] name = "charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl" upload-time = 2026-09-30 04:38:28.032254+00:00 @@ -266,15 +371,6 @@ size = 253276 [packages.wheels.hashes] sha256 = "1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7" -[[packages.wheels]] -name = "charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl" -upload-time = 2026-09-30 04:38:49.707190+00:00 -url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl" -size = 200092 - -[packages.wheels.hashes] -sha256 = "9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036" - [[packages.wheels]] name = "charset_normalizer-3.5.2-py3-none-any.whl" upload-time = 2026-09-30 04:39:21.828049+00:00 @@ -287,7 +383,7 @@ sha256 = "b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685" [[packages]] name = "check-wheel-contents" version = "0.6.3" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "attrs" }, @@ -319,7 +415,7 @@ sha256 = "5ae39c8c434b972f0740d04610759168590713175aab584b012b1b84f6771874" [[packages]] name = "click" version = "8.5.0" -marker = "\"pkg-meta\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -341,35 +437,10 @@ size = 125251 [packages.wheels.hashes] sha256 = "255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360" -[[packages]] -name = "colorama" -version = "0.4.6" -marker = "(\"test\" in dependency_groups or \"type\" in dependency_groups) and sys_platform == \"win32\"" -requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "colorama-0.4.6.tar.gz" -upload-time = 2022-10-25 02:36:22.414799+00:00 -url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz" -size = 27697 - -[packages.sdist.hashes] -sha256 = "08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44" - -[[packages.wheels]] -name = "colorama-0.4.6-py2.py3-none-any.whl" -upload-time = 2022-10-25 02:36:20.889702+00:00 -url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl" -size = 25335 - -[packages.wheels.hashes] -sha256 = "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" - [[packages]] name = "covdefaults" version = "2.3.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.7" dependencies = [ { name = "coverage" }, @@ -397,7 +468,7 @@ sha256 = "2832961f6ffcfe4b57c338bc3418a3526f495c26fb9c54565409c5532f7c41be" [[packages]] name = "coverage" version = "7.17.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.11" index = "https://pypi.org/simple/" @@ -410,15 +481,6 @@ size = 991021 [packages.sdist.hashes] sha256 = "cc010a0a7fbdd9bb8bdfd0d3f04a5151275860213cbd7e2b876b29778489c712" -[[packages.wheels]] -name = "coverage-7.17.0-cp315-cp315-macosx_11_0_arm64.whl" -upload-time = 2026-10-22 16:06:06.167385+00:00 -url = "https://files.pythonhosted.org/packages/87/db/94697b9ccf4ec94cdad07bf94e2a21b9702247cb212d2d5f18d2b9c3d0de/coverage-7.17.0-cp315-cp315-macosx_11_0_arm64.whl" -size = 226193 - -[packages.wheels.hashes] -sha256 = "66efc0fb99cee34626fc17d3adac9c3cd816bfdad563bb1e983c3ef432f566d4" - [[packages.wheels]] name = "coverage-7.17.0-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl" upload-time = 2026-10-22 16:06:08.861264+00:00 @@ -428,15 +490,6 @@ size = 261735 [packages.wheels.hashes] sha256 = "aed06129a9f530a948928260d7d2d81c47261355b0127e3c782251b38958463c" -[[packages.wheels]] -name = "coverage-7.17.0-cp315-cp315-win_amd64.whl" -upload-time = 2026-10-22 16:06:23.756207+00:00 -url = "https://files.pythonhosted.org/packages/43/14/283120142ac281834808584249b89957de997c28573ddd65ae106b622121/coverage-7.17.0-cp315-cp315-win_amd64.whl" -size = 228532 - -[packages.wheels.hashes] -sha256 = "79cf71e5f9a48877ccfed5baa2c72f45af4feeaa596249e23eb15b473134830e" - [[packages.wheels]] name = "coverage-7.17.0-py3-none-any.whl" upload-time = 2026-10-22 16:06:48.471847+00:00 @@ -449,7 +502,7 @@ sha256 = "bac487dce0d2aff28ddf1a533a916f771416eee0e493960f611e49037b878e9d" [[packages]] name = "cryptography" version = "50.0.2" -marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = "!=3.9.0,!=3.9.1,>=3.9" dependencies = [ { name = "cffi" }, @@ -549,7 +602,7 @@ sha256 = "1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c" [[packages]] name = "cyclonedx-python-lib" version = "11.12.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "<4.0,>=3.9" dependencies = [ { name = "jsonschema" }, @@ -582,7 +635,7 @@ sha256 = "0e807521a921a5c3cb8ce1153f8a61d29eedfe76a46aac2796b7c6b573391a54" [[packages]] name = "defusedxml" version = "0.7.1" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" index = "https://pypi.org/simple/" @@ -604,10 +657,34 @@ size = 25604 [packages.wheels.hashes] sha256 = "a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" +[[packages]] +name = "distlib" +version = "0.4.4" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "distlib-0.4.4.tar.gz" +upload-time = 2026-10-16 13:56:34.203008+00:00 +url = "https://files.pythonhosted.org/packages/68/bc/d36966de7304047a1001a7559fbf266c464bb5d8cc85464536ef8c9bcc32/distlib-0.4.4.tar.gz" +size = 621304 + +[packages.sdist.hashes] +sha256 = "ca4175b4ac7f71223cc30ed7248c45d2b38c2439d8ee6e498ab67ed63cf3508d" + +[[packages.wheels]] +name = "distlib-0.4.4-py2.py3-none-any.whl" +upload-time = 2026-10-16 13:56:32.053304+00:00 +url = "https://files.pythonhosted.org/packages/c1/2b/238f02a4927fa2e27aa995371d240a468b52e42eae44a678434aff736b38/distlib-0.4.4-py2.py3-none-any.whl" +size = 472099 + +[packages.wheels.hashes] +sha256 = "cf09d55b1e85a89576b036aeb71550bfd68320f3f21761ca329eadadad918638" + [[packages]] name = "docutils" version = "0.23" -marker = "\"pkg-meta\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -629,10 +706,35 @@ size = 634701 [packages.wheels.hashes] sha256 = "25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea" +[[packages]] +name = "filelock" +version = "4.2.5" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +requires-python = ">=3.11" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "filelock-4.2.5.tar.gz" +upload-time = 2026-10-30 19:27:22.237276+00:00 +url = "https://files.pythonhosted.org/packages/aa/4b/2320d9fa03b65b4beb43b521fe1b12b7f87ab91ee42de3a286075016d205/filelock-4.2.5.tar.gz" +size = 659943 + +[packages.sdist.hashes] +sha256 = "37d8fa4f8b5e9233612e68ab23bf64cae41771322c8797361088ae2895b12894" + +[[packages.wheels]] +name = "filelock-4.2.5-py3-none-any.whl" +upload-time = 2026-10-30 19:27:20.409828+00:00 +url = "https://files.pythonhosted.org/packages/14/2d/2cf36eca330b7a97c0e9076e8fc4272127c29398fa2bb7b24d4321b8b476/filelock-4.2.5-py3-none-any.whl" +size = 163877 + +[packages.wheels.hashes] +sha256 = "46ce42d27e5cdde09ee7197349663e97cb106679bd9e75db3a09679fbc7aa1a8" + [[packages]] name = "fqdn" version = "1.6.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<4,>=2.7" index = "https://pypi.org/simple/" @@ -654,6 +756,66 @@ size = 12569 [packages.wheels.hashes] sha256 = "6c793c312ccfa29981e1135b3ef5e1277ebe639731a6b0f7350d90e96e63f0b4" +[[packages]] +name = "furo" +version = "2025.12.19" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.8" +dependencies = [ + { name = "accessible-pygments" }, + { name = "beautifulsoup4" }, + { name = "pygments" }, + { name = "sphinx" }, + { name = "sphinx-basic-ng" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "furo-2025.12.19.tar.gz" +upload-time = 2025-12-19 17:34:40.889109+00:00 +url = "https://files.pythonhosted.org/packages/ec/20/5f5ad4da6a5a27c80f2ed2ee9aee3f9e36c66e56e21c00fde467b2f8f88f/furo-2025.12.19.tar.gz" +size = 1661473 + +[packages.sdist.hashes] +sha256 = "188d1f942037d8b37cd3985b955839fea62baa1730087dc29d157677c857e2a7" + +[[packages.wheels]] +name = "furo-2025.12.19-py3-none-any.whl" +upload-time = 2025-12-19 17:34:38.905692+00:00 +url = "https://files.pythonhosted.org/packages/f4/b2/50e9b292b5cac13e9e81272c7171301abc753a60460d21505b606e15cf21/furo-2025.12.19-py3-none-any.whl" +size = 339262 + +[packages.wheels.hashes] +sha256 = "bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f" + +[[packages]] +name = "hatch-fancy-pypi-readme" +version = "25.1.0" +marker = "\"build\" in dependency_groups" +requires-python = ">=3.8" +dependencies = [ + { name = "hatchling" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "hatch_fancy_pypi_readme-25.1.0.tar.gz" +upload-time = 2025-05-01 10:34:42.989334+00:00 +url = "https://files.pythonhosted.org/packages/f3/0f/aed57c301f339936eb91cb4d8c1e5088a101081854bd3ec18a889df32365/hatch_fancy_pypi_readme-25.1.0.tar.gz" +size = 29701 + +[packages.sdist.hashes] +sha256 = "9c58ed3dff90d51f43414ce37009ad1d5b0f08ffc9fc216998a06380f01c0045" + +[[packages.wheels]] +name = "hatch_fancy_pypi_readme-25.1.0-py3-none-any.whl" +upload-time = 2025-05-01 10:34:41.204793+00:00 +url = "https://files.pythonhosted.org/packages/58/b4/ad71e051b34f1935783de37beffee16e343ff2514b204dc515f9e939d4af/hatch_fancy_pypi_readme-25.1.0-py3-none-any.whl" +size = 10046 + +[packages.wheels.hashes] +sha256 = "ce0134c40d63d874ac48f48ccc678b8f3b62b8e50e9318520d2bffc752eedaf3" + [[packages]] name = "hatch-vcs" version = "0.5.0" @@ -718,7 +880,7 @@ sha256 = "415290b84f464d2884303fca8974d38e7b85579069fab02a15db0828fbb1446d" [[packages]] name = "id" version = "1.6.1" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.9" dependencies = [ { name = "urllib3" }, @@ -743,10 +905,35 @@ size = 14689 [packages.wheels.hashes] sha256 = "f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca" +[[packages]] +name = "identify" +version = "2.6.20" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "identify-2.6.20.tar.gz" +upload-time = 2026-09-26 20:29:07.187608+00:00 +url = "https://files.pythonhosted.org/packages/53/35/d70c0006c7cee65999ea94a6273e60b2094f600a3d8b71b04318253fc643/identify-2.6.20.tar.gz" +size = 99681 + +[packages.sdist.hashes] +sha256 = "ad729860a923858d26917c2f4fb0a1d83d27a75b1e090c06440c573f048f3285" + +[[packages.wheels]] +name = "identify-2.6.20-py2.py3-none-any.whl" +upload-time = 2026-09-26 20:29:06.054092+00:00 +url = "https://files.pythonhosted.org/packages/fa/70/fffc9613501877c0a10a1ef73a165e6ded3e53f6e7502227fef56d973933/identify-2.6.20-py2.py3-none-any.whl" +size = 99506 + +[packages.wheels.hashes] +sha256 = "6a16b69b93187244e0548cbfd25b3e4a6f9a7a2ad784625c3bec2b8d27b81aaa" + [[packages]] name = "idna" version = "3.20" -marker = "\"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -768,10 +955,35 @@ size = 69583 [packages.wheels.hashes] sha256 = "ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c" +[[packages]] +name = "imagesize" +version = "2.0.1" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "imagesize-2.0.1.tar.gz" +upload-time = 2026-08-24 12:35:19.130667+00:00 +url = "https://files.pythonhosted.org/packages/fb/5e/513ff06670c84e7b9887c1fdf61b2d42b4f574a831f2f1d2222023049d8a/imagesize-2.0.1.tar.gz" +size = 1883774 + +[packages.sdist.hashes] +sha256 = "b2ba6a4dea487a7ebcd53248d3476aca449d30db12a2dde5e0c5ca9624fd77e5" + +[[packages.wheels]] +name = "imagesize-2.0.1-py3-none-any.whl" +upload-time = 2026-08-24 12:35:12.548956+00:00 +url = "https://files.pythonhosted.org/packages/01/f9/575c8d760eae1fc99651b7cc5efd96ad5379ca4d6b53750b0fb4fe983f34/imagesize-2.0.1-py3-none-any.whl" +size = 14794 + +[packages.wheels.hashes] +sha256 = "ea0c9a0384df69ed86a943a15cde37d0360b82491b3910dc2215e202e62b5b02" + [[packages]] name = "iniconfig" version = "2.3.1" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -793,10 +1005,35 @@ size = 7552 [packages.wheels.hashes] sha256 = "9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7" +[[packages]] +name = "installer" +version = "1.1.0" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "installer-1.1.0.tar.gz" +upload-time = 2026-10-12 19:35:52.851170+00:00 +url = "https://files.pythonhosted.org/packages/56/d3/2fab33d15004d41ae8e7b5d3e310510f7f34a7e1dd032a3e60d8b38d8fba/installer-1.1.0.tar.gz" +size = 481257 + +[packages.sdist.hashes] +sha256 = "5adc0c01ce62d809989e246987cd68944fe5df8af562d0358f65b2ab15cb5de7" + +[[packages.wheels]] +name = "installer-1.1.0-py3-none-any.whl" +upload-time = 2026-10-12 19:35:51.096942+00:00 +url = "https://files.pythonhosted.org/packages/d7/7a/63547a792a12148882205d79e07644091d5b7690f40c2bb9d41452a8bdc2/installer-1.1.0-py3-none-any.whl" +size = 464466 + +[packages.wheels.hashes] +sha256 = "f55f5728ff61d3dd54dbc5a397cf3f7307cb64b180c20b72afbebf01c9e2c8a0" + [[packages]] name = "isoduration" version = "20.11.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.7" dependencies = [ { name = "arrow" }, @@ -824,7 +1061,7 @@ sha256 = "b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042" [[packages]] name = "jaraco-classes" version = "3.4.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.8" dependencies = [ { name = "more-itertools" }, @@ -852,7 +1089,7 @@ sha256 = "f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790" [[packages]] name = "jaraco-context" version = "6.1.2" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -877,7 +1114,7 @@ sha256 = "bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535" [[packages]] name = "jaraco-functools" version = "4.6.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "more-itertools" }, @@ -905,7 +1142,7 @@ sha256 = "99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30" [[packages]] name = "jeepney" version = "0.9.0" -marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.7" index = "https://pypi.org/simple/" @@ -930,7 +1167,7 @@ sha256 = "97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683" [[packages]] name = "jinja2" version = "3.1.6" -marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.7" dependencies = [ { name = "markupsafe" }, @@ -958,7 +1195,7 @@ sha256 = "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" [[packages]] name = "jsonpointer" version = "3.6" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -983,7 +1220,7 @@ sha256 = "a45c858f8a9e171f6b3790ed9446c9382782f6c4cb4589176813c5a553f4cfcd" [[packages]] name = "jsonschema" version = "4.26.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" dependencies = [ { name = "attrs" }, @@ -1023,7 +1260,7 @@ sha256 = "d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce" [[packages]] name = "jsonschema-specifications" version = "2025.9.1" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" dependencies = [ { name = "referencing" }, @@ -1051,14 +1288,13 @@ sha256 = "98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe" [[packages]] name = "keyring" version = "25.7.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.9" dependencies = [ { name = "jaraco-classes" }, { name = "jaraco-context" }, { name = "jaraco-functools" }, { name = "jeepney" }, - { name = "pywin32-ctypes" }, { name = "secretstorage" }, ] index = "https://pypi.org/simple/" @@ -1084,7 +1320,7 @@ sha256 = "be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f" [[packages]] name = "lark" version = "1.3.1" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -1109,7 +1345,7 @@ sha256 = "c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12" [[packages]] name = "license-expression" version = "30.4.4" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" dependencies = [ { name = "boolean-py" }, @@ -1137,7 +1373,7 @@ sha256 = "421788fdcadb41f049d2dc934ce666626265aeccefddd25e162a26f23bcbf8a4" [[packages]] name = "markdown-it-py" version = "4.2.0" -marker = "\"pkg-meta\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "mdurl" }, @@ -1165,7 +1401,7 @@ sha256 = "9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a" [[packages]] name = "markupsafe" version = "3.0.4" -marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1178,15 +1414,6 @@ size = 153777 [packages.sdist.hashes] sha256 = "2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6" -[[packages.wheels]] -name = "markupsafe-3.0.4-cp315-cp315-macosx_11_0_arm64.whl" -upload-time = 2026-10-02 23:06:17.855823+00:00 -url = "https://files.pythonhosted.org/packages/96/24/3ec292b44064c16229e064d770b2625bd8ea941aa61f44905a9fa44942c0/markupsafe-3.0.4-cp315-cp315-macosx_11_0_arm64.whl" -size = 12075 - -[packages.wheels.hashes] -sha256 = "ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97" - [[packages.wheels]] name = "markupsafe-3.0.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" upload-time = 2026-10-02 23:06:22.524416+00:00 @@ -1196,19 +1423,10 @@ size = 23111 [packages.wheels.hashes] sha256 = "925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369" -[[packages.wheels]] -name = "markupsafe-3.0.4-cp315-cp315-win_amd64.whl" -upload-time = 2026-10-02 23:06:33.258900+00:00 -url = "https://files.pythonhosted.org/packages/5b/ec/6000fd82e8791e58fcd0456ec20f098957e2b03d5ed02eb73241a577c0ba/markupsafe-3.0.4-cp315-cp315-win_amd64.whl" -size = 14496 - -[packages.wheels.hashes] -sha256 = "d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214" - [[packages]] name = "mdformat" version = "1.0.0" -marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "markdown-it-py" }, @@ -1236,7 +1454,7 @@ sha256 = "bca015d65a1d063a02e885a91daee303057bc7829c2cd37b2075a50dbb65944b" [[packages]] name = "mdformat-gfm" version = "1.0.0" -marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "markdown-it-py" }, @@ -1267,7 +1485,7 @@ sha256 = "7305a50efd2a140d7c83505b58e3ac5df2b09e293f9bbe72f6c7bee8c678b005" [[packages]] name = "mdit-py-plugins" version = "0.6.1" -marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "markdown-it-py" }, @@ -1295,7 +1513,7 @@ sha256 = "214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d" [[packages]] name = "mdurl" version = "0.1.2" -marker = "\"pkg-meta\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" requires-python = ">=3.7" index = "https://pypi.org/simple/" @@ -1320,7 +1538,7 @@ sha256 = "84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" [[packages]] name = "more-itertools" version = "11.2.1" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.11" index = "https://pypi.org/simple/" @@ -1345,7 +1563,7 @@ sha256 = "35a7377edd1dd6608dcb2cdf534ded55ea32d49448875ad042cd3f879fb1ded0" [[packages]] name = "nh3" version = "0.3.7" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -1358,15 +1576,6 @@ size = 25662 [packages.sdist.hashes] sha256 = "71860d01c16f4d8c72e334e0674beb2b0899dbd0bf760de18932ef4390303848" -[[packages.wheels]] -name = "nh3-0.3.7-cp38-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl" -upload-time = 2026-08-23 14:26:09.025294+00:00 -url = "https://files.pythonhosted.org/packages/94/0d/c257754bf57f829f307aa226bbe136d3a1356b5a0d08324c7b6bd2a8aacd/nh3-0.3.7-cp38-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl" -size = 1493959 - -[packages.wheels.hashes] -sha256 = "6c3aa50eb26e9228238271db9f983cbc3b006dfbfeca2d4dc34c33ddc6ac5ea5" - [[packages.wheels]] name = "nh3-0.3.7-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" upload-time = 2026-08-23 14:26:17.272038+00:00 @@ -1376,19 +1585,35 @@ size = 832161 [packages.wheels.hashes] sha256 = "70f5ac8626e899a4bab0ef74ca2f5bd602f49c7b739e6e5026b4afc6d63dac42" +[[packages]] +name = "nodeenv" +version = "1.11.1" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "nodeenv-1.11.1.tar.gz" +upload-time = 2026-10-29 20:54:04.261005+00:00 +url = "https://files.pythonhosted.org/packages/ab/27/79acfd3d6f526a4a16f3d209dedcf7b5bc1ea15c5bf2bbb82f2c2285057e/nodeenv-1.11.1.tar.gz" +size = 99853 + +[packages.sdist.hashes] +sha256 = "d7fa027c1df437bdd974b0502a8dc76fa1b3183e369ac9539602bc6bab9100ba" + [[packages.wheels]] -name = "nh3-0.3.7-cp38-abi3-win_amd64.whl" -upload-time = 2026-08-23 14:26:28.294566+00:00 -url = "https://files.pythonhosted.org/packages/b4/b9/34433ccb1f0fe6968dabbb7d4bf5721c6221878ef07832748c06655a6a80/nh3-0.3.7-cp38-abi3-win_amd64.whl" -size = 644462 +name = "nodeenv-1.11.1-py2.py3-none-any.whl" +upload-time = 2026-10-29 20:54:02.752512+00:00 +url = "https://files.pythonhosted.org/packages/40/35/a09e0b1dac4027e35b872e4f123a9f9e7bbc39e6424bdc9776920d3a6491/nodeenv-1.11.1-py2.py3-none-any.whl" +size = 34473 [packages.wheels.hashes] -sha256 = "618e3059caf41ccdf5dcccb3fa9df4cf6e4efe23d1382a8bbfca272a8a4f8bfc" +sha256 = "bc0fb085d8ed0d67a6471892535f3d9b795eb911b7cd129d3a24967aab7a538f" [[packages]] name = "packageurl-python" version = "0.17.6" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -1413,7 +1638,7 @@ sha256 = "31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9" [[packages]] name = "packaging" version = "26.3" -marker = "\"build\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"project-runtime\" in dependency_groups or \"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"lint\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"project-runtime\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1460,10 +1685,35 @@ size = 57328 [packages.wheels.hashes] sha256 = "a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189" +[[packages]] +name = "platformdirs" +version = "4.14.0" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +requires-python = ">=3.11" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "platformdirs-4.14.0.tar.gz" +upload-time = 2026-10-28 18:27:30.973421+00:00 +url = "https://files.pythonhosted.org/packages/40/28/1c058a9143d2304b3078f8dadcbb9dc27f7d3581b5d05157c84869c4cc0d/platformdirs-4.14.0.tar.gz" +size = 70387 + +[packages.sdist.hashes] +sha256 = "83f009c1e8c5fc36869b5171755b291887bd2cade7965233cdfabd23d754720b" + +[[packages.wheels]] +name = "platformdirs-4.14.0-py3-none-any.whl" +upload-time = 2026-10-28 18:27:29.659262+00:00 +url = "https://files.pythonhosted.org/packages/69/2b/887969ef4bb53e0cb271dc7c3f4a73d50c5d58bb88aeaae311e3340cd57e/platformdirs-4.14.0-py3-none-any.whl" +size = 35259 + +[packages.wheels.hashes] +sha256 = "1dadfc59a8a5d7565b3d9270ae8fa71c065595352c4277b41c11ee6c222c4b91" + [[packages]] name = "pluggy" version = "1.7.0" -marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -1488,7 +1738,7 @@ sha256 = "7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec" [[packages]] name = "ply" version = "3.11" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" index = "https://pypi.org/simple/" [packages.sdist] @@ -1509,10 +1759,71 @@ size = 49567 [packages.wheels.hashes] sha256 = "096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce" +[[packages]] +name = "pre-commit" +version = "4.7.0" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +requires-python = ">=3.11" +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pre_commit-4.7.0.tar.gz" +upload-time = 2026-10-12 20:45:40.320969+00:00 +url = "https://files.pythonhosted.org/packages/3a/c3/6965c2690ddce885ec75121572d065a5b94c5a5d8d3e657927ecda9fd715/pre_commit-4.7.0.tar.gz" +size = 198120 + +[packages.sdist.hashes] +sha256 = "1f3deee12b914c8fa184bd2b5953391c2e4f36763e403d4a23dc9983fa7c6e86" + +[[packages.wheels]] +name = "pre_commit-4.7.0-py2.py3-none-any.whl" +upload-time = 2026-10-12 20:45:38.858255+00:00 +url = "https://files.pythonhosted.org/packages/eb/83/ce0a196e4f7b20b11ce3aa1fc72fb973f64b7628d51c1b8aebad37fb7f81/pre_commit-4.7.0-py2.py3-none-any.whl" +size = 226261 + +[packages.wheels.hashes] +sha256 = "2e229038ad3656081b70c27913157e4b636ef306f643f024a8a8e3f0f7b2f877" + +[[packages]] +name = "pre-commit-uv" +version = "4.4.1" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +requires-python = ">=3.11" +dependencies = [ + { name = "pre-commit" }, + { name = "uv" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pre_commit_uv-4.4.1.tar.gz" +upload-time = 2026-10-27 07:00:28.638309+00:00 +url = "https://files.pythonhosted.org/packages/98/62/2791f79c1cf48993ef0bee64e0636c9e86212467cdd3d0486a21d51f2438/pre_commit_uv-4.4.1.tar.gz" +size = 11474 + +[packages.sdist.hashes] +sha256 = "e59ce77dc49d0b0481efbc97293c4fe235ddeb5ab5028c8a9e20e6cb5cce9756" + +[[packages.wheels]] +name = "pre_commit_uv-4.4.1-py3-none-any.whl" +upload-time = 2026-10-27 07:00:27.254598+00:00 +url = "https://files.pythonhosted.org/packages/46/3d/ca2481fa9d5fd3c99e88be090dbadf5320dda8ddb80df13931e832ea46ef/pre_commit_uv-4.4.1-py3-none-any.whl" +size = 5525 + +[packages.wheels.hashes] +sha256 = "5c387a044d74c215090835c97bd41e469659025c6551ddc73f9f8e7d8d323c00" + [[packages]] name = "prek" version = "0.5.9" -marker = "\"release\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"release-prepare\" in dependency_groups" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -1525,15 +1836,6 @@ size = 603716 [packages.sdist.hashes] sha256 = "47bc660a7e959860a44afb5ff3dc167d96f04d929b405020a0f2c7683d3d00d8" -[[packages.wheels]] -name = "prek-0.5.9-py3-none-macosx_11_0_arm64.whl" -upload-time = 2026-10-30 10:31:46.060256+00:00 -url = "https://files.pythonhosted.org/packages/5d/17/cef855f5a9c40d588557bf7a703bce0420d9592fa8407d2320df9a48135a/prek-0.5.9-py3-none-macosx_11_0_arm64.whl" -size = 5511607 - -[packages.wheels.hashes] -sha256 = "db03274472abdc033fce78ff28d6abe221eebd2388f0c6d293376b1b09df0978" - [[packages.wheels]] name = "prek-0.5.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" upload-time = 2026-10-30 10:31:52.132595+00:00 @@ -1543,19 +1845,10 @@ size = 5942047 [packages.wheels.hashes] sha256 = "1ef04fd7b4bae3a31e1180bc87e877a3a5584063fe79320a125267afee47090d" -[[packages.wheels]] -name = "prek-0.5.9-py3-none-win_amd64.whl" -upload-time = 2026-10-30 10:31:57.930703+00:00 -url = "https://files.pythonhosted.org/packages/cf/1e/2c35ff3b059b6fbb43273c3ebe996b6d9d37c1889607c5ba6b9eea66530c/prek-0.5.9-py3-none-win_amd64.whl" -size = 5501419 - -[packages.wheels.hashes] -sha256 = "b8fe1a01273203932b46318ebf304304838b6b73ff99b89531f42c86b63b86d5" - [[packages]] name = "py-serializable" version = "2.1.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "<4.0,>=3.8" dependencies = [ { name = "defusedxml" }, @@ -1583,7 +1876,7 @@ sha256 = "b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304" [[packages]] name = "pycparser" version = "3.12" -marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.11" index = "https://pypi.org/simple/" @@ -1608,7 +1901,7 @@ sha256 = "a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e" [[packages]] name = "pydantic" version = "2.14.2" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "annotated-types" }, @@ -1639,7 +1932,7 @@ sha256 = "7efbf7507b937931cd32a5755acb00ea1cf5ccb1d4e8067a5850d5069e382118" [[packages]] name = "pydantic-core" version = "2.50.2" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "typing-extensions" }, @@ -1655,15 +1948,6 @@ size = 518711 [packages.sdist.hashes] sha256 = "1b0e8dc3932cf4e9ed5727570325cacc716b9409ffc794a221c0ccc72e8eb775" -[[packages.wheels]] -name = "pydantic_core-2.50.2-cp315-cp315-macosx_11_0_arm64.whl" -upload-time = 2026-10-29 16:33:44.946711+00:00 -url = "https://files.pythonhosted.org/packages/ae/ff/af99a09eddc348c1d0744c0df28dfcd4839ca497067f0fa2168208b41b1b/pydantic_core-2.50.2-cp315-cp315-macosx_11_0_arm64.whl" -size = 2100409 - -[packages.wheels.hashes] -sha256 = "f73930dacad5e611fe019e18df6f53c30629b1843d038d303b8dec86a32493bd" - [[packages.wheels]] name = "pydantic_core-2.50.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" upload-time = 2026-10-29 16:33:53.333286+00:00 @@ -1673,19 +1957,10 @@ size = 2249999 [packages.wheels.hashes] sha256 = "5299296ff5cafcb69851fd652de8f9268a0bb036bcbedbfcaa94cf743baf3986" -[[packages.wheels]] -name = "pydantic_core-2.50.2-cp315-cp315-win_amd64.whl" -upload-time = 2026-10-29 16:34:01.773818+00:00 -url = "https://files.pythonhosted.org/packages/24/6b/702adc91486fb96ba55675d58ee15ddc38fb48553fb35e26fd005e303206/pydantic_core-2.50.2-cp315-cp315-win_amd64.whl" -size = 2261064 - -[packages.wheels.hashes] -sha256 = "04cf672960b8c4fc28bb159c8cdd101fbd311c520237a5c0360448b851adfb41" - [[packages]] name = "pygments" version = "2.21.0" -marker = "\"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1710,7 +1985,7 @@ sha256 = "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9" [[packages]] name = "pyparsing" version = "3.3.3" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1735,10 +2010,9 @@ sha256 = "ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4" [[packages]] name = "pytest" version = "9.2.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" dependencies = [ - { name = "colorama" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, @@ -1767,7 +2041,7 @@ sha256 = "07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588" [[packages]] name = "pytest-cov" version = "7.1.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" dependencies = [ { name = "coverage" }, @@ -1797,7 +2071,7 @@ sha256 = "a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678" [[packages]] name = "pytest-mock" version = "3.17.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "pytest" }, @@ -1825,7 +2099,7 @@ sha256 = "1b82aa4b37137248bd687055e109723debeb07b489de1565297dc9589c5f2d6b" [[packages]] name = "python-dateutil" version = "3.0.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" dependencies = [ { name = "six" }, @@ -1852,34 +2126,37 @@ size = 94659 sha256 = "3c79630202e4152c6add678446984236efe4f57e84d8e892aab0637bab82e9fd" [[packages]] -name = "pywin32-ctypes" -version = "0.2.3" -marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"win32\"" -requires-python = ">=3.6" +name = "python-discovery" +version = "1.9.0" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +requires-python = ">=3.9" +dependencies = [ + { name = "filelock" }, +] index = "https://pypi.org/simple/" [packages.sdist] -name = "pywin32-ctypes-0.2.3.tar.gz" -upload-time = 2024-08-14 10:15:34.626878+00:00 -url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz" -size = 29471 +name = "python_discovery-1.9.0.tar.gz" +upload-time = 2026-10-30 23:24:24.910342+00:00 +url = "https://files.pythonhosted.org/packages/86/15/0405206e0cfe0c14b91ac038d31f30d1dd78139b3b2a285e6be41bf6e5af/python_discovery-1.9.0.tar.gz" +size = 125151 [packages.sdist.hashes] -sha256 = "d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755" +sha256 = "88beb57e51b919c30f303b053ec41359298ca349192035b0a802c76347ae96d7" [[packages.wheels]] -name = "pywin32_ctypes-0.2.3-py3-none-any.whl" -upload-time = 2024-08-14 10:15:33.187242+00:00 -url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl" -size = 30756 +name = "python_discovery-1.9.0-py3-none-any.whl" +upload-time = 2026-10-30 23:24:23.452190+00:00 +url = "https://files.pythonhosted.org/packages/cb/3c/da1cae963a076132dbc4d3c1f0292dd2f373b9d3e072d42f4620801159c1/python_discovery-1.9.0-py3-none-any.whl" +size = 39841 [packages.wheels.hashes] -sha256 = "8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8" +sha256 = "fb65edfff3541b7ceb3a39be334a0f69f4cb5148287a053e1874748742fda93b" [[packages]] name = "pyyaml" version = "6.0.3" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -1895,7 +2172,7 @@ sha256 = "d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f" [[packages]] name = "rdflib" version = "7.6.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8.1" dependencies = [ { name = "pyparsing" }, @@ -1923,7 +2200,7 @@ sha256 = "30c0a3ebf4c0e09215f066be7246794b6492e054e782d7ac2a34c9f70a15e0dd" [[packages]] name = "readme-renderer" version = "46.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "docutils" }, @@ -1953,7 +2230,7 @@ sha256 = "d0dae1f74bb273b534770cb4cccb6bb78735540afdb03c2146f4e19dcd412560" [[packages]] name = "referencing" version = "0.37.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" dependencies = [ { name = "attrs" }, @@ -1982,7 +2259,7 @@ sha256 = "381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231" [[packages]] name = "release-creator" version = "4.4.0" -marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" requires-python = ">=3.15" dependencies = [ { name = "docutils" }, @@ -2015,7 +2292,7 @@ sha256 = "560ed208cca0dc53bc227878e5596e3e26e44fa277658c5d0dc662901d95f8cb" [[packages]] name = "requests" version = "2.34.2" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "certifi" }, @@ -2046,7 +2323,7 @@ sha256 = "2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0" [[packages]] name = "requests-toolbelt" version = "1.0.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" dependencies = [ { name = "requests" }, @@ -2074,7 +2351,7 @@ sha256 = "cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" [[packages]] name = "rfc3339-validator" version = "0.1.4" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" dependencies = [ { name = "six" }, @@ -2102,7 +2379,7 @@ sha256 = "24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa" [[packages]] name = "rfc3986" version = "2.0.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.7" index = "https://pypi.org/simple/" @@ -2127,7 +2404,7 @@ sha256 = "50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd" [[packages]] name = "rfc3986-validator" version = "0.1.1" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" index = "https://pypi.org/simple/" @@ -2152,7 +2429,7 @@ sha256 = "2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9" [[packages]] name = "rfc3987-syntax" version = "1.1.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" dependencies = [ { name = "lark" }, @@ -2180,7 +2457,7 @@ sha256 = "6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f" [[packages]] name = "rich" version = "15.0.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.9.0" dependencies = [ { name = "markdown-it-py" }, @@ -2206,10 +2483,35 @@ size = 310654 [packages.wheels.hashes] sha256 = "33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb" +[[packages]] +name = "roman-numerals" +version = "4.1.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "roman_numerals-4.1.0.tar.gz" +upload-time = 2025-12-17 18:25:34.381818+00:00 +url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz" +size = 9077 + +[packages.sdist.hashes] +sha256 = "1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2" + +[[packages.wheels]] +name = "roman_numerals-4.1.0-py3-none-any.whl" +upload-time = 2025-12-17 18:25:33.098993+00:00 +url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl" +size = 7676 + +[packages.wheels.hashes] +sha256 = "647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7" + [[packages]] name = "rpds-py" version = "2026.9.1" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.11" index = "https://pypi.org/simple/" @@ -2222,15 +2524,6 @@ size = 63948 [packages.sdist.hashes] sha256 = "4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12" -[[packages.wheels]] -name = "rpds_py-2026.9.1-cp315-cp315-macosx_11_0_arm64.whl" -upload-time = 2026-10-04 16:30:49.872953+00:00 -url = "https://files.pythonhosted.org/packages/ae/3f/ad45d03df4f84ebae5439577ee81f3999d182711e82235c8037b6528890e/rpds_py-2026.9.1-cp315-cp315-macosx_11_0_arm64.whl" -size = 342051 - -[packages.wheels.hashes] -sha256 = "00ba2d8c7dd4ee537978ddf4b3fbd712bef2d8751603f7f3146b3f4287768e25" - [[packages.wheels]] name = "rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" upload-time = 2026-10-04 16:30:58.295504+00:00 @@ -2240,19 +2533,10 @@ size = 372022 [packages.wheels.hashes] sha256 = "2c16ab111bc27c646ba8aa005d0527754edc538ebb636f0b1bf8e244b48d1945" -[[packages.wheels]] -name = "rpds_py-2026.9.1-cp315-cp315-win_amd64.whl" -upload-time = 2026-10-04 16:31:10.840084+00:00 -url = "https://files.pythonhosted.org/packages/6d/a5/71b5cd00e0521e3b6b81828baea368c62b6b700ebdd9554cd7d41ddf12fa/rpds_py-2026.9.1-cp315-cp315-win_amd64.whl" -size = 223121 - -[packages.wheels.hashes] -sha256 = "75c38c50ab9aca840225d9a9a3810bf11d04bd5c1f186cabbb8aee56db3e9b15" - [[packages]] name = "secretstorage" version = "3.5.0" -marker = "\"pkg-meta\" in dependency_groups and sys_platform == \"linux\"" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "cryptography" }, @@ -2281,7 +2565,7 @@ sha256 = "0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137" [[packages]] name = "semantic-version" version = "2.10.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=2.7" index = "https://pypi.org/simple/" @@ -2361,7 +2645,7 @@ sha256 = "82f34c3e3084fc2b57d397200637cc13f2338004052d2fee47baa5f0e902464e" [[packages]] name = "six" version = "1.17.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" index = "https://pypi.org/simple/" @@ -2383,10 +2667,35 @@ size = 11050 [packages.wheels.hashes] sha256 = "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274" +[[packages]] +name = "snowballstemmer" +version = "3.1.1" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.3" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "snowballstemmer-3.1.1.tar.gz" +upload-time = 2026-06-03 00:56:40.194915+00:00 +url = "https://files.pythonhosted.org/packages/43/f8/0a71edf031f03c40db17503cb8ca78a69a171254e568e7db241b0ab57ea1/snowballstemmer-3.1.1.tar.gz" +size = 123314 + +[packages.sdist.hashes] +sha256 = "e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260" + +[[packages.wheels]] +name = "snowballstemmer-3.1.1-py3-none-any.whl" +upload-time = 2026-06-03 00:56:38.614300+00:00 +url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl" +size = 104164 + +[packages.wheels.hashes] +sha256 = "7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752" + [[packages]] name = "sortedcontainers" version = "2.4.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" index = "https://pypi.org/simple/" [packages.sdist] @@ -2407,10 +2716,35 @@ size = 29575 [packages.wheels.hashes] sha256 = "a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" +[[packages]] +name = "soupsieve" +version = "3.0.4" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.11.5" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "soupsieve-3.0.4.tar.gz" +upload-time = 2026-10-22 00:45:26.362373+00:00 +url = "https://files.pythonhosted.org/packages/19/63/0b6bae54546aed98d72f12d09df57b19a287f01c84d22047fa2e23772ab9/soupsieve-3.0.4.tar.gz" +size = 119017 + +[packages.sdist.hashes] +sha256 = "924fab84ae21604c19e175e1f39815eb6fa83ba23836e09e9de180925e5bc4c0" + +[[packages.wheels]] +name = "soupsieve-3.0.4-py3-none-any.whl" +upload-time = 2026-10-22 00:45:24.843035+00:00 +url = "https://files.pythonhosted.org/packages/07/66/473f7ca0be0b06e38ee8c85ee2db184e3981bd769fac2e7c19c8a8865211/soupsieve-3.0.4-py3-none-any.whl" +size = 42458 + +[packages.wheels.hashes] +sha256 = "8412826614f9c632227067d0985aeb9f584bd15efc852955430e097a9fe7afbf" + [[packages]] name = "spdx-tools" version = "0.8.5" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" dependencies = [ { name = "beartype" }, @@ -2443,6 +2777,283 @@ size = 286573 [packages.wheels.hashes] sha256 = "7c2d5865941be9d2e898f5b084e8d5422dd298dc5a29320ddb198fec304f59c4" +[[packages]] +name = "sphinx" +version = "9.2.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.13" +dependencies = [ + { name = "alabaster" }, + { name = "babel" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinx-9.2.0.tar.gz" +upload-time = 2026-10-21 00:05:15.019079+00:00 +url = "https://files.pythonhosted.org/packages/64/d4/413466f0be8b1cb0e3e43893a148b96d7462d0923d82b4166b21afc5f30c/sphinx-9.2.0.tar.gz" +size = 8782438 + +[packages.sdist.hashes] +sha256 = "90470459f3cab15d9dfed146c71cf77694b0445df17b2e0e69059cb9f7fc9ee4" + +[[packages.wheels]] +name = "sphinx-9.2.0-py3-none-any.whl" +upload-time = 2026-10-21 00:05:12.241734+00:00 +url = "https://files.pythonhosted.org/packages/c4/28/a0c50761cd215b7042f4bc617a6bc5c0cd7be62f75bc908e169e28a4d21e/sphinx-9.2.0-py3-none-any.whl" +size = 3925388 + +[packages.wheels.hashes] +sha256 = "4c3ec13b612712c0d3ce4cbdd9e3eb91e68577e07a8e31f32f21cf178453578c" + +[[packages]] +name = "sphinx-argparse-cli" +version = "1.24.1" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.11" +dependencies = [ + { name = "sphinx" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinx_argparse_cli-1.24.1.tar.gz" +upload-time = 2026-10-27 07:02:09.017213+00:00 +url = "https://files.pythonhosted.org/packages/ec/83/5502c61d82db593198adcb3a376505b767f0f464ebf2db50253cf3c7f319/sphinx_argparse_cli-1.24.1.tar.gz" +size = 17808 + +[packages.sdist.hashes] +sha256 = "9adc72739eb3b099f5de94d8f60af031aaef8355f57398d33ff3e6ea848abd00" + +[[packages.wheels]] +name = "sphinx_argparse_cli-1.24.1-py3-none-any.whl" +upload-time = 2026-10-27 07:02:07.884391+00:00 +url = "https://files.pythonhosted.org/packages/51/c6/16119ac3a8b54b3e92ad668bc6e787cbc79353e7f177f247d4e3f74b3de2/sphinx_argparse_cli-1.24.1-py3-none-any.whl" +size = 12299 + +[packages.wheels.hashes] +sha256 = "6bc2e83f4ecf7c110a91a526ca239465fef6d457c0665d89d5e0a7dc3eb71c73" + +[[packages]] +name = "sphinx-autodoc-typehints" +version = "3.17.4" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.12" +dependencies = [ + { name = "sphinx" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinx_autodoc_typehints-3.17.4.tar.gz" +upload-time = 2026-10-27 19:18:44.225219+00:00 +url = "https://files.pythonhosted.org/packages/9e/a9/cfe13ff89678c6d310a8a9060333b7823bae1dbce9615c7032be89aaec20/sphinx_autodoc_typehints-3.17.4.tar.gz" +size = 511398 + +[packages.sdist.hashes] +sha256 = "a5ddd545192380292f13dd9fc232717a0925fbc66617bc51b3db1e0480db0fec" + +[[packages.wheels]] +name = "sphinx_autodoc_typehints-3.17.4-py3-none-any.whl" +upload-time = 2026-10-27 19:18:42.674074+00:00 +url = "https://files.pythonhosted.org/packages/fa/66/40004fe2aea60edc4800328599dd115b3058f79bf148be1746349281f308/sphinx_autodoc_typehints-3.17.4-py3-none-any.whl" +size = 140603 + +[packages.wheels.hashes] +sha256 = "251271a49e46573c2e752bcc92dae232def822a2c9983272cb04d282eadc2bfa" + +[[packages]] +name = "sphinx-basic-ng" +version = "1.0.0b2" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.7" +dependencies = [ + { name = "sphinx" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinx_basic_ng-1.0.0b2.tar.gz" +upload-time = 2023-07-08 18:40:54.166146+00:00 +url = "https://files.pythonhosted.org/packages/98/0b/a866924ded68efec7a1759587a4e478aec7559d8165fac8b2ad1c0e774d6/sphinx_basic_ng-1.0.0b2.tar.gz" +size = 20736 + +[packages.sdist.hashes] +sha256 = "9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9" + +[[packages.wheels]] +name = "sphinx_basic_ng-1.0.0b2-py3-none-any.whl" +upload-time = 2023-07-08 18:40:52.659879+00:00 +url = "https://files.pythonhosted.org/packages/3c/dd/018ce05c532a22007ac58d4f45232514cd9d6dd0ee1dc374e309db830983/sphinx_basic_ng-1.0.0b2-py3-none-any.whl" +size = 22496 + +[packages.wheels.hashes] +sha256 = "eb09aedbabfb650607e9b4b68c9d240b90b1e1be221d6ad71d61c52e29f7932b" + +[[packages]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinxcontrib_applehelp-2.0.0.tar.gz" +upload-time = 2024-07-29 01:09:00.465812+00:00 +url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz" +size = 20053 + +[packages.sdist.hashes] +sha256 = "2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1" + +[[packages.wheels]] +name = "sphinxcontrib_applehelp-2.0.0-py3-none-any.whl" +upload-time = 2024-07-29 01:08:58.990722+00:00 +url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl" +size = 119300 + +[packages.wheels.hashes] +sha256 = "4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5" + +[[packages]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinxcontrib_devhelp-2.0.0.tar.gz" +upload-time = 2024-07-29 01:09:23.417467+00:00 +url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz" +size = 12967 + +[packages.sdist.hashes] +sha256 = "411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad" + +[[packages.wheels]] +name = "sphinxcontrib_devhelp-2.0.0-py3-none-any.whl" +upload-time = 2024-07-29 01:09:21.945229+00:00 +url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl" +size = 82530 + +[packages.wheels.hashes] +sha256 = "aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2" + +[[packages]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinxcontrib_htmlhelp-2.1.0.tar.gz" +upload-time = 2024-07-29 01:09:37.889318+00:00 +url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz" +size = 22617 + +[packages.sdist.hashes] +sha256 = "c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9" + +[[packages.wheels]] +name = "sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl" +upload-time = 2024-07-29 01:09:36.407072+00:00 +url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl" +size = 98705 + +[packages.wheels.hashes] +sha256 = "166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8" + +[[packages]] +name = "sphinxcontrib-jsmath" +version = "2.0.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinxcontrib_jsmath-2.0.0.tar.gz" +upload-time = 2026-10-20 18:22:05.514380+00:00 +url = "https://files.pythonhosted.org/packages/59/02/4f334c7fc3e54c51d2364575a41cd2fae6847e7f53001949f0be5f32e18b/sphinxcontrib_jsmath-2.0.0.tar.gz" +size = 14511 + +[packages.sdist.hashes] +sha256 = "551f780cf9687f9c3c74f0cf9f18aca06e0d6958853453f3803fe4d357c5b924" + +[[packages.wheels]] +name = "sphinxcontrib_jsmath-2.0.0-py3-none-any.whl" +upload-time = 2026-10-20 18:22:04.420584+00:00 +url = "https://files.pythonhosted.org/packages/da/c2/2965c35b39587d4b7a397f65a9c7d4b942b031061e7ac1ce91782f394a0d/sphinxcontrib_jsmath-2.0.0-py3-none-any.whl" +size = 78095 + +[packages.wheels.hashes] +sha256 = "15ca92fe0eec82f0baf91a7efcc51ffacdb605924be6327683a52ebdacb6744c" + +[[packages]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinxcontrib_qthelp-2.0.0.tar.gz" +upload-time = 2024-07-29 01:09:56.435212+00:00 +url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz" +size = 17165 + +[packages.sdist.hashes] +sha256 = "4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab" + +[[packages.wheels]] +name = "sphinxcontrib_qthelp-2.0.0-py3-none-any.whl" +upload-time = 2024-07-29 01:09:54.885300+00:00 +url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl" +size = 88743 + +[packages.wheels.hashes] +sha256 = "b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb" + +[[packages]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinxcontrib_serializinghtml-2.0.0.tar.gz" +upload-time = 2024-07-29 01:10:09.332393+00:00 +url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz" +size = 16080 + +[packages.sdist.hashes] +sha256 = "e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d" + +[[packages.wheels]] +name = "sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl" +upload-time = 2024-07-29 01:10:08.203690+00:00 +url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl" +size = 92072 + +[packages.wheels.hashes] +sha256 = "6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331" + [[packages]] name = "tomlkit" version = "0.15.2" @@ -2471,7 +3082,7 @@ sha256 = "05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437" [[packages]] name = "towncrier" version = "26.9.0" -marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" requires-python = ">=3.9" dependencies = [ { name = "click" }, @@ -2524,7 +3135,7 @@ sha256 = "8b1ff4f9c191b1040b71c37f1e445ab99732911e3cd91de52838453a854d7a17" [[packages]] name = "twine" version = "7.0.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "id" }, @@ -2560,7 +3171,7 @@ sha256 = "b854164df26db268af05f49aa5c0344b10e27a494343ff05b1e0bad3b135f5a7" [[packages]] name = "ty" version = "0.0.95" -marker = "\"type\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -2573,15 +3184,6 @@ size = 8115525 [packages.sdist.hashes] sha256 = "a480853a74d43074b2d4b28025d406b9d7223f7fe9f3a6c2adec3eacdbf14ca0" -[[packages.wheels]] -name = "ty-0.0.95-py3-none-macosx_11_0_arm64.whl" -upload-time = 2026-10-29 19:21:16.734602+00:00 -url = "https://files.pythonhosted.org/packages/dc/6b/2a084a72c8069f8496fcc0f576ec053f648b1baeed07532b3eb84c4982e4/ty-0.0.95-py3-none-macosx_11_0_arm64.whl" -size = 13144133 - -[packages.wheels.hashes] -sha256 = "3e7902477410e897a60d2cf29ca89c28ef2a2abdb81698d8ae4be2e33b934d4b" - [[packages.wheels]] name = "ty-0.0.95-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" upload-time = 2026-10-29 19:21:30.801674+00:00 @@ -2591,19 +3193,10 @@ size = 16312046 [packages.wheels.hashes] sha256 = "6d47be2db5db80cd70c582b240a71dd8e9be215ce9288298c863c7c3611ef02b" -[[packages.wheels]] -name = "ty-0.0.95-py3-none-win_amd64.whl" -upload-time = 2026-10-29 19:21:47.462829+00:00 -url = "https://files.pythonhosted.org/packages/b2/df/46fdcc84de81159e1d40ca103d24ac08b2450a2a51a1572333e8e796b8fc/ty-0.0.95-py3-none-win_amd64.whl" -size = 14097073 - -[packages.wheels.hashes] -sha256 = "d07bd09f8a525759a595e04051e923aa54f8dc483f90e6a0b003d77a3fae0f04" - [[packages]] name = "typing-extensions" version = "4.16.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -2628,7 +3221,7 @@ sha256 = "481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8" [[packages]] name = "typing-inspection" version = "0.4.4" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "typing-extensions" }, @@ -2656,7 +3249,7 @@ sha256 = "65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147" [[packages]] name = "tzdata" version = "2026.6" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=2" index = "https://pypi.org/simple/" @@ -2681,7 +3274,7 @@ sha256 = "0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec" [[packages]] name = "uri-template" version = "1.3.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.7" index = "https://pypi.org/simple/" @@ -2706,7 +3299,7 @@ sha256 = "a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363" [[packages]] name = "uritools" version = "6.1.3" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -2731,7 +3324,7 @@ sha256 = "136f113e76e53f85bf2d9cce0c3d40ceec775d0409e7d6de9b28f046a7d42838" [[packages]] name = "urllib3" version = "2.8.0" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -2756,7 +3349,7 @@ sha256 = "0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3" [[packages]] name = "uv" version = "0.13.6" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -2769,15 +3362,6 @@ size = 7545242 [packages.sdist.hashes] sha256 = "d762215478854f6cf20c3faed8f1eb30c8101d568e7bc32a3abafc063929d5c5" -[[packages.wheels]] -name = "uv-0.13.6-py3-none-macosx_11_0_arm64.whl" -upload-time = 2026-10-27 00:45:09.125609+00:00 -url = "https://files.pythonhosted.org/packages/f4/14/77230cc4f6a73a31578cb8070a5f67f58a76946ead643ae8a4155620d6da/uv-0.13.6-py3-none-macosx_11_0_arm64.whl" -size = 14344205 - -[packages.wheels.hashes] -sha256 = "51361029b7cf91185d0092c4b356d110b4ad629e9cb11593f1e006ac72a72ca9" - [[packages.wheels]] name = "uv-0.13.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" upload-time = 2026-10-27 00:45:31.291019+00:00 @@ -2787,15 +3371,6 @@ size = 17333810 [packages.wheels.hashes] sha256 = "22759b8895e1103578de6fb4cd27e94b4399d021023378d29f01333717751fba" -[[packages.wheels]] -name = "uv-0.13.6-py3-none-win_amd64.whl" -upload-time = 2026-10-27 00:45:52.086199+00:00 -url = "https://files.pythonhosted.org/packages/7c/9d/714523188aea3054644d8e69ae8089aac70325a3cee2d3d05509d4f23398/uv-0.13.6-py3-none-win_amd64.whl" -size = 15334118 - -[packages.wheels.hashes] -sha256 = "7c9bc35ccb65932d8dc10a0669f9c805eaa3c9827ce18ecbecb1bd67944b2cdf" - [[packages]] name = "vcs-versioning" version = "2.6.0" @@ -2824,10 +3399,42 @@ size = 123154 [packages.wheels.hashes] sha256 = "f8bebd1302ebc12e67ceb30fc7dd698ec969414b6fa43b30c3f14ac9b1e72c41" +[[packages]] +name = "virtualenv" +version = "21.15.2" +marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups" +requires-python = ">=3.9" +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "python-discovery" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "virtualenv-21.15.2.tar.gz" +upload-time = 2026-10-30 19:18:04.759827+00:00 +url = "https://files.pythonhosted.org/packages/38/9e/40c7098240cedd3a2f32e42a74b35c4563c617fdd7e41ae6268d4fdd8915/virtualenv-21.15.2.tar.gz" +size = 5520597 + +[packages.sdist.hashes] +sha256 = "dbd886e6e5689eb4dde6a9bbc478ad14b4fc8ed0852690d8273ccfb8c467f635" + +[[packages.wheels]] +name = "virtualenv-21.15.2-py3-none-any.whl" +upload-time = 2026-10-30 19:18:01.134751+00:00 +url = "https://files.pythonhosted.org/packages/f7/d8/09c9d6df253b2016312dd5ccbb962dfa86a5202f2452604b52e0a9844661/virtualenv-21.15.2-py3-none-any.whl" +size = 5559530 + +[packages.wheels.hashes] +sha256 = "fb8c62c7c328e0238061c721068f06b7bc6bd10f7a0f5987195cef1008837ee6" + [[packages]] name = "wcwidth" version = "0.9.4" -marker = "\"release\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" +marker = "\"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -2840,15 +3447,6 @@ size = 970051 [packages.sdist.hashes] sha256 = "fd8d57d3cdfe570e3265c8fab1885c24e34ef40889c0759c0c304a19e9835954" -[[packages.wheels]] -name = "wcwidth-0.9.4-cp310-abi3-macosx_11_0_arm64.whl" -upload-time = 2026-10-24 18:11:02.008720+00:00 -url = "https://files.pythonhosted.org/packages/04/e9/dfdca30ca36c472de3e260406dee29fc108499ec590c63b3fe279887ee9d/wcwidth-0.9.4-cp310-abi3-macosx_11_0_arm64.whl" -size = 618904 - -[packages.wheels.hashes] -sha256 = "f279f32c8e206a9092fa350e53ca0c2bd8072e45b1c9fd31bb6d476b59b591c4" - [[packages.wheels]] name = "wcwidth-0.9.4-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" upload-time = 2026-10-24 18:11:18.748328+00:00 @@ -2858,15 +3456,6 @@ size = 785946 [packages.wheels.hashes] sha256 = "b185c8b927b23af2721be1783499d1cfa5c028b098b0858951b4ae224477f1c2" -[[packages.wheels]] -name = "wcwidth-0.9.4-cp310-abi3-win_amd64.whl" -upload-time = 2026-10-24 18:11:36.952282+00:00 -url = "https://files.pythonhosted.org/packages/d6/f1/c123f902bc2ff9e9c2ab66241b8872204df87cbb18ecdf035e120860fedc/wcwidth-0.9.4-cp310-abi3-win_amd64.whl" -size = 597780 - -[packages.wheels.hashes] -sha256 = "f28d77153303450ee13ebbe888297e75cf532c754555b6cabafc9bdda16c421c" - [[packages.wheels]] name = "wcwidth-0.9.4-py3-none-any.whl" upload-time = 2026-10-24 18:10:52.651030+00:00 @@ -2879,7 +3468,7 @@ sha256 = "901eace6d2dd25ebb917d0ce5db18e5c537e1504c905cf32f0d006c12098b05d" [[packages]] name = "webcolors" version = "25.10.0" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -2904,7 +3493,7 @@ sha256 = "032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d" [[packages]] name = "wheel-filename" version = "1.4.2" -marker = "\"pkg-meta\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -2929,7 +3518,7 @@ sha256 = "3fa599046443d4ca830d06e3d180cd0a675d5871af0a68daa5623318bb4d17e3" [[packages]] name = "xmltodict" version = "1.0.4" -marker = "\"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -2953,7 +3542,7 @@ sha256 = "a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a" [tool.nab] nab-version = "0.0.23" -created-at = 2026-11-07 02:33:11.469337+00:00 +created-at = 2026-11-07 03:47:52.490850+00:00 command-line = [ "nab", "lock", @@ -2967,6 +3556,4 @@ mode = "universal" python-specifier = ">=3.15,<3.16" platforms = [ "linux_x86_64", - "macos_arm64", - "windows_amd64", ] diff --git a/pyproject.toml b/pyproject.toml index ead9044..6398fff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [build-system] build-backend = "hatchling.build" requires = [ + "hatch-fancy-pypi-readme>=25.1", "hatch-vcs>=0.5", "hatchling>=1.32.4", ] @@ -8,8 +9,7 @@ requires = [ [project] name = "hatch-pep770" import-names = [ "hatch_pep770" ] -description = "Hatchling build hook that embeds a PEP 770 CycloneDX SBOM in the wheel." -readme = "README.md" +description = "Write PEP 770 CycloneDX SBOMs into wheels, from a hatchling build hook or the command line." keywords = [ "cyclonedx", "hatch", @@ -18,10 +18,11 @@ keywords = [ "spdx", ] license = "MIT" +license-files = [ "LICENSE" ] maintainers = [ { name = "Bernรกt Gรกbor", email = "gaborjbernat@gmail.com" }, ] -requires-python = ">=3.9" +requires-python = ">=3.11" classifiers = [ "Development Status :: 4 - Beta", "Framework :: Hatch", @@ -29,29 +30,30 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3.15", + "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Topic :: Software Development :: Build Tools", + "Typing :: Typed", ] dynamic = [ + "readme", "version", ] dependencies = [ - "hatchling>=1.27", - "packaging>=24.2", + "hatchling>=1.28", + "packaging>=26.3", ] optional-dependencies.validate = [ - "cyclonedx-python-lib[json-validation]>=11", - "spdx-tools>=0.8.3", + "cyclonedx-python-lib[json-validation]>=11.12", + "spdx-tools>=0.8.5", ] -urls.Changelog = "https://github.com/tox-dev/hatch-pep770/blob/main/CHANGELOG.md" -urls.Documentation = "https://github.com/tox-dev/hatch-pep770#hatch-pep770" +urls.Changelog = "https://hatch-pep770.readthedocs.io/en/latest/changelog.html" +urls.Documentation = "https://hatch-pep770.readthedocs.io" urls.Homepage = "https://github.com/tox-dev/hatch-pep770" urls.Source = "https://github.com/tox-dev/hatch-pep770" urls.Tracker = "https://github.com/tox-dev/hatch-pep770/issues" @@ -59,32 +61,39 @@ scripts.hatch-pep770 = "hatch_pep770.__main__:main" entry-points.hatch.pep770 = "hatch_pep770.hooks" [dependency-groups] +dev = [ + { include-group = "lint" }, + { include-group = "pkg-meta" }, + { include-group = "test" }, + { include-group = "type" }, +] test = [ "covdefaults>=2.3", - "cyclonedx-python-lib[json-validation]>=11", - "pytest>=8.4.2", + "cyclonedx-python-lib[json-validation]>=11.12", + "installer>=0.7", + "pytest>=9.1.1", "pytest-cov>=7.1", - "pytest-mock>=3.15.1", - "spdx-tools>=0.8.3", + "pytest-mock>=3.16", + "spdx-tools>=0.8.5", ] type = [ "ty>=0.0.84", + { include-group = "docs" }, { include-group = "test" }, ] +docs = [ + "furo>=2025.12.19", + "sphinx>=9.0.4", + "sphinx-argparse-cli>=1.23", + "sphinx-autodoc-typehints>=3.17", +] build = [ + "hatch-fancy-pypi-readme>=25.1", "hatch-vcs>=0.5", "hatchling>=1.32.4", ] -pkg-meta = [ - "check-wheel-contents>=0.6.3", - "twine>=7", - "uv>=0.12.22", -] -release = [ - "prek>=0.5", - "release-creator>=4.4", - "towncrier>=26.9", -] +lint = [ "pre-commit-uv>=4.3" ] +pkg-meta = [ "check-wheel-contents>=0.6.3", "twine>=7", "uv>=0.12.22" ] release-news = [ "release-creator>=4.4", "towncrier>=26.9", @@ -100,19 +109,17 @@ release-prepare = [ [tool.hatch] version.source = "vcs" +metadata.hooks.fancy-pypi-readme.content-type = "text/markdown" +metadata.hooks.fancy-pypi-readme.fragments = [ { path = "README.md" } ] build.hooks.vcs.version-file = "src/hatch_pep770/version.py" -build.targets.sdist.include = [ - "/src", - "/tests", -] [tool.uv] -dependency-groups.release = { requires-python = ">=3.15" } dependency-groups.release-news = { requires-python = ">=3.15" } dependency-groups.release-notes = { requires-python = ">=3.15" } dependency-groups.release-prepare = { requires-python = ">=3.15" } [tool.ruff] +target-version = "py311" line-length = 120 format.preview = true format.docstring-code-line-length = 100 @@ -121,31 +128,28 @@ lint.select = [ "ALL", ] lint.ignore = [ - "CPY", # No copyright statements - "D1", # docstrings only on the public API, enforced in review - "DOC", # not supported + "CPY", # The project does not add copyright headers to individual files + "D1", # names and types document the code; simplint asks for public API docs + "DOC", # Public API documentation is generated from signatures "incorrect-blank-line-before-class", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible "missing-trailing-comma", # Conflict with formatter "multi-line-summary-first-line", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible + "non-empty-init-module", # `__init__` module should only contain docstrings and re-exports "single-line-implicit-string-concatenation", # Conflict with formatter - "subprocess-without-shell-equals-true", # subprocess calls are fine - "suspicious-subprocess-import", # Using subprocess is alright ] lint.per-file-ignores."tests/**/*.py" = [ - "assert", # asserts allowed in tests - "D", # don't care about documentation in tests - "FBT", # don't care about booleans as positional arguments in tests - "implicit-namespace-package", # no implicit namespace - "magic-value-comparison", # Magic value used in comparison, consider replacing with a constant variable - "pytest-fixture-autouse", # shared setup fixtures are meant to apply to every test - "start-process-with-partial-path", # partial executable path is fine in tests - "subprocess-without-shell-equals-true", # `subprocess` call: check for execution of untrusted input - "too-many-arguments", # fixtures arrive as arguments - "too-many-positional-arguments", # fixtures arrive as arguments + "assert", # Tests use assertions + "D", # Test names document behavior + "FBT", # Boolean parameters keep parameterized cases compact + "implicit-namespace-package", # The tests directory is not a package + "import-private-name", # Tests cover private parsing boundaries directly + "magic-value-comparison", # Literal expected values keep assertions local + "subprocess-without-shell-equals-true", # Entry-point arguments are fixed by the tests + "suspicious-subprocess-import", # Entry-point tests invoke a local Python executable + "too-many-positional-arguments", # Test inputs favor explicit parameters over setup objects ] lint.isort = { known-first-party = [ "hatch_pep770", - "tests", ], required-imports = [ "from __future__ import annotations", ] } @@ -154,7 +158,6 @@ lint.preview = true [tool.codespell] builtin = "clear,usage,en-GB_to_en-US" ignore-words-list = "acknowledgement" -write-changes = true count = true [tool.pyproject-fmt] @@ -163,11 +166,19 @@ max_supported_python = "3.15" [tool.ty] environment.python-version = "3.15" +[tool.pytest] +ini_options.testpaths = [ "tests" ] +ini_options.filterwarnings = [ + "error", +] +ini_options.verbosity_assertions = 2 + [tool.coverage] run.parallel = true run.plugins = [ "covdefaults", ] +run.core = "ctrace" # sysmon (3.14+ default) cannot record the per-test contexts we collect paths.source = [ "src", ".tox*/*/lib/python*/site-packages", @@ -176,15 +187,17 @@ paths.source = [ "*/src", "*\\src", ] +report.fail_under = 100 html.show_contexts = true html.skip_covered = false [tool.towncrier] name = "hatch-pep770" -directory = "changelog.d" -filename = "CHANGELOG.md" -template = "release_creator.fragments:template.jinja2" +directory = "docs/changelog.d" +filename = "docs/changelog.rst" +template = "release_creator.fragments:template.rst.jinja2" title_format = false +underlines = [ "-", "~", "^" ] type = [ { directory = "feature", name = "Added", showcontent = true }, { directory = "change", name = "Changes", showcontent = true }, @@ -204,7 +217,7 @@ base-group = "project-runtime" uploaded-prior-to = "P7D" decision-order = "stable" matrix.python = ">=3.15,<3.16" -matrix.platforms = [ "linux_x86_64", "macos_arm64", "windows_amd64" ] +matrix.platforms = [ "linux_x86_64" ] # release-creator 4.4.0 needs a seven-day cutoff exception. # expires 2026-11-09 packages.release-creator.uploaded-prior-to = 2026-11-02T21:45:00Z diff --git a/tox.toml b/tox.toml index bcb5f79..75ba15b 100644 --- a/tox.toml +++ b/tox.toml @@ -1,5 +1,5 @@ requires = [ "tox>=4.64.7", "tox-uv>=1.36.1" ] -env_list = [ "3.15", "3.14", "3.13", "3.12", "3.11", "3.10", "3.9", "3.15t", "fix", "pkg_meta", "type" ] +env_list = [ "3.15", "3.14", "3.13", "3.12", "3.11", "3.14t", "3.15t", "docs", "fix", "pkg_meta", "type" ] skip_missing_interpreters = true [env_run_base] @@ -9,18 +9,15 @@ package = "wheel" wheel_build_env = ".pkg" dependency_groups = [ "test" ] pass_env = [ "PYTEST_*" ] -set_env.COVERAGE_FILE = { replace = "env", name = "COVERAGE_FILE", default = "{work_dir}{/}.coverage.{env_name}" } +set_env.COVERAGE_FILE = { replace = "env", name = "COVERAGE_FILE", default = "{env_tmp_dir}{/}.coverage" } +set_env.COVERAGE_FILECOVERAGE_PROCESS_START = "{tox_root}{/}pyproject.toml" commands = [ [ "pytest", { replace = "posargs", default = [ - "--durations", - "5", - "--junitxml", - "{work_dir}{/}junit.{env_name}.xml", "--no-cov-on-fail", "--cov", - "{env_site_packages_dir}{/}hatch_pep770", + "hatch_pep770", "--cov", "{tox_root}{/}tests", "--cov-config", @@ -31,9 +28,6 @@ commands = [ "term-missing:skip-covered", "--cov-report", "html:{env_tmp_dir}{/}htmlcov", - "--cov-report", - "xml:{work_dir}{/}coverage.{env_name}.xml", - "--cov-fail-under=100", "tests", ], extend = true }, ], @@ -42,21 +36,82 @@ commands = [ [env_pkg_base] default_base_python = [ "3.15" ] +[env.docs] +description = "Build documentation" +dependency_groups = [ "docs" ] +commands = [ + [ + "python", + "-c", + # RE2 lacks Python 3.15 wheels: https://github.com/google/re2/issues/647 + """\ + import glob; import pathlib; import subprocess; subprocess.check_call(['uv', 'tool', 'run', '--python', '3.14', \ + '--from', 'proselint>=0.16', 'proselint', 'check', *(path for path in glob.glob('docs/*.rst') if \ + pathlib.Path(path).name != 'changelog.rst')])\ + """, + ], + [ + "sphinx-build", + "-d", + "{env_tmp_dir}/doctree", + "docs", + "{env:READTHEDOCS_OUTPUT:{env_tmp_dir}{/}docs_out}/html", + "--color", + "-b", + "html", + { replace = "posargs", default = [ + "-W", + ], extend = true }, + ], + [ + "python", + "-c", + """\ + import pathlib; print('documentation available under ' + \ + (pathlib.Path(r'{env:READTHEDOCS_OUTPUT:{env_tmp_dir}{/}docs_out}/html') / 'index.html').as_uri())\ + """, + ], +] + [env.fix] description = "format the code base to adhere to our styles, and complain about what we cannot do automatically" skip_install = true -deps = [ "pre-commit-uv>=4.3" ] -pass_env = [ { replace = "ref", of = [ "env_run_base", "pass_env" ], extend = true }, "PROGRAMDATA" ] +dependency_groups = [ "lint" ] +pass_env = [ + { replace = "ref", of = [ + "env_run_base", + "pass_env", + ], extend = true }, + "DISABLE_PRE_COMMIT_UV_PATCH", + "PROGRAMDATA", +] commands = [ [ "pre-commit", "run", "--all-files", "--show-diff-on-failure", { replace = "posargs", extend = true } ] ] [env.pkg_meta] description = "check that the long description is valid" skip_install = true -dependency_groups = [ "pkg-meta" ] +dependency_groups = [ "pkg_meta" ] commands = [ - [ "uv", "build", "--sdist", "--wheel", "--out-dir", "{env_tmp_dir}", "." ], - [ "twine", "check", "{env_tmp_dir}{/}*" ], - [ "check-wheel-contents", "--no-config", "{env_tmp_dir}" ], + [ + "uv", + "build", + "--sdist", + "--wheel", + "--out-dir", + "{env_tmp_dir}", + ".", + ], + [ + "twine", + "check", + "--strict", + "{env_tmp_dir}{/}*", + ], + [ + "check-wheel-contents", + "--no-config", + "{env_tmp_dir}", + ], ] [env.type] @@ -67,6 +122,8 @@ commands = [ [ "ty", "check", "--output-format", "concise", "--error-on-warning" [env.dev] description = "dev environment with all deps at {envdir}" package = "editable" -deps = [ { replace = "ref", of = [ "requires" ], extend = true } ] -dependency_groups = [ "release", "type" ] -commands = [ [ "uv", "pip", "tree" ], [ "python", "-c", 'print(r"{env_python}")' ] ] +dependency_groups = [ "dev" ] +commands = [ + [ "uv", "pip", "tree" ], + [ "python", "-c", "import sys; print(sys.executable)" ], +] From 522b147c1f361b0dc4915170b91ba50bc6e6d571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 20:07:36 -0800 Subject: [PATCH 008/115] =?UTF-8?q?=F0=9F=9A=9A=20refactor:=20rename=20the?= =?UTF-8?q?=20project=20to=20python-sbom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The distribution, import package, console script and hatch hook are now python-sbom (python_sbom for imports), and the build hook class is SbomBuildHook. --- .github/SECURITY.md | 12 ++--- .github/THREAT_MODEL.md | 20 ++++---- .github/workflows/release.yaml | 6 +-- .github/workflows/scorecard.yaml | 2 +- .gitignore | 2 +- README.md | 30 ++++++------ ...k.feature.rst => +python-sbom.feature.rst} | 2 +- docs/cli.rst | 2 +- docs/conf.py | 16 +++---- docs/explanation.rst | 2 +- docs/extend.rst | 41 +++++++++-------- docs/index.rst | 18 ++++---- docs/py_api.rst | 46 +++++++++---------- docs/usage.rst | 30 ++++++------ pyproject.toml | 26 +++++------ src/{hatch_pep770 => python_sbom}/__init__.py | 6 +-- src/{hatch_pep770 => python_sbom}/__main__.py | 0 src/{hatch_pep770 => python_sbom}/_cli.py | 8 ++-- .../_cyclonedx.py | 0 src/{hatch_pep770 => python_sbom}/_embed.py | 0 src/{hatch_pep770 => python_sbom}/_extra.py | 0 src/{hatch_pep770 => python_sbom}/_hook.py | 12 ++--- src/{hatch_pep770 => python_sbom}/_project.py | 2 +- src/{hatch_pep770 => python_sbom}/_run.py | 4 +- src/{hatch_pep770 => python_sbom}/_spdx.py | 2 +- .../_validate.py | 0 src/{hatch_pep770 => python_sbom}/hooks.py | 8 ++-- src/{hatch_pep770 => python_sbom}/py.typed | 0 tests/conftest.py | 8 ++-- tests/seeder/hatch_build.py | 6 +-- tests/seeder/pyproject.toml | 2 +- tests/test_cli.py | 6 +-- tests/test_embed.py | 8 ++-- tests/test_hook.py | 8 ++-- tests/test_main.py | 8 ++-- tests/test_project.py | 14 ++++-- tests/test_spdx.py | 8 ++-- tests/test_validate.py | 8 ++-- tests/test_version.py | 2 +- tox.toml | 2 +- 40 files changed, 191 insertions(+), 186 deletions(-) rename docs/changelog.d/{+pep770-hook.feature.rst => +python-sbom.feature.rst} (53%) rename src/{hatch_pep770 => python_sbom}/__init__.py (84%) rename src/{hatch_pep770 => python_sbom}/__main__.py (100%) rename src/{hatch_pep770 => python_sbom}/_cli.py (90%) rename src/{hatch_pep770 => python_sbom}/_cyclonedx.py (100%) rename src/{hatch_pep770 => python_sbom}/_embed.py (100%) rename src/{hatch_pep770 => python_sbom}/_extra.py (100%) rename src/{hatch_pep770 => python_sbom}/_hook.py (91%) rename src/{hatch_pep770 => python_sbom}/_project.py (99%) rename src/{hatch_pep770 => python_sbom}/_run.py (94%) rename src/{hatch_pep770 => python_sbom}/_spdx.py (99%) rename src/{hatch_pep770 => python_sbom}/_validate.py (100%) rename src/{hatch_pep770 => python_sbom}/hooks.py (64%) rename src/{hatch_pep770 => python_sbom}/py.typed (100%) diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 0804a5f..90a06ec 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -1,13 +1,13 @@ # Security policy -hatch-pep770 writes a CycloneDX SBOM into wheels, from a hatchling build hook or from a command that rewrites a wheel +python-sbom writes a CycloneDX SBOM into wheels, from a hatchling build hook or from a command that rewrites a wheel another build backend built, and renders or validates that SBOM. This policy says how to report a security problem and how we decide whether a report is one. ## Reporting a vulnerability **Do not open a public issue for a security problem.** Use -[GitHub's private vulnerability reporting](https://github.com/tox-dev/hatch-pep770/security/advisories/new). If neither +[GitHub's private vulnerability reporting](https://github.com/tox-dev/python-sbom/security/advisories/new). If neither works for you, email [gaborjbernat@gmail.com](mailto:gaborjbernat@gmail.com). Include what the attacker controls and what they gain, the steps or a proof of concept, and the versions and platform @@ -15,7 +15,7 @@ you tested. Write the report yourself; unreviewed tool or model output takes lon ## What to expect -Volunteers maintain hatch-pep770, with no on-call rotation. We aim to acknowledge a report within **5 working days** and +Volunteers maintain python-sbom, with no on-call rotation. We aim to acknowledge a report within **5 working days** and assess it within **15 working days**. Fixes are developed in a private fork attached to the draft advisory, ship in a new release, and the advisory, with a CVE requested through GitHub, goes out with that release. The default embargo is **90 days** from the report; we agree any change with the reporter. We credit reporters in the advisory under the name @@ -28,14 +28,14 @@ Only the latest release receives fixes. Confirm the problem reproduces on it bef ## Vulnerability or bug A report is a vulnerability when an attacker who controls only an input that the [threat model](THREAT_MODEL.md) lists -as untrusted breaks a guarantee hatch-pep770 makes. A report that first needs control of something the threat model +as untrusted breaks a guarantee python-sbom makes. A report that first needs control of something the threat model trusts, such as the project source, its `pyproject.toml` or the build environment, is a bug: we fix it in the open and publish no advisory. Classify by who supplies the dangerous input, not by how bad the outcome sounds. **In scope** -- A bundled wheel, or an SBOM given to `hatch-pep770 spdx` or `validate`, that makes hatch-pep770 write outside its - output file or the wheel it rewrites, or run code. +- A bundled wheel, or an SBOM given to `python-sbom spdx` or `validate`, that makes python-sbom write outside its output + file or the wheel it rewrites, or run code. - An SBOM that misstates what a wheel holds in a way its inputs cannot explain, such as a wrong hash. **Out of scope** diff --git a/.github/THREAT_MODEL.md b/.github/THREAT_MODEL.md index e84cf8b..5f3b8ca 100644 --- a/.github/THREAT_MODEL.md +++ b/.github/THREAT_MODEL.md @@ -1,19 +1,19 @@ # Threat model -This model says what hatch-pep770 trusts, where untrusted input enters, and which guarantees hold at each boundary, so a +This model says what python-sbom trusts, where untrusted input enters, and which guarantees hold at each boundary, so a report can be classified as a vulnerability or a bug. [SECURITY.md](SECURITY.md) covers reporting; if the two disagree, SECURITY.md wins and we fix this file. -## How hatch-pep770 works +## How python-sbom works The build hook runs inside a hatchling wheel build. It reads the wheel's core metadata, the project's license files, the wheels the project bundles and the installed distributions of the build environment, and writes a CycloneDX document -that hatchling adds to the wheel. `hatch-pep770 embed` does the same for a wheel another backend built and rewrites that -wheel. `hatch-pep770 spdx` and `validate` read a CycloneDX document and write or check its SPDX form. +that hatchling adds to the wheel. `python-sbom embed` does the same for a wheel another backend built and rewrites that +wheel. `python-sbom spdx` and `validate` read a CycloneDX document and write or check its SPDX form. ```mermaid flowchart LR - project[("Project source, pyproject.toml
(trusted)")] --> hook["hatch-pep770"] + project[("Project source, pyproject.toml
(trusted)")] --> hook["python-sbom"] env[("Build environment
(trusted)")] --> hook bundled[("Bundled wheels
(third-party content)")] -->|"metadata, RECORD, vendor list"| hook sbom[("SBOM files given to spdx, validate")] -->|"untrusted JSON"| hook @@ -26,15 +26,15 @@ flowchart LR ## Trusted -An attacker who controls any of these already has what hatch-pep770 could give them: +An attacker who controls any of these already has what python-sbom could give them: - The project source, its `pyproject.toml` and the build hook script. - The build environment and the build backend. -- The wheel handed to `hatch-pep770 embed`, which the project's own build produced. +- The wheel handed to `python-sbom embed`, which the project's own build produced. ## Untrusted inputs -| Input | Comes from | What hatch-pep770 guarantees | +| Input | Comes from | What python-sbom guarantees | | --------------------------------------- | ------------------------- | ----------------------------------------------------------------------------------------- | | Metadata and `RECORD` of bundled wheels | Upstream projects | Read as text, copied into the document as JSON strings; nothing is extracted or executed. | | `_vendor/vendor.txt` of bundled wheels | Upstream projects | Each line must pin one exact version, or the build fails. | @@ -58,7 +58,7 @@ Every workflow pins third-party actions to a commit SHA. ## Severity -- **High:** a bundled wheel or SBOM file that makes hatch-pep770 run code or write outside its output. +- **High:** a bundled wheel or SBOM file that makes python-sbom run code or write outside its output. - **Medium:** an SBOM that misstates a hash, a license or the contents of a wheel. - **Low:** a crash on a malformed bundled wheel or SBOM file instead of a clean error. @@ -68,5 +68,5 @@ Every workflow pins third-party actions to a commit SHA. ## Keeping this current -Revisit this model when hatch-pep770 gains a new input, a new way to run code, or a new place it writes to, and whenever +Revisit this model when python-sbom gains a new input, a new way to run code, or a new place it writes to, and whenever an advisory is published. diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 63e208b..c11f6c9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -21,14 +21,14 @@ jobs: with: tag: ${{ github.ref_name }} changelog: docs/changelog.rst - artifact-url: https://pypi.org/project/hatch-pep770/${{ github.ref_name }}/ + artifact-url: https://pypi.org/project/python-sbom/${{ github.ref_name }}/ publish: name: ๐Ÿš€ Publish to PyPI needs: [build, notes] runs-on: ubuntu-26.04 environment: name: release - url: https://pypi.org/project/hatch-pep770/${{ github.ref_name }} + url: https://pypi.org/project/python-sbom/${{ github.ref_name }} permissions: id-token: write steps: @@ -49,7 +49,7 @@ jobs: permissions: contents: write with: - project: hatch-pep770 + project: python-sbom version: ${{ github.ref_name }} tag: ${{ github.ref_name }} artifact-pattern: python-package-distributions diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index a219ba2..eafe9e6 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -13,7 +13,7 @@ permissions: {} jobs: analysis: name: ๐Ÿ›ก๏ธ Scorecard analysis - if: github.repository == 'tox-dev/hatch-pep770' + if: github.repository == 'tox-dev/python-sbom' permissions: actions: read # read workflow runs for the Packaging check checks: read # read check runs for the CI-Tests and SAST checks diff --git a/.gitignore b/.gitignore index 52482c9..69be040 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,5 @@ __pycache__ **.pyc /build dist -src/hatch_pep770/version.py +src/python_sbom/version.py /uv.lock diff --git a/README.md b/README.md index 341f9b7..191a4f4 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ -# hatch-pep770 +# python-sbom -[![PyPI](https://img.shields.io/pypi/v/hatch-pep770?style=flat-square)](https://pypi.org/project/hatch-pep770/) -[![Supported Python versions](https://img.shields.io/pypi/pyversions/hatch-pep770.svg)](https://pypi.org/project/hatch-pep770/) -[![check](https://github.com/tox-dev/hatch-pep770/actions/workflows/check.yaml/badge.svg)](https://github.com/tox-dev/hatch-pep770/actions/workflows/check.yaml) -[![Documentation Status](https://readthedocs.org/projects/hatch-pep770/badge/?version=latest)](https://hatch-pep770.readthedocs.io/en/latest/?badge=latest) -[![Downloads](https://static.pepy.tech/badge/hatch-pep770/month)](https://pepy.tech/project/hatch-pep770) +[![PyPI](https://img.shields.io/pypi/v/python-sbom?style=flat-square)](https://pypi.org/project/python-sbom/) +[![Supported Python versions](https://img.shields.io/pypi/pyversions/python-sbom.svg)](https://pypi.org/project/python-sbom/) +[![check](https://github.com/tox-dev/python-sbom/actions/workflows/check.yaml/badge.svg)](https://github.com/tox-dev/python-sbom/actions/workflows/check.yaml) +[![Documentation Status](https://readthedocs.org/projects/python-sbom/badge/?version=latest)](https://python-sbom.readthedocs.io/en/latest/?badge=latest) +[![Downloads](https://static.pepy.tech/badge/python-sbom/month)](https://pepy.tech/project/python-sbom) -Write a [CycloneDX](https://cyclonedx.org/) SBOM into every wheel you build, where -[PEP 770](https://peps.python.org/pep-0770/) tells installers to keep it. The SBOM records your project, the runtime -dependencies it declares, any wheels it bundles, and the build toolchain, and a rebuild with the same inputs reproduces -it byte for byte. +SBOMs for Python distributions. python-sbom writes a [CycloneDX](https://cyclonedx.org/) SBOM into every wheel you +build, where [PEP 770](https://peps.python.org/pep-0770/) tells installers to keep it. The SBOM records your project, +the runtime dependencies it declares, any wheels it bundles, and the build toolchain, and a rebuild with the same inputs +reproduces it byte for byte. ## Use @@ -18,16 +18,16 @@ With [hatchling](https://hatch.pypa.io/latest/), add the hook to the build requi ```toml [build-system] build-backend = "hatchling.build" -requires = ["hatch-pep770>=0.1", "hatchling>=1.28"] +requires = ["python-sbom>=0.1", "hatchling>=1.28"] -[tool.hatch.build.hooks.pep770] +[tool.hatch.build.hooks.python-sbom] ``` With another build backend, such as meson-python, embed the SBOM after the build: ```console -hatch-pep770 embed dist/*.whl +python-sbom embed dist/*.whl ``` -The [documentation](https://hatch-pep770.readthedocs.io) covers the options, extending the SBOM from a custom build -hook, rendering it as SPDX and validating it. +The [documentation](https://python-sbom.readthedocs.io) covers the options, extending the SBOM from a custom build hook, +rendering it as SPDX and validating it. diff --git a/docs/changelog.d/+pep770-hook.feature.rst b/docs/changelog.d/+python-sbom.feature.rst similarity index 53% rename from docs/changelog.d/+pep770-hook.feature.rst rename to docs/changelog.d/+python-sbom.feature.rst index d1db1f1..d356c8f 100644 --- a/docs/changelog.d/+pep770-hook.feature.rst +++ b/docs/changelog.d/+python-sbom.feature.rst @@ -1,3 +1,3 @@ -Add the ``pep770`` hatchling build hook, which writes a CycloneDX 1.6 SBOM into each wheel, the ``hatch-pep770`` +Add the ``python-sbom`` hatchling build hook, which writes a CycloneDX 1.6 SBOM into each wheel, the ``python-sbom`` command that embeds the same SBOM into wheels other build backends build and renders or validates it, and the Python API behind both. diff --git a/docs/cli.rst b/docs/cli.rst index d28756c..bf07088 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1,5 +1,5 @@ .. sphinx_argparse_cli:: - :module: hatch_pep770._cli + :module: python_sbom._cli :func: _build_parser :title: Command line interface :group_title_prefix: diff --git a/docs/conf.py b/docs/conf.py index c49d3ef..ef378d3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,8 +7,8 @@ from tomllib import loads from typing import TYPE_CHECKING, Final -import hatch_pep770 -from hatch_pep770 import __version__ +import python_sbom +from python_sbom import __version__ if TYPE_CHECKING: from docutils.nodes import Element, reference @@ -19,14 +19,14 @@ def _inheritance_aliases() -> dict[str, str]: aliases: dict[str, str] = {} - for entry in hatch_pep770.__all__: - to_module = getattr(getattr(hatch_pep770, entry), "__module__", "") + for entry in python_sbom.__all__: + to_module = getattr(getattr(python_sbom, entry), "__module__", "") # first instance wins - aliases.setdefault(f"{to_module}{'.' if to_module else ''}{entry}", f"{hatch_pep770.__name__}.{entry}") + aliases.setdefault(f"{to_module}{'.' if to_module else ''}{entry}", f"{python_sbom.__name__}.{entry}") return aliases -project: Final[str] = "hatch-pep770" +project: Final[str] = "python-sbom" _NOW: Final[datetime] = datetime.now(tz=UTC) globals()["copyright"] = f"2026-{_NOW.year}" version: Final[str] = __version__ @@ -48,7 +48,7 @@ def _inheritance_aliases() -> dict[str, str]: autosectionlabel_maxdepth: Final[int] = 2 html_theme: Final[str] = "furo" -html_title: Final[str] = "hatch-pep770" +html_title: Final[str] = "python-sbom" html_last_updated_fmt: Final[str] = _NOW.isoformat() html_static_path: Final[list[str]] = ["_static"] html_css_files: Final[list[str]] = ["changelog-toc.css"] @@ -67,7 +67,7 @@ def _inheritance_aliases() -> dict[str, str]: intersphinx_mapping: Final[dict[str, tuple[str, None]]] = {"python": ("https://docs.python.org/3.11", None)} extlinks: Final[dict[str, tuple[str, str]]] = { - "pr": ("https://github.com/tox-dev/hatch-pep770/pull/%s", "#%s"), + "pr": ("https://github.com/tox-dev/python-sbom/pull/%s", "#%s"), "user": ("https://github.com/%s", "@%s"), } nitpicky: Final[bool] = True diff --git a/docs/explanation.rst b/docs/explanation.rst index 35b97b9..f0c8bc5 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -39,7 +39,7 @@ Limitations depends on the environment the wheel is installed into. - The links to the PyPI release and attestation assume the project publishes to PyPI with attestations. - A build from an sdist has no git checkout, so its SBOM names no commit. -- ``hatch-pep770 embed`` records the environment it runs in. Unless the wheel was built without isolation in that +- ``python-sbom embed`` records the environment it runs in. Unless the wheel was built without isolation in that environment, the build tool record does not match the real build. - The SPDX rendering leaves out per-file hashes and the build record, which SPDX 2.3 cannot hold. - A hook that changes the wheel tag after this one runs leaves the attestation link pointing at the wrong file name. diff --git a/docs/extend.rst b/docs/extend.rst index cdff8ae..58aa1d6 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -8,7 +8,7 @@ pieces the hook uses. Add properties to bundled wheels -------------------------------- -:class:`~hatch_pep770.Pep770BuildHook` calls :meth:`~hatch_pep770.Pep770BuildHook.bundled_wheel_properties` once per +:class:`~python_sbom.SbomBuildHook` calls :meth:`~python_sbom.SbomBuildHook.bundled_wheel_properties` once per bundled wheel. Override it in a hatchling `custom build hook `_ to add properties. They follow ``:bundled-wheel`` and come before those taken from the wheel's metadata. The options then live in the custom hook's table, and the SBOM lists the script among the build tools, hashed, because the @@ -25,7 +25,7 @@ Python versions receive each wheel. Its ``pyproject.toml`` names the supplier an [build-system] build-backend = 'hatchling.build' - requires = ['hatch-pep770>=0.1; python_version>='3.11'', 'hatch-vcs>=0.5', 'hatchling>=1.28'] + requires = ['hatch-vcs>=0.5', 'hatchling>=1.28', 'python-sbom>=0.1'] [tool.hatch.build.hooks.custom] path = 'hatch_build.py' @@ -42,13 +42,13 @@ Python versions receive each wheel. Its ``pyproject.toml`` names the supplier an from pathlib import Path from typing import TYPE_CHECKING - from hatch_pep770 import Pep770BuildHook + from python_sbom import SbomBuildHook if TYPE_CHECKING: - from hatch_pep770 import Property + from python_sbom import Property - class SbomBuildHook(Pep770BuildHook): + class VirtualenvSbomHook(SbomBuildHook): def bundled_wheel_properties(self, wheel: Path) -> list[Property]: embed = Path(self.root) / 'src' / 'virtualenv' / 'seed' / 'wheels' / 'embed' / '__init__.py' tree = ast.parse(embed.read_text(encoding='utf-8')) @@ -62,8 +62,8 @@ Python versions receive each wheel. Its ``pyproject.toml`` names the supplier an ] - def get_build_hook() -> type[SbomBuildHook]: - return SbomBuildHook + def get_build_hook() -> type[VirtualenvSbomHook]: + return VirtualenvSbomHook The ``get_build_hook`` function tells hatchling which class to load, since the module also sees the base class it imports. The pip component of the wheel's SBOM then carries these properties, in this order: @@ -82,16 +82,17 @@ imports. The pip component of the wheel's SBOM then carries these properties, in * - ``python:requires-python`` - ``>=3.10`` -A project that still builds on a Python older than hatch-pep770 supports keeps the requirement's environment marker -and makes the script fall back to hatchling's plain hook there, so those builds produce a wheel without an SBOM: +A project that still builds on a Python older than python-sbom supports, as virtualenv does on 3.9 and 3.10, puts an +environment marker on the requirement, ``python-sbom>=0.1; python_version >= '3.11'``. Its script then falls back to +hatchling's plain hook there, so those builds produce a wheel without an SBOM: .. code-block:: python import sys if sys.version_info >= (3, 11): - from hatch_pep770 import Pep770BuildHook as BaseHook - else: # hatch-pep770 needs Python 3.11; older builds get no SBOM + from python_sbom import SbomBuildHook as BaseHook + else: # python-sbom needs Python 3.11; older builds get no SBOM from hatchling.builders.hooks.plugin.interface import BuildHookInterface as BaseHook Describe another artifact @@ -100,18 +101,18 @@ Describe another artifact virtualenv also publishes a zipapp. Its SBOM describes what the archive holds and records the same build toolchain as the wheel's. The functions the hook uses are public for that: -- :func:`~hatch_pep770.component_from_metadata` turns a distribution's ``METADATA`` into a component, with licenses, +- :func:`~python_sbom.component_from_metadata` turns a distribution's ``METADATA`` into a component, with licenses, project URLs and metadata properties. -- :func:`~hatch_pep770.build_tools` records the interpreter and every distribution of the running environment, with +- :func:`~python_sbom.build_tools` records the interpreter and every distribution of the running environment, with the files of the pure-Python ones and the dependencies between them. -- :func:`~hatch_pep770.timestamp` gives the time from ``SOURCE_DATE_EPOCH``. -- :func:`~hatch_pep770.document` adds the CycloneDX header, with a serial number derived from the content. +- :func:`~python_sbom.timestamp` gives the time from ``SOURCE_DATE_EPOCH``. +- :func:`~python_sbom.document` adds the CycloneDX header, with a serial number derived from the content. .. code-block:: python import uuid - from hatch_pep770 import Body, Component, build_tools, document, timestamp + from python_sbom import Body, Component, build_tools, document, timestamp tools, tool_dependencies = build_tools() zipapp: Component = {'type': 'application', 'bom-ref': 'pkg:generic/virtualenv.pyz@21.0', 'name': 'virtualenv.pyz'} @@ -128,20 +129,20 @@ the wheel's. The functions the hook uses are public for that: } sbom = document(body, uuid.uuid5(uuid.NAMESPACE_URL, 'https://github.com/pypa/virtualenv/sboms')) -:func:`~hatch_pep770.validate` checks the result the way ``hatch-pep770 validate`` checks a wheel's SBOM. +:func:`~python_sbom.validate` checks the result the way ``python-sbom validate`` checks a wheel's SBOM. Use it as a library ------------------- -:class:`~hatch_pep770.Project` holds the options and writes the SBOM of a wheel from its ``METADATA``, and -:func:`~hatch_pep770.embed` rewrites a built wheel with it. Pass ``bundled_wheel_properties`` to add properties to +:class:`~python_sbom.Project` holds the options and writes the SBOM of a wheel from its ``METADATA``, and +:func:`~python_sbom.embed` rewrites a built wheel with it. Pass ``bundled_wheel_properties`` to add properties to bundled wheels without a hatchling hook: .. code-block:: python from pathlib import Path - from hatch_pep770 import Project, embed + from python_sbom import Project, embed project = Project.from_table( {'bundled-wheels': ['src/project/wheels/*.whl']}, diff --git a/docs/index.rst b/docs/index.rst index 750ce82..c056ab5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,10 +1,10 @@ -hatch-pep770 -============ +python-sbom +=========== -hatch-pep770 writes a `CycloneDX `_ 1.6 SBOM into every wheel you build, at -``-.dist-info/sboms/.cdx.json``. That is where `PEP 770 `_ -tells installers to keep SBOMs, so they land in the environment next to the installed package, where scanners look for -them. +SBOMs for Python distributions. python-sbom writes a `CycloneDX `_ 1.6 SBOM into every wheel +you build, at ``-.dist-info/sboms/.cdx.json``. That is where `PEP 770 +`_ tells installers to keep SBOMs, so they land in the environment next to the +installed package, where scanners look for them. Why a wheel needs its own SBOM ------------------------------ @@ -24,12 +24,12 @@ enable it in ``pyproject.toml``: [build-system] build-backend = 'hatchling.build' - requires = ['hatch-pep770>=0.1', 'hatchling>=1.28'] + requires = ['python-sbom>=0.1', 'hatchling>=1.28'] - [tool.hatch.build.hooks.pep770] + [tool.hatch.build.hooks.python-sbom] Every wheel you build now carries the SBOM. For a project built with another backend, such as meson-python, run -``hatch-pep770 embed dist/*.whl`` after the build; :doc:`usage` shows both. +``python-sbom embed dist/*.whl`` after the build; :doc:`usage` shows both. .. toctree:: :maxdepth: 2 diff --git a/docs/py_api.rst b/docs/py_api.rst index 631dbef..9733bce 100644 --- a/docs/py_api.rst +++ b/docs/py_api.rst @@ -1,80 +1,80 @@ Python API ========== -Everything below is importable from ``hatch_pep770``. +Everything below is importable from ``python_sbom``. -.. autodata:: hatch_pep770.__version__ +.. autodata:: python_sbom.__version__ Build hook ---------- -.. autoclass:: hatch_pep770.Pep770BuildHook +.. autoclass:: python_sbom.SbomBuildHook :members: bundled_wheel_properties Wheels from any build backend ----------------------------- -.. autoclass:: hatch_pep770.Project +.. autoclass:: python_sbom.Project :members: from_table, describe :exclude-members: root, supplier, repository, default_branch, bundled_wheels, build_script, bundled_wheel_properties -.. autoclass:: hatch_pep770.BundledWheelProperties +.. autoclass:: python_sbom.BundledWheelProperties :members: __call__ -.. autofunction:: hatch_pep770.embed +.. autofunction:: python_sbom.embed Building blocks --------------- -.. autofunction:: hatch_pep770.component_from_metadata +.. autofunction:: python_sbom.component_from_metadata -.. autofunction:: hatch_pep770.build_tools +.. autofunction:: python_sbom.build_tools -.. autofunction:: hatch_pep770.timestamp +.. autofunction:: python_sbom.timestamp -.. autofunction:: hatch_pep770.document +.. autofunction:: python_sbom.document SPDX and validation ------------------- -.. autofunction:: hatch_pep770.to_spdx +.. autofunction:: python_sbom.to_spdx -.. autofunction:: hatch_pep770.validate +.. autofunction:: python_sbom.validate -.. autofunction:: hatch_pep770.validate_wheel +.. autofunction:: python_sbom.validate_wheel Types ----- These are :class:`~typing.TypedDict` types of the JSON documents, for type checking code that builds or reads them. -.. autoclass:: hatch_pep770.Document +.. autoclass:: python_sbom.Document :no-members: -.. autoclass:: hatch_pep770.Body +.. autoclass:: python_sbom.Body :no-members: -.. autoclass:: hatch_pep770.Component +.. autoclass:: python_sbom.Component :no-members: -.. autoclass:: hatch_pep770.Dependency +.. autoclass:: python_sbom.Dependency :no-members: -.. autoclass:: hatch_pep770.OrganizationalEntity +.. autoclass:: python_sbom.OrganizationalEntity :no-members: -.. autoclass:: hatch_pep770.Property +.. autoclass:: python_sbom.Property :no-members: -.. autoclass:: hatch_pep770.Spdx +.. autoclass:: python_sbom.Spdx :no-members: -.. autodata:: hatch_pep770.Toml +.. autodata:: python_sbom.Toml Command line ------------ -.. autofunction:: hatch_pep770.main +.. autofunction:: python_sbom.main -.. autoclass:: hatch_pep770.Options +.. autoclass:: python_sbom.Options :members: diff --git a/docs/usage.rst b/docs/usage.rst index c1bb4d3..7eeb658 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -23,7 +23,7 @@ Each option lives in the hook's table and is optional. .. code-block:: toml - [tool.hatch.build.hooks.pep770] + [tool.hatch.build.hooks.python-sbom] supplier = { name = 'Python Packaging Authority', url = ['https://www.pypa.io'] } ``repository`` @@ -32,7 +32,7 @@ Each option lives in the hook's table and is optional. .. code-block:: toml - [tool.hatch.build.hooks.pep770] + [tool.hatch.build.hooks.python-sbom] repository = 'https://gitlab.com/example/project' ``default-branch`` @@ -40,7 +40,7 @@ Each option lives in the hook's table and is optional. .. code-block:: toml - [tool.hatch.build.hooks.pep770] + [tool.hatch.build.hooks.python-sbom] default-branch = 'trunk' ``bundled-wheels`` @@ -49,7 +49,7 @@ Each option lives in the hook's table and is optional. .. code-block:: toml - [tool.hatch.build.hooks.pep770] + [tool.hatch.build.hooks.python-sbom] bundled-wheels = ['src/project/wheels/*.whl'] A bundled wheel whose project URLs point to the same forge account as ``repository`` gets the same ``supplier``. Its @@ -60,12 +60,12 @@ With another build backend -------------------------- A backend other than hatchling, such as `meson-python `_, cannot run the hook. -``hatch-pep770 embed`` writes the same document into a wheel that backend built, adds the ``RECORD`` row, and keeps -every other entry of the archive as it was. It reads the options above from a ``[tool.hatch-pep770]`` table: +``python-sbom embed`` writes the same document into a wheel that backend built, adds the ``RECORD`` row, and keeps +every other entry of the archive as it was. It reads the options above from a ``[tool.python-sbom]`` table: .. code-block:: toml - [tool.hatch-pep770] + [tool.python-sbom] supplier = { name = 'tox-dev', url = ['https://github.com/tox-dev'] } Build, then embed, from the project root: @@ -73,7 +73,7 @@ Build, then embed, from the project root: .. code-block:: console $ python -m build --wheel - $ hatch-pep770 embed dist/*.whl + $ python-sbom embed dist/*.whl wrote turbohtml-1.0.dist-info/sboms/turbohtml.cdx.json into turbohtml-1.0-cp314-cp314-linux_x86_64.whl The build tool record describes the environment ``embed`` runs in, not the isolated environment the backend built in. @@ -81,20 +81,20 @@ To record the build toolchain, build without isolation in the same locked enviro .. code-block:: console - $ python -m pip install hatch-pep770 meson-python meson ninja + $ python -m pip install python-sbom meson-python meson ninja $ python -m build --wheel --no-isolation - $ hatch-pep770 embed dist/*.whl + $ python-sbom embed dist/*.whl Render it as SPDX ----------------- -Some tools read only `SPDX `_. ``hatch-pep770 spdx`` renders the CycloneDX document as SPDX 2.3 +Some tools read only `SPDX `_. ``python-sbom spdx`` renders the CycloneDX document as SPDX 2.3 JSON: .. code-block:: console $ unzip -p dist/project-1.0-py3-none-any.whl '*/sboms/project.cdx.json' > project.cdx.json - $ hatch-pep770 spdx project.cdx.json project.spdx.json + $ python-sbom spdx project.cdx.json project.spdx.json The SPDX document keeps the packages, their hashes, licenses and purls, and the containment, dependency and build tool relationships. It leaves out the per-file hashes, because SPDX 2.3 wants a SHA-1 for every file and wheel ``RECORD`` @@ -104,14 +104,14 @@ number, so both documents name the same build. Check it in CI -------------- -``hatch-pep770 validate`` needs the ``validate`` extra. It checks each wheel's SBOM against the CycloneDX 1.6 schema, +``python-sbom validate`` needs the ``validate`` extra. It checks each wheel's SBOM against the CycloneDX 1.6 schema, requires the ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, checks that every reference resolves, and validates the SPDX rendering: .. code-block:: console - $ python -m pip install 'hatch-pep770[validate]' - $ hatch-pep770 validate dist/ --allow-license MIT --allow-license Apache-2.0 + $ python -m pip install 'python-sbom[validate]' + $ python-sbom validate dist/ --allow-license MIT --allow-license Apache-2.0 Each ``--allow-license`` adds an SPDX id that bundled wheels may use. With any given, a bundled wheel whose license the ids do not allow fails the check, and so does one that declares a license only by name. diff --git a/pyproject.toml b/pyproject.toml index 6398fff..70efbe4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,9 +7,9 @@ requires = [ ] [project] -name = "hatch-pep770" -import-names = [ "hatch_pep770" ] -description = "Write PEP 770 CycloneDX SBOMs into wheels, from a hatchling build hook or the command line." +name = "python-sbom" +import-names = [ "python_sbom" ] +description = "SBOMs for Python distributions: write PEP 770 CycloneDX SBOMs into wheels." keywords = [ "cyclonedx", "hatch", @@ -52,13 +52,13 @@ optional-dependencies.validate = [ "cyclonedx-python-lib[json-validation]>=11.12", "spdx-tools>=0.8.5", ] -urls.Changelog = "https://hatch-pep770.readthedocs.io/en/latest/changelog.html" -urls.Documentation = "https://hatch-pep770.readthedocs.io" -urls.Homepage = "https://github.com/tox-dev/hatch-pep770" -urls.Source = "https://github.com/tox-dev/hatch-pep770" -urls.Tracker = "https://github.com/tox-dev/hatch-pep770/issues" -scripts.hatch-pep770 = "hatch_pep770.__main__:main" -entry-points.hatch.pep770 = "hatch_pep770.hooks" +urls.Changelog = "https://python-sbom.readthedocs.io/en/latest/changelog.html" +urls.Documentation = "https://python-sbom.readthedocs.io" +urls.Homepage = "https://github.com/tox-dev/python-sbom" +urls.Source = "https://github.com/tox-dev/python-sbom" +urls.Tracker = "https://github.com/tox-dev/python-sbom/issues" +scripts.python-sbom = "python_sbom.__main__:main" +entry-points.hatch.python-sbom = "python_sbom.hooks" [dependency-groups] dev = [ @@ -111,7 +111,7 @@ release-prepare = [ version.source = "vcs" metadata.hooks.fancy-pypi-readme.content-type = "text/markdown" metadata.hooks.fancy-pypi-readme.fragments = [ { path = "README.md" } ] -build.hooks.vcs.version-file = "src/hatch_pep770/version.py" +build.hooks.vcs.version-file = "src/python_sbom/version.py" [tool.uv] dependency-groups.release-news = { requires-python = ">=3.15" } @@ -149,7 +149,7 @@ lint.per-file-ignores."tests/**/*.py" = [ "too-many-positional-arguments", # Test inputs favor explicit parameters over setup objects ] lint.isort = { known-first-party = [ - "hatch_pep770", + "python_sbom", ], required-imports = [ "from __future__ import annotations", ] } @@ -192,7 +192,7 @@ html.show_contexts = true html.skip_covered = false [tool.towncrier] -name = "hatch-pep770" +name = "python-sbom" directory = "docs/changelog.d" filename = "docs/changelog.rst" template = "release_creator.fragments:template.rst.jinja2" diff --git a/src/hatch_pep770/__init__.py b/src/python_sbom/__init__.py similarity index 84% rename from src/hatch_pep770/__init__.py rename to src/python_sbom/__init__.py index b8d369b..6a40641 100644 --- a/src/hatch_pep770/__init__.py +++ b/src/python_sbom/__init__.py @@ -1,4 +1,4 @@ -"""Write PEP 770 CycloneDX SBOMs into wheels, from a hatchling build hook or the command line.""" +"""SBOMs for Python distributions: write PEP 770 CycloneDX SBOMs into wheels.""" from __future__ import annotations @@ -16,7 +16,7 @@ timestamp, ) from ._embed import embed -from ._hook import Pep770BuildHook +from ._hook import SbomBuildHook from ._project import BundledWheelProperties, Project, Toml from ._spdx import Spdx, to_spdx from ._validate import validate, validate_wheel @@ -30,9 +30,9 @@ "Document", "Options", "OrganizationalEntity", - "Pep770BuildHook", "Project", "Property", + "SbomBuildHook", "Spdx", "Toml", "__version__", diff --git a/src/hatch_pep770/__main__.py b/src/python_sbom/__main__.py similarity index 100% rename from src/hatch_pep770/__main__.py rename to src/python_sbom/__main__.py diff --git a/src/hatch_pep770/_cli.py b/src/python_sbom/_cli.py similarity index 90% rename from src/hatch_pep770/_cli.py rename to src/python_sbom/_cli.py index 006d087..d4460f5 100644 --- a/src/hatch_pep770/_cli.py +++ b/src/python_sbom/_cli.py @@ -25,8 +25,8 @@ def main(args: Sequence[str] | None = None) -> None: def _build_parser() -> ArgumentParser: parser = ArgumentParser( - prog="hatch-pep770", - description="Write, convert and check PEP 770 CycloneDX SBOMs of wheels.", + prog="python-sbom", + description="SBOMs for Python distributions: write, convert and check the PEP 770 CycloneDX SBOMs of wheels.", formatter_class=RawDescriptionHelpFormatter, epilog=f"running {version} at {Path(__file__).parent}", ) @@ -35,7 +35,7 @@ def _build_parser() -> ArgumentParser: "embed", help="write the SBOM into wheels another build backend built", description="Write the SBOM the hatchling hook would write into wheels another build backend built, such as" - " meson-python. Settings come from the [tool.hatch-pep770] table of the project's pyproject.toml.", + " meson-python. Settings come from the [tool.python-sbom] table of the project's pyproject.toml.", ) embed.add_argument("wheels", type=Path, nargs="+", metavar="wheel", help="wheel to rewrite in place") embed.add_argument( @@ -48,7 +48,7 @@ def _build_parser() -> ArgumentParser: spdx.add_argument("source", type=Path, help="CycloneDX JSON file to read") spdx.add_argument("target", type=Path, help="SPDX JSON file to write") validate = commands.add_parser( - "validate", help="validate the SBOMs in wheels", description="Needs the validate extra: hatch-pep770[validate]." + "validate", help="validate the SBOMs in wheels", description="Needs the validate extra: python-sbom[validate]." ) validate.add_argument("paths", type=Path, nargs="+", metavar="path", help="wheel, or directory holding wheels") validate.add_argument( diff --git a/src/hatch_pep770/_cyclonedx.py b/src/python_sbom/_cyclonedx.py similarity index 100% rename from src/hatch_pep770/_cyclonedx.py rename to src/python_sbom/_cyclonedx.py diff --git a/src/hatch_pep770/_embed.py b/src/python_sbom/_embed.py similarity index 100% rename from src/hatch_pep770/_embed.py rename to src/python_sbom/_embed.py diff --git a/src/hatch_pep770/_extra.py b/src/python_sbom/_extra.py similarity index 100% rename from src/hatch_pep770/_extra.py rename to src/python_sbom/_extra.py diff --git a/src/hatch_pep770/_hook.py b/src/python_sbom/_hook.py similarity index 91% rename from src/hatch_pep770/_hook.py rename to src/python_sbom/_hook.py index 7ac1677..86b3812 100644 --- a/src/hatch_pep770/_hook.py +++ b/src/python_sbom/_hook.py @@ -17,16 +17,16 @@ from ._cyclonedx import Property -class Pep770BuildHook(BuildHookInterface): +class SbomBuildHook(BuildHookInterface): """ Hatchling build hook that writes a PEP 770 CycloneDX SBOM into the wheel's ``.dist-info/sboms/``. - Enable it with a ``[tool.hatch.build.hooks.pep770]`` table, or subclass it in a hatchling custom build hook to add - project-specific detail through :meth:`bundled_wheel_properties`. + Enable it with a ``[tool.hatch.build.hooks.python-sbom]`` table, or subclass it in a hatchling custom build hook to + add project-specific detail through :meth:`bundled_wheel_properties`. """ - PLUGIN_NAME = "pep770" + PLUGIN_NAME = "python-sbom" _sbom_directory: Path | None = None def initialize( @@ -62,7 +62,7 @@ def initialize( content = json.dumps(project.describe(metadata, f"{builder.artifact_project_id}-{tag}.whl"), indent=2) # written outside self.directory, which is also the wheel's output directory, so no stray copy lands next to # the built artifacts - self._sbom_directory = Path(tempfile.mkdtemp(prefix="hatch-pep770-")) + self._sbom_directory = Path(tempfile.mkdtemp(prefix="python-sbom-")) # bytes, so the line endings do not depend on the build machine's OS (sbom := self._sbom_directory / sbom_file_name(metadata)).write_bytes(f"{content}\n".encode()) build_data["sbom_files"].append(str(sbom)) @@ -103,5 +103,5 @@ def finalize( __all__ = [ - "Pep770BuildHook", + "SbomBuildHook", ] diff --git a/src/hatch_pep770/_project.py b/src/python_sbom/_project.py similarity index 99% rename from src/hatch_pep770/_project.py rename to src/python_sbom/_project.py index 1d1c865..1052477 100644 --- a/src/hatch_pep770/_project.py +++ b/src/python_sbom/_project.py @@ -104,7 +104,7 @@ def from_table( :param table: the ``supplier``, ``repository``, ``default-branch`` and ``bundled-wheels`` settings :param root: the project directory - :param where: where the table lives, for error messages, such as ``[tool.hatch-pep770]`` + :param where: where the table lives, for error messages, such as ``[tool.python-sbom]`` :param build_script: path, relative to ``root``, of a build script to list among the build tools :param bundled_wheel_properties: extra properties for each bundled wheel :returns: the project diff --git a/src/hatch_pep770/_run.py b/src/python_sbom/_run.py similarity index 94% rename from src/hatch_pep770/_run.py rename to src/python_sbom/_run.py index 21553a9..b494965 100644 --- a/src/hatch_pep770/_run.py +++ b/src/python_sbom/_run.py @@ -17,8 +17,8 @@ def run(options: Options) -> bool: if options.command == "embed": pyproject = options.root / "pyproject.toml" - table = tomllib.loads(pyproject.read_text(encoding="utf-8")).get("tool", {}).get("hatch-pep770", {}) - project = Project.from_table(table, root=options.root, where=f"[tool.hatch-pep770] of {pyproject}") + table = tomllib.loads(pyproject.read_text(encoding="utf-8")).get("tool", {}).get("python-sbom", {}) + project = Project.from_table(table, root=options.root, where=f"[tool.python-sbom] of {pyproject}") for wheel in options.wheels: sys.stdout.write(f"wrote {embed(wheel, project)} into {wheel.name}\n") return True diff --git a/src/hatch_pep770/_spdx.py b/src/python_sbom/_spdx.py similarity index 99% rename from src/hatch_pep770/_spdx.py rename to src/python_sbom/_spdx.py index 3da6484..665fd95 100644 --- a/src/hatch_pep770/_spdx.py +++ b/src/python_sbom/_spdx.py @@ -111,7 +111,7 @@ def to_spdx(bom: Document) -> Spdx: "creators": [ *supplier, *(f"Tool: {tool['name']}-{tool['version']}" for tool in tools if tool["type"] == "application"), - f"Tool: hatch-pep770-{version}", + f"Tool: python-sbom-{version}", ], }, "packages": [_package(component, f"{namespace}:") for component in components], diff --git a/src/hatch_pep770/_validate.py b/src/python_sbom/_validate.py similarity index 100% rename from src/hatch_pep770/_validate.py rename to src/python_sbom/_validate.py diff --git a/src/hatch_pep770/hooks.py b/src/python_sbom/hooks.py similarity index 64% rename from src/hatch_pep770/hooks.py rename to src/python_sbom/hooks.py index a5a51db..ed86134 100644 --- a/src/hatch_pep770/hooks.py +++ b/src/python_sbom/hooks.py @@ -4,15 +4,15 @@ from hatchling.plugin import hookimpl -from ._hook import Pep770BuildHook +from ._hook import SbomBuildHook @hookimpl -def hatch_register_build_hook() -> type[Pep770BuildHook]: +def hatch_register_build_hook() -> type[SbomBuildHook]: """ - Return the build hook hatchling loads for ``[tool.hatch.build.hooks.pep770]``. + Return the build hook hatchling loads for ``[tool.hatch.build.hooks.python-sbom]``. :returns: the build hook class """ - return Pep770BuildHook + return SbomBuildHook diff --git a/src/hatch_pep770/py.typed b/src/python_sbom/py.typed similarity index 100% rename from src/hatch_pep770/py.typed rename to src/python_sbom/py.typed diff --git a/tests/conftest.py b/tests/conftest.py index 84d09d9..b0b1c1d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,10 +13,10 @@ import pytest from hatchling.builders.wheel import WheelBuilder -from hatch_pep770 import Project +from python_sbom import Project if TYPE_CHECKING: - from hatch_pep770 import Document + from python_sbom import Document _PROJECT: Final[str] = """\ [project] @@ -34,7 +34,7 @@ urls.Changelog = "https://example.com/changelog" urls.Source = "https://github.com/example/example" """ -HOOK: Final[str] = "[tool.hatch.build.hooks.pep770]\n" +HOOK: Final[str] = "[tool.hatch.build.hooks.python-sbom]\n" BUNDLED: Final[str] = 'bundled-wheels = ["src/example/wheels/*.whl"]\n' _SEEDER: Final[Path] = Path(__file__).parent / "seeder" @@ -189,4 +189,4 @@ def install(site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> No @pytest.fixture def plain_project(tmp_path: Path) -> Project: - return Project.from_table({}, root=tmp_path, where="[tool.hatch-pep770]") + return Project.from_table({}, root=tmp_path, where="[tool.python-sbom]") diff --git a/tests/seeder/hatch_build.py b/tests/seeder/hatch_build.py index fa93007..38fcda9 100644 --- a/tests/seeder/hatch_build.py +++ b/tests/seeder/hatch_build.py @@ -4,10 +4,10 @@ from pathlib import Path from typing import TYPE_CHECKING -from hatch_pep770 import Pep770BuildHook +from python_sbom import SbomBuildHook if TYPE_CHECKING: - from hatch_pep770 import Property + from python_sbom import Property def get_build_hook() -> type[SeederBuildHook]: @@ -15,7 +15,7 @@ def get_build_hook() -> type[SeederBuildHook]: return SeederBuildHook -class SeederBuildHook(Pep770BuildHook): +class SeederBuildHook(SbomBuildHook): def bundled_wheel_properties(self, wheel: Path) -> list[Property]: # the table that decides which Python versions receive each wheel, read without importing the package tree = ast.parse((Path(self.root) / "src" / "seeder" / "embed" / "bundle.py").read_text(encoding="utf-8")) diff --git a/tests/seeder/pyproject.toml b/tests/seeder/pyproject.toml index 8b063c9..338f2af 100644 --- a/tests/seeder/pyproject.toml +++ b/tests/seeder/pyproject.toml @@ -1,6 +1,6 @@ [build-system] build-backend = "hatchling.build" -requires = [ "hatch-pep770", "hatchling>=1.28" ] +requires = [ "hatchling>=1.28", "python-sbom" ] [project] name = "seeder" diff --git a/tests/test_cli.py b/tests/test_cli.py index 2c29892..b17a2b7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,19 +6,19 @@ import pytest -from hatch_pep770 import main, to_spdx +from python_sbom import main, to_spdx if TYPE_CHECKING: from pathlib import Path from conftest import BuildSbom, MakeWheel - from hatch_pep770 import Document + from python_sbom import Document def test_cli_embed(make_wheel: MakeWheel, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: wheel: Final[Path] = make_wheel("") - (tmp_path / "pyproject.toml").write_text('[tool.hatch-pep770]\nsupplier = { name = "Org" }\n', encoding="utf-8") + (tmp_path / "pyproject.toml").write_text('[tool.python-sbom]\nsupplier = { name = "Org" }\n', encoding="utf-8") main(["embed", str(wheel), "--root", str(tmp_path)]) with zipfile.ZipFile(wheel) as archive: document: Final[Document] = json.loads(archive.read("native-1.0.dist-info/sboms/native.cdx.json")) diff --git a/tests/test_embed.py b/tests/test_embed.py index 9320fe4..f1cc3d4 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -12,12 +12,12 @@ from installer.destinations import SchemeDictionaryDestination from installer.sources import WheelFile -from hatch_pep770 import Project, embed, validate_wheel +from python_sbom import Project, embed, validate_wheel if TYPE_CHECKING: from conftest import MakeWheel - from hatch_pep770 import Document + from python_sbom import Document _METADATA: Final[str] = "License-Expression: MIT\nProject-URL: Source, https://github.com/example/native\n" @@ -103,7 +103,7 @@ def test_embed_survives_install(make_wheel: MakeWheel, plain_project: Project, t def test_embed_uses_project_settings(make_wheel: MakeWheel, tmp_path: Path) -> None: wheel: Final[Path] = make_wheel(_METADATA) project: Final[Project] = Project.from_table( - {"supplier": {"name": "Org"}}, root=tmp_path, where="[tool.hatch-pep770]" + {"supplier": {"name": "Org"}}, root=tmp_path, where="[tool.python-sbom]" ) name: Final[str] = embed(wheel, project) with zipfile.ZipFile(wheel) as archive: @@ -116,7 +116,7 @@ def test_embed_bundled_wheel(make_wheel: MakeWheel, tmp_path: Path) -> None: seed: Final[Path] = Path(__file__).parent / "seeder" / "src" / "seeder" / "embed" / "pip-1.0-py3-none-any.whl" shutil.copyfile(seed, tmp_path / seed.name) project: Final[Project] = Project.from_table( - {"bundled-wheels": ["*.whl"]}, root=tmp_path, where="[tool.hatch-pep770]" + {"bundled-wheels": ["*.whl"]}, root=tmp_path, where="[tool.python-sbom]" ) name: Final[str] = embed(wheel, project) with zipfile.ZipFile(wheel) as archive: diff --git a/tests/test_hook.py b/tests/test_hook.py index 6adbbf4..f61daf2 100644 --- a/tests/test_hook.py +++ b/tests/test_hook.py @@ -11,12 +11,12 @@ from hatchling.builders.sdist import SdistBuilder from hatchling.builders.wheel import WheelBuilder -from hatch_pep770 import validate_wheel +from python_sbom import validate_wheel if TYPE_CHECKING: from conftest import BundleWheel, WriteProject - from hatch_pep770 import Document + from python_sbom import Document def test_wheel_embeds_sbom(write_project: WriteProject, tmp_path: Path) -> None: @@ -99,8 +99,8 @@ def test_custom_build_script_recorded( root: Path, write_project: WriteProject, urls: str, references: dict[str, list[dict[str, str]]] ) -> None: (script := root / "build.py").write_text( - "from __future__ import annotations\n\nfrom hatch_pep770 import Pep770BuildHook\n\n\n" - "def get_build_hook() -> type[Pep770BuildHook]:\n return Pep770BuildHook\n", + "from __future__ import annotations\n\nfrom python_sbom import SbomBuildHook\n\n\n" + "def get_build_hook() -> type[SbomBuildHook]:\n return SbomBuildHook\n", encoding="utf-8", ) project = f'[project]\nname = "example"\nversion = "1.0"\n{urls}\n' diff --git a/tests/test_main.py b/tests/test_main.py index 8b1a328..ec363b1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -10,12 +10,12 @@ def test_main(capfd: pytest.CaptureFixture[str]) -> None: - subprocess.check_call([sys.executable, "-m", "hatch_pep770", "-h"]) + subprocess.check_call([sys.executable, "-m", "python_sbom", "-h"]) out, _err = capfd.readouterr() - assert out.startswith("usage: hatch-pep770 ") + assert out.startswith("usage: python-sbom ") def test_script(capfd: pytest.CaptureFixture[str]) -> None: - subprocess.check_call([Path(sys.executable).parent / "hatch-pep770", "-h"]) + subprocess.check_call([Path(sys.executable).parent / "python-sbom", "-h"]) out, _err = capfd.readouterr() - assert out.startswith("usage: hatch-pep770 ") + assert out.startswith("usage: python-sbom ") diff --git a/tests/test_project.py b/tests/test_project.py index ce2c4ec..f50f14e 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -17,7 +17,7 @@ from conftest import BuildSbom, BundleWheel, InstallTool, WriteProject from pytest_mock import MockerFixture - from hatch_pep770 import Component, Document + from python_sbom import Component, Document from conftest import BUNDLED, HOOK, build_hook_sbom @@ -534,11 +534,15 @@ def test_serial_number_per_project(build_sbom: BuildSbom) -> None: @pytest.mark.parametrize( ("hook", "message"), [ - pytest.param(f'{HOOK}supplier = "Org"\n', "`supplier` in build hook `pep770` must be a table", id="supplier"), + pytest.param( + f'{HOOK}supplier = "Org"\n', "`supplier` in build hook `python-sbom` must be a table", id="supplier" + ), pytest.param(f'{HOOK}supplier = {{ name = "Org", email = "x" }}\n', "`supplier`", id="supplier-unknown-key"), - pytest.param(f"{HOOK}repository = 1\n", "`repository` in build hook `pep770` must be a string", id="repo"), - pytest.param(f"{HOOK}default-branch = 1\n", "`default-branch` in build hook `pep770` must be", id="branch"), - pytest.param(f'{HOOK}bundled-wheels = "*.whl"\n', "`bundled-wheels` in build hook `pep770`", id="bundled"), + pytest.param(f"{HOOK}repository = 1\n", "`repository` in build hook `python-sbom` must be a string", id="repo"), + pytest.param( + f"{HOOK}default-branch = 1\n", "`default-branch` in build hook `python-sbom` must be", id="branch" + ), + pytest.param(f'{HOOK}bundled-wheels = "*.whl"\n', "`bundled-wheels` in build hook `python-sbom`", id="bundled"), ], ) def test_invalid_option(build_sbom: BuildSbom, hook: str, message: str) -> None: diff --git a/tests/test_spdx.py b/tests/test_spdx.py index 7feb0a3..bface45 100644 --- a/tests/test_spdx.py +++ b/tests/test_spdx.py @@ -5,14 +5,14 @@ import pytest from conftest import BUNDLED, HOOK -from hatch_pep770 import __version__, to_spdx +from python_sbom import __version__, to_spdx if TYPE_CHECKING: from pathlib import Path from conftest import BuildSbom, BundleWheel - from hatch_pep770 import Document + from python_sbom import Document _SUPPLIED: Final[str] = f'{HOOK}{BUNDLED}supplier = {{ name = "Example Org" }}\n' @@ -163,7 +163,7 @@ def test_spdx_creation_info( ) -> None: monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") (root / "build.py").write_text( - "from hatch_pep770 import Pep770BuildHook\n\n\ndef get_build_hook():\n return Pep770BuildHook\n", + "from python_sbom import SbomBuildHook\n\n\ndef get_build_hook():\n return SbomBuildHook\n", encoding="utf-8", ) cyclonedx: Final[Document] = build_sbom(hook=hook) @@ -171,7 +171,7 @@ def test_spdx_creation_info( namespace, creators = expected assert (spdx["documentNamespace"], spdx["creationInfo"], spdx["comment"]) == ( f"{namespace}{cyclonedx['serialNumber'].removeprefix('urn:uuid:')}", - {"created": "2023-11-14T22:13:20Z", "creators": [*creators, f"Tool: hatch-pep770-{__version__}"]}, + {"created": "2023-11-14T22:13:20Z", "creators": [*creators, f"Tool: python-sbom-{__version__}"]}, ( f"Rendered from the CycloneDX SBOM {cyclonedx['serialNumber']} (example.cdx.json), which also carries" " per-file hashes and the build formulation." diff --git a/tests/test_validate.py b/tests/test_validate.py index 3b18969..0948826 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -6,7 +6,7 @@ import pytest -from hatch_pep770 import validate, validate_wheel +from python_sbom import validate, validate_wheel if TYPE_CHECKING: from pathlib import Path @@ -14,8 +14,8 @@ from conftest import BuildSbom from pytest_mock import MockerFixture - from hatch_pep770 import Component, Document - from hatch_pep770._cyclonedx import License + from python_sbom import Component, Document + from python_sbom._cyclonedx import License _ALLOWED: Final[frozenset[str]] = frozenset({"Apache-2.0", "BSD-3-Clause", "MIT"}) @@ -39,7 +39,7 @@ def _denied(expression: str, bom_ref: str = "pkg:pypi/bundled@1.0") -> str: @pytest.fixture def sbom_document(build_sbom: BuildSbom, mocker: MockerFixture) -> Document: # schema validation time grows with the document, and the test environment's packages are irrelevant here - mocker.patch("hatch_pep770._cyclonedx.distributions", autospec=True, return_value=[]) + mocker.patch("python_sbom._cyclonedx.distributions", autospec=True, return_value=[]) return build_sbom() diff --git a/tests/test_version.py b/tests/test_version.py index 1436630..4730451 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -2,7 +2,7 @@ from packaging.version import Version -from hatch_pep770 import __version__ +from python_sbom import __version__ def test_version() -> None: diff --git a/tox.toml b/tox.toml index 75ba15b..ed5773e 100644 --- a/tox.toml +++ b/tox.toml @@ -17,7 +17,7 @@ commands = [ { replace = "posargs", default = [ "--no-cov-on-fail", "--cov", - "hatch_pep770", + "python_sbom", "--cov", "{tox_root}{/}tests", "--cov-config", From 993f1a7f7e0dea67246268838d060a4108177b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 20:43:32 -0800 Subject: [PATCH 009/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20hash=20absolute?= =?UTF-8?q?=20build=20scripts=20on=20Windows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Join the build script path as a string so a drive letter survives on Windows, and admit simplint 0.7.1 past the seven-day cutoff so the cooled hook job can install it. --- pyproject.toml | 3 +++ src/python_sbom/_project.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 70efbe4..ca503d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -221,3 +221,6 @@ matrix.platforms = [ "linux_x86_64" ] # release-creator 4.4.0 needs a seven-day cutoff exception. # expires 2026-11-09 packages.release-creator.uploaded-prior-to = 2026-11-02T21:45:00Z +# simplint 0.7.1, the pre-commit hook, needs a seven-day cutoff exception. +# expires 2026-11-13 +packages.simplint.uploaded-prior-to = 2026-11-06T21:30:00Z diff --git a/src/python_sbom/_project.py b/src/python_sbom/_project.py index 1052477..13fe8d9 100644 --- a/src/python_sbom/_project.py +++ b/src/python_sbom/_project.py @@ -386,7 +386,10 @@ def _build_script_tool(self, script: PurePosixPath, version: str, repository: st "bom-ref": f"tool:{script}", "name": script.name, "version": version, - "hashes": [{"alg": "SHA-256", "content": hashlib.sha256((self.root / script).read_bytes()).hexdigest()}], + # str, so a Windows drive in an absolute path survives the join with the native root + "hashes": [ + {"alg": "SHA-256", "content": hashlib.sha256((self.root / str(script)).read_bytes()).hexdigest()} + ], } if repository: tool["externalReferences"] = [{"type": "vcs", "url": f"{repository}/blob/{self.default_branch}/{script}"}] From 112030036e857dcfb2d2bcd81181882474f1fd9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 20:46:39 -0800 Subject: [PATCH 010/115] =?UTF-8?q?=F0=9F=91=B7=20ci:=20trim=20the=20check?= =?UTF-8?q?=20matrix=20to=2013=20jobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pure-Python package needs every supported Python on Linux, the oldest and newest on Windows, the newest on macOS, and the tooling once. ty checks every platform in its single job instead. --- .github/workflows/check.yaml | 21 ++------------------- tox.toml | 2 +- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index ada78bd..f74b8b8 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -22,36 +22,19 @@ jobs: with: include: | [ - {"env": "3.15t", "python": "3.15t"}, {"env": "3.15", "python": "3.15"}, - {"env": "3.14t", "python": "3.14t"}, {"env": "3.14", "python": "3.14"}, {"env": "3.13", "python": "3.13"}, {"env": "3.12", "python": "3.12"}, {"env": "3.11", "python": "3.11"}, - {"env": "3.15t", "python": "3.15t", "os": "macos-26"}, - {"env": "3.15", "python": "3.15", "os": "macos-26"}, - {"env": "3.14t", "python": "3.14t", "os": "macos-26"}, - {"env": "3.14", "python": "3.14", "os": "macos-26"}, - {"env": "3.13", "python": "3.13", "os": "macos-26"}, - {"env": "3.12", "python": "3.12", "os": "macos-26"}, - {"env": "3.11", "python": "3.11", "os": "macos-26"}, - {"env": "3.15t", "python": "3.15t", "os": "windows-2025"}, {"env": "3.15", "python": "3.15", "os": "windows-2025"}, - {"env": "3.14t", "python": "3.14t", "os": "windows-2025"}, - {"env": "3.14", "python": "3.14", "os": "windows-2025"}, - {"env": "3.13", "python": "3.13", "os": "windows-2025"}, - {"env": "3.12", "python": "3.12", "os": "windows-2025"}, {"env": "3.11", "python": "3.11", "os": "windows-2025"}, + {"env": "3.15", "python": "3.15", "os": "macos-26"}, {"env": "dev"}, - {"env": "dev", "os": "windows-2025"}, {"env": "docs"}, - {"env": "docs", "os": "windows-2025"}, {"env": "fix"}, {"env": "type"}, - {"env": "type", "os": "windows-2025"}, - {"env": "pkg_meta"}, - {"env": "pkg_meta", "os": "windows-2025"} + {"env": "pkg_meta"} ] release: name: ๐Ÿท๏ธ Release dry run diff --git a/tox.toml b/tox.toml index ed5773e..177bca1 100644 --- a/tox.toml +++ b/tox.toml @@ -117,7 +117,7 @@ commands = [ [env.type] description = "run type check on code base" dependency_groups = [ "type" ] -commands = [ [ "ty", "check", "--output-format", "concise", "--error-on-warning", "." ] ] +commands = [ [ "ty", "check", "--output-format", "concise", "--error-on-warning", "--python-platform", "all", "." ] ] [env.dev] description = "dev environment with all deps at {envdir}" From 2c0a8997abda5749601e1b615f16bd153723de1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 20:49:34 -0800 Subject: [PATCH 011/115] =?UTF-8?q?=E2=9C=85=20test:=20build=20the=20bundl?= =?UTF-8?q?ed=20seed=20wheel=20at=20test=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The committed wheel tripped Scorecard's Binary-Artifacts check and could not be reviewed; a fixture now writes the same small wheel with fixed timestamps, so its hashes stay stable. --- tests/conftest.py | 20 ++++++++++++++++++ .../src/seeder/embed/pip-1.0-py3-none-any.whl | Bin 1080 -> 0 bytes tests/test_embed.py | 6 ++---- 3 files changed, 22 insertions(+), 4 deletions(-) delete mode 100644 tests/seeder/src/seeder/embed/pip-1.0-py3-none-any.whl diff --git a/tests/conftest.py b/tests/conftest.py index b0b1c1d..951c55b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -111,9 +111,29 @@ def seeder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: ), encoding="utf-8", ) + write_seed_wheel(target / "src" / "seeder" / "embed") return target +def write_seed_wheel(directory: Path) -> Path: + # the pip wheel a seeder bundles, built here rather than committed so the repository holds no binaries + files: Final[dict[str, bytes]] = { + "pip/__init__.py": b"", + "pip/_vendor/vendor.txt": b"urllib3==1.26.4\n", + "pip-1.0.dist-info/METADATA": ( + b"Metadata-Version: 2.4\nName: pip\nVersion: 1.0\nLicense-Expression: MIT\n" + b"Project-URL: Source, https://github.com/example/pip\n" + ), + "pip-1.0.dist-info/WHEEL": b"Wheel-Version: 1.0\nGenerator: tests\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + } + record = "".join(f"{name},{digest(content)},{len(content)}\n" for name, content in files.items()) + files["pip-1.0.dist-info/RECORD"] = f"{record}pip-1.0.dist-info/RECORD,,\n".encode() + with zipfile.ZipFile(wheel := directory / "pip-1.0-py3-none-any.whl", "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in files.items(): + archive.writestr(zipfile.ZipInfo(name, (2020, 2, 2, 0, 0, 0)), content, zipfile.ZIP_DEFLATED) + return wheel + + @pytest.fixture def bundle_wheel(root: Path) -> BundleWheel: def bundle(files: dict[str, str], metadata: str = "", name: str = "pip") -> Path: diff --git a/tests/seeder/src/seeder/embed/pip-1.0-py3-none-any.whl b/tests/seeder/src/seeder/embed/pip-1.0-py3-none-any.whl deleted file mode 100644 index b34215e4fc575672818da0f02df4dee7e7e1359a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1080 zcmWIWW@Zs#U|`??Vy6Hg!N3F}fPf!}3o;Az8!n2&Gq2QKGGT>f}iuqitJFuNWI$Hh##2X38s_m`Qm+-IYKr1u;d} zP|rXwC9}9hH#09SU*Fd?#L>ku#PPGQuBVTuj^`C#Z(Xf(XU=a9GPq>?@RRo$->pF! zCv+Y(aZA6FEcltCw)~ms@>jdUL`0T~IUlLDb>6|$^=Wg^WbYE^=9)DkO`pBHCk2;^ z7d1!f*MhAJmVv^@ABe@Vx-#6u)zydf zpdnX-0T1hiy~|&)Ds@HcDdZ_FDqbQlYBF()?4G^vbJYtjbDf{Ad3;^wVTM+3!{-%U z%iM(y<%DINF$q0h`&Onqyv3ET@BQ`3vzlD?e_%!p@S9~**Z%-o?#sx)Ac55pL9WjJ zK`yyxc1~(CU|_g#FQ0Qcx5q|SCx+NS$0<|WIva$}c{E+W{@%j6By4>(i?E_+L3iY& z4^KEV8dF^V{=1~}Vsg>5bLAc0hg^jVPhC30ZqL8UEwVaA^60rOH+a9vbqIg*%d4DS zy7vSBdAXcDGJD?pZF(lpfBfH6y+^xwtMh;Vc`L+xYx%D=Gj%=7Q^N8!CtN8%=DFuY z`V1q!?a%!_BxFRmW%MZahONB1=Jms})V4`Sy*Pt^YQB1SR+8`Y!8xY4dEK5w`Dji^ zd3ji3r@*>nep3Sj4WGQ&^4M-HV-aa{9hKx9m2p>b_U)aM?bG(Zsxw z^}rMU0B=Sn5oX-!85m?>uml)>7^xdwGkWTVXk}nn!l)0^jFHyS%|K7v2s26{X5dKa y=;olOK7={aK(7F^DVjOpw1{p9dg?(KvH+_gkhB!w&B_K+$qIxaKy~N91q1-cq*8eR diff --git a/tests/test_embed.py b/tests/test_embed.py index f1cc3d4..0fed65a 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -1,13 +1,12 @@ from __future__ import annotations import json -import shutil import sys import zipfile from pathlib import Path from typing import TYPE_CHECKING, Final -from conftest import digest +from conftest import digest, write_seed_wheel from installer import install from installer.destinations import SchemeDictionaryDestination from installer.sources import WheelFile @@ -113,8 +112,7 @@ def test_embed_uses_project_settings(make_wheel: MakeWheel, tmp_path: Path) -> N def test_embed_bundled_wheel(make_wheel: MakeWheel, tmp_path: Path) -> None: wheel: Final[Path] = make_wheel(_METADATA) - seed: Final[Path] = Path(__file__).parent / "seeder" / "src" / "seeder" / "embed" / "pip-1.0-py3-none-any.whl" - shutil.copyfile(seed, tmp_path / seed.name) + write_seed_wheel(tmp_path) project: Final[Project] = Project.from_table( {"bundled-wheels": ["*.whl"]}, root=tmp_path, where="[tool.python-sbom]" ) From f91c1c9069f97d35ebc69b94f53d43d50b944293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 21:11:59 -0800 Subject: [PATCH 012/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20split?= =?UTF-8?q?=20the=20core=20from=20the=20hatch=20backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The core (collect, model, formats, wheel, validate, cli) no longer imports hatchling; the build hook moves to python_sbom.hatch behind a Backend ABC, with hatchling as the hatch extra. Build facts are gathered once at the edge and passed in. Also fixes five review findings: unique SPDX ids for conditional requirements, distinct refs for bundled wheel variants, marker-aware runtime dependencies, structural validation before deeper checks, and SPDX rendering of versionless roots. --- README.md | 2 +- docs/cli.rst | 2 +- docs/conf.py | 1 + docs/examples/__init__.py | 1 + docs/examples/zipapp.py | 38 ++ docs/explanation.rst | 39 ++ docs/extend.rst | 66 +-- docs/index.rst | 2 +- docs/py_api.rst | 59 ++- docs/usage.rst | 3 +- pyproject.toml | 12 +- src/python_sbom/__init__.py | 48 +- src/python_sbom/__main__.py | 2 +- src/python_sbom/_backend.py | 98 ++++ src/python_sbom/_cli/__init__.py | 1 + src/python_sbom/{_cli.py => _cli/_parser.py} | 13 +- src/python_sbom/{ => _cli}/_run.py | 20 +- src/python_sbom/_collect/__init__.py | 1 + src/python_sbom/_collect/_facts.py | 156 ++++++ src/python_sbom/_collect/_settings.py | 80 +++ src/python_sbom/_collect/_source.py | 139 +++++ src/python_sbom/_cyclonedx.py | 450 ---------------- src/python_sbom/_embed.py | 74 --- src/python_sbom/_errors.py | 25 + src/python_sbom/_extension.py | 29 ++ src/python_sbom/_formats/__init__.py | 1 + src/python_sbom/_formats/_json.py | 25 + src/python_sbom/{ => _formats}/_spdx.py | 14 +- src/python_sbom/_model/__init__.py | 1 + src/python_sbom/_model/_components.py | 191 +++++++ src/python_sbom/_model/_types.py | 192 +++++++ src/python_sbom/_model/_wheel.py | 329 ++++++++++++ src/python_sbom/_project.py | 487 ------------------ src/python_sbom/_validate/__init__.py | 1 + .../{_validate.py => _validate/_checks.py} | 34 +- .../{_extra.py => _validate/_schemas.py} | 2 +- src/python_sbom/_wheel/__init__.py | 1 + src/python_sbom/_wheel/_embed.py | 128 +++++ src/python_sbom/hatch/__init__.py | 9 + src/python_sbom/{ => hatch}/_hook.py | 67 ++- src/python_sbom/{ => hatch}/hooks.py | 0 tests/conftest.py | 16 +- tests/seeder/hatch_build.py | 2 +- tests/test_cli.py | 12 + tests/test_embed.py | 72 ++- tests/test_hook.py | 2 +- tests/test_project.py | 61 ++- tests/test_spdx.py | 42 +- tests/test_validate.py | 26 +- 49 files changed, 1869 insertions(+), 1207 deletions(-) create mode 100644 docs/examples/__init__.py create mode 100644 docs/examples/zipapp.py create mode 100644 src/python_sbom/_backend.py create mode 100644 src/python_sbom/_cli/__init__.py rename src/python_sbom/{_cli.py => _cli/_parser.py} (89%) rename src/python_sbom/{ => _cli}/_run.py (61%) create mode 100644 src/python_sbom/_collect/__init__.py create mode 100644 src/python_sbom/_collect/_facts.py create mode 100644 src/python_sbom/_collect/_settings.py create mode 100644 src/python_sbom/_collect/_source.py delete mode 100644 src/python_sbom/_cyclonedx.py delete mode 100644 src/python_sbom/_embed.py create mode 100644 src/python_sbom/_errors.py create mode 100644 src/python_sbom/_extension.py create mode 100644 src/python_sbom/_formats/__init__.py create mode 100644 src/python_sbom/_formats/_json.py rename src/python_sbom/{ => _formats}/_spdx.py (92%) create mode 100644 src/python_sbom/_model/__init__.py create mode 100644 src/python_sbom/_model/_components.py create mode 100644 src/python_sbom/_model/_types.py create mode 100644 src/python_sbom/_model/_wheel.py delete mode 100644 src/python_sbom/_project.py create mode 100644 src/python_sbom/_validate/__init__.py rename src/python_sbom/{_validate.py => _validate/_checks.py} (84%) rename src/python_sbom/{_extra.py => _validate/_schemas.py} (96%) create mode 100644 src/python_sbom/_wheel/__init__.py create mode 100644 src/python_sbom/_wheel/_embed.py create mode 100644 src/python_sbom/hatch/__init__.py rename src/python_sbom/{ => hatch}/_hook.py (61%) rename src/python_sbom/{ => hatch}/hooks.py (100%) diff --git a/README.md b/README.md index 191a4f4..5592f8c 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ With [hatchling](https://hatch.pypa.io/latest/), add the hook to the build requi ```toml [build-system] build-backend = "hatchling.build" -requires = ["python-sbom>=0.1", "hatchling>=1.28"] +requires = ["hatchling>=1.28", "python-sbom[hatch]>=0.1"] [tool.hatch.build.hooks.python-sbom] ``` diff --git a/docs/cli.rst b/docs/cli.rst index bf07088..dc29159 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1,5 +1,5 @@ .. sphinx_argparse_cli:: - :module: python_sbom._cli + :module: python_sbom._cli._parser :func: _build_parser :title: Command line interface :group_title_prefix: diff --git a/docs/conf.py b/docs/conf.py index ef378d3..f776780 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,7 @@ def _inheritance_aliases() -> dict[str, str]: "sphinx.ext.inheritance_diagram", "sphinx_argparse_cli", "sphinx_autodoc_typehints", + "sphinxcontrib.mermaid", ] master_doc: Final[str] = "index" diff --git a/docs/examples/__init__.py b/docs/examples/__init__.py new file mode 100644 index 0000000..5b1d075 --- /dev/null +++ b/docs/examples/__init__.py @@ -0,0 +1 @@ +"""Examples the documentation includes and the tests run.""" diff --git a/docs/examples/zipapp.py b/docs/examples/zipapp.py new file mode 100644 index 0000000..f1ed45f --- /dev/null +++ b/docs/examples/zipapp.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING + +from python_sbom import BuildFacts, document, git_revision + +if TYPE_CHECKING: + from pathlib import Path + + from python_sbom import Body, Component, Document + + +def describe_zipapp(root: Path, version: str) -> Document: # simplint: ignore[SMP408] # the docs include it + # the build toolchain and time, recorded the same way the wheel SBOM records them + facts = BuildFacts.collect(root, git_revision) + zipapp: Component = { + "type": "application", + "bom-ref": f"pkg:generic/virtualenv.pyz@{version}", + "name": "virtualenv.pyz", + "version": version, + } + body: Body = { + "metadata": { + "timestamp": facts.timestamp, + "lifecycles": [{"phase": "build"}], + "tools": {"components": facts.tools}, + "component": zipapp, + }, + "components": [], + "dependencies": [{"ref": zipapp["bom-ref"], "dependsOn": []}, *facts.tool_dependencies], + } + return document(body, uuid.uuid5(uuid.NAMESPACE_URL, "https://github.com/pypa/virtualenv/sboms")) + + +__all__ = [ + "describe_zipapp", +] diff --git a/docs/explanation.rst b/docs/explanation.rst index f0c8bc5..b3cd785 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -1,6 +1,45 @@ How it works ============ +Architecture +------------ + +The package keeps each concern in its own place, and only the backends know a build system. A backend gathers what +the edge knows: the wheel's metadata and file name, the project's settings, and, through +:class:`~python_sbom.BuildFacts`, the build time, the source revision and the build environment. The core turns those +explicit inputs into the document with pure functions, then serializes it. The hatchling hook hands the result to +hatchling; :class:`~python_sbom.BuiltWheel` writes it into a finished wheel. SPDX rendering and validation read a +document and nothing else. + +.. mermaid:: + + flowchart LR + subgraph backends[Backends] + hatch[python_sbom.hatch
hatchling build hook] + built[BuiltWheel
wheel from any backend] + end + subgraph collect[Collect, at the edge] + settings[Settings
hook table or tool.python-sbom table] + facts[BuildFacts
time, revision, build tools] + source[Source tree
bundled wheels, license files] + end + model[Model
pure document building] + formats[Formats
CycloneDX JSON, SPDX 2.3] + validate[Validate
schema, references, licenses] + hatch & built --> settings & facts & source + settings & facts & source --> model + formats --> hatch & built & validate + model --> formats + classDef edge fill:#cfe4ff,stroke:#1f6feb,color:#0b1f3a; + classDef core fill:#d8f3dc,stroke:#2d6a4f,color:#081c15; + class hatch,built,settings,facts,source edge; + class model,formats,validate core; + +Extension points are explicit: a new build system subclasses :class:`~python_sbom.Backend`, a project adds content +through the :class:`~python_sbom.ContentProvider` method its hook or backend implements, and another version control +system plugs in as a :class:`~python_sbom.RevisionReader`. Errors are subclasses of :class:`~python_sbom.SbomError`, so +the command line reports them without a traceback. + What the SBOM records --------------------- diff --git a/docs/extend.rst b/docs/extend.rst index 58aa1d6..a14576e 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -8,7 +8,7 @@ pieces the hook uses. Add properties to bundled wheels -------------------------------- -:class:`~python_sbom.SbomBuildHook` calls :meth:`~python_sbom.SbomBuildHook.bundled_wheel_properties` once per +:class:`~python_sbom.hatch.SbomBuildHook` calls :meth:`~python_sbom.hatch.SbomBuildHook.bundled_wheel_properties` once per bundled wheel. Override it in a hatchling `custom build hook `_ to add properties. They follow ``:bundled-wheel`` and come before those taken from the wheel's metadata. The options then live in the custom hook's table, and the SBOM lists the script among the build tools, hashed, because the @@ -25,7 +25,7 @@ Python versions receive each wheel. Its ``pyproject.toml`` names the supplier an [build-system] build-backend = 'hatchling.build' - requires = ['hatch-vcs>=0.5', 'hatchling>=1.28', 'python-sbom>=0.1'] + requires = ['hatch-vcs>=0.5', 'hatchling>=1.28', 'python-sbom[hatch]>=0.1'] [tool.hatch.build.hooks.custom] path = 'hatch_build.py' @@ -42,7 +42,7 @@ Python versions receive each wheel. Its ``pyproject.toml`` names the supplier an from pathlib import Path from typing import TYPE_CHECKING - from python_sbom import SbomBuildHook + from python_sbom.hatch import SbomBuildHook if TYPE_CHECKING: from python_sbom import Property @@ -83,7 +83,7 @@ imports. The pip component of the wheel's SBOM then carries these properties, in - ``>=3.10`` A project that still builds on a Python older than python-sbom supports, as virtualenv does on 3.9 and 3.10, puts an -environment marker on the requirement, ``python-sbom>=0.1; python_version >= '3.11'``. Its script then falls back to +environment marker on the requirement, ``python-sbom[hatch]>=0.1; python_version >= '3.11'``. Its script then falls back to hatchling's plain hook there, so those builds produce a wheel without an SBOM: .. code-block:: python @@ -91,7 +91,7 @@ hatchling's plain hook there, so those builds produce a wheel without an SBOM: import sys if sys.version_info >= (3, 11): - from python_sbom import SbomBuildHook as BaseHook + from python_sbom.hatch import SbomBuildHook as BaseHook else: # python-sbom needs Python 3.11; older builds get no SBOM from hatchling.builders.hooks.plugin.interface import BuildHookInterface as BaseHook @@ -99,55 +99,27 @@ Describe another artifact ------------------------- virtualenv also publishes a zipapp. Its SBOM describes what the archive holds and records the same build toolchain as -the wheel's. The functions the hook uses are public for that: +the wheel's. The pieces the hook uses are public for that: +- :class:`~python_sbom.BuildFacts` collects the build time from ``SOURCE_DATE_EPOCH``, the source revision and the + interpreter and distributions of the running environment, once, at the edge. - :func:`~python_sbom.component_from_metadata` turns a distribution's ``METADATA`` into a component, with licenses, project URLs and metadata properties. -- :func:`~python_sbom.build_tools` records the interpreter and every distribution of the running environment, with - the files of the pure-Python ones and the dependencies between them. -- :func:`~python_sbom.timestamp` gives the time from ``SOURCE_DATE_EPOCH``. - :func:`~python_sbom.document` adds the CycloneDX header, with a serial number derived from the content. -.. code-block:: python +This function, which the test suite runs and validates, describes a zipapp with no components yet: - import uuid - - from python_sbom import Body, Component, build_tools, document, timestamp - - tools, tool_dependencies = build_tools() - zipapp: Component = {'type': 'application', 'bom-ref': 'pkg:generic/virtualenv.pyz@21.0', 'name': 'virtualenv.pyz'} - body: Body = { - 'metadata': { - 'timestamp': timestamp(), - 'lifecycles': [{'phase': 'build'}], - 'tools': {'components': tools}, - 'component': zipapp, - }, - 'components': [], - 'dependencies': [{'ref': zipapp['bom-ref'], 'dependsOn': []}, *tool_dependencies], - 'formulation': [], - } - sbom = document(body, uuid.uuid5(uuid.NAMESPACE_URL, 'https://github.com/pypa/virtualenv/sboms')) +.. literalinclude:: examples/zipapp.py + :language: python + :pyobject: describe_zipapp :func:`~python_sbom.validate` checks the result the way ``python-sbom validate`` checks a wheel's SBOM. -Use it as a library -------------------- - -:class:`~python_sbom.Project` holds the options and writes the SBOM of a wheel from its ``METADATA``, and -:func:`~python_sbom.embed` rewrites a built wheel with it. Pass ``bundled_wheel_properties`` to add properties to -bundled wheels without a hatchling hook: - -.. code-block:: python - - from pathlib import Path - - from python_sbom import Project, embed +Add another build backend +------------------------- - project = Project.from_table( - {'bundled-wheels': ['src/project/wheels/*.whl']}, - root=Path.cwd(), - where='my build script', - bundled_wheel_properties=lambda wheel: [{'name': 'project:role', 'value': 'seed'}], - ) - embed(Path('dist/project-1.0-py3-none-any.whl'), project) +A build system other than hatchling plugs in by subclassing :class:`~python_sbom.Backend`. The subclass supplies the +wheel's metadata, its file name, the project directory and the settings; :meth:`~python_sbom.Backend.sbom` does the +rest. :class:`~python_sbom.BuiltWheel`, the backend behind ``python-sbom embed``, reads all of that from a finished +wheel; subclass it and override :meth:`~python_sbom.Backend.bundled_wheel_properties` to add properties to bundled wheels +without hatchling. diff --git a/docs/index.rst b/docs/index.rst index c056ab5..1041067 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ enable it in ``pyproject.toml``: [build-system] build-backend = 'hatchling.build' - requires = ['python-sbom>=0.1', 'hatchling>=1.28'] + requires = ['hatchling>=1.28', 'python-sbom[hatch]>=0.1'] [tool.hatch.build.hooks.python-sbom] diff --git a/docs/py_api.rst b/docs/py_api.rst index 9733bce..8ccbfc1 100644 --- a/docs/py_api.rst +++ b/docs/py_api.rst @@ -1,39 +1,55 @@ Python API ========== -Everything below is importable from ``python_sbom``. +The core is importable from ``python_sbom`` without hatchling; the hatchling build hook lives in ``python_sbom.hatch``. .. autodata:: python_sbom.__version__ -Build hook ----------- +Hatchling build hook +-------------------- -.. autoclass:: python_sbom.SbomBuildHook +.. autoclass:: python_sbom.hatch.SbomBuildHook :members: bundled_wheel_properties -Wheels from any build backend ------------------------------ +Backends +-------- -.. autoclass:: python_sbom.Project - :members: from_table, describe - :exclude-members: root, supplier, repository, default_branch, bundled_wheels, build_script, bundled_wheel_properties +.. autoclass:: python_sbom.Backend + :members: + +.. autoclass:: python_sbom.BuiltWheel + :members: embed + +.. autoclass:: python_sbom.ContentProvider + :members: + +Inputs +------ -.. autoclass:: python_sbom.BundledWheelProperties +.. autoclass:: python_sbom.Settings + :members: from_table + :exclude-members: supplier, repository, default_branch, bundled_wheels + +.. autodata:: python_sbom.Toml + +.. autoclass:: python_sbom.BuildFacts + :members: collect, timestamp + :exclude-members: epoch, commit, tools, tool_dependencies + +.. autoclass:: python_sbom.RevisionReader :members: __call__ -.. autofunction:: python_sbom.embed +.. autofunction:: python_sbom.git_revision Building blocks --------------- .. autofunction:: python_sbom.component_from_metadata -.. autofunction:: python_sbom.build_tools - -.. autofunction:: python_sbom.timestamp - .. autofunction:: python_sbom.document +.. autofunction:: python_sbom.dumps + SPDX and validation ------------------- @@ -43,6 +59,17 @@ SPDX and validation .. autofunction:: python_sbom.validate_wheel +Errors +------ + +.. autoexception:: python_sbom.SbomError + +.. autoexception:: python_sbom.SettingsError + +.. autoexception:: python_sbom.WheelError + +.. autoexception:: python_sbom.SourceRevisionError + Types ----- @@ -69,8 +96,6 @@ These are :class:`~typing.TypedDict` types of the JSON documents, for type check .. autoclass:: python_sbom.Spdx :no-members: -.. autodata:: python_sbom.Toml - Command line ------------ diff --git a/docs/usage.rst b/docs/usage.rst index 7eeb658..9f7cade 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -5,7 +5,8 @@ With hatchling -------------- Add the hook to ``build-system.requires`` and enable it with an empty table, as the quick start on the :doc:`index` -shows. hatchling 1.28 added SBOM support to wheels, so the hook requires it. The hook acts on wheel builds only; an +shows. The ``hatch`` extra brings hatchling 1.28, the first release that puts an SBOM into a wheel; the command line +and the library need no hatchling. The hook acts on wheel builds only; an sdist carries no SBOM. The SBOM links to the wheel's PyPI attestation, so it needs the wheel's tag. List the hook after any hook that changes diff --git a/pyproject.toml b/pyproject.toml index ca503d1..a11b283 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,9 +45,12 @@ dynamic = [ "version", ] dependencies = [ - "hatchling>=1.28", "packaging>=26.3", ] +optional-dependencies.hatch = [ + # 1.28 is the first release that puts an SBOM into a wheel + "hatchling>=1.28", +] optional-dependencies.validate = [ "cyclonedx-python-lib[json-validation]>=11.12", "spdx-tools>=0.8.5", @@ -58,7 +61,7 @@ urls.Homepage = "https://github.com/tox-dev/python-sbom" urls.Source = "https://github.com/tox-dev/python-sbom" urls.Tracker = "https://github.com/tox-dev/python-sbom/issues" scripts.python-sbom = "python_sbom.__main__:main" -entry-points.hatch.python-sbom = "python_sbom.hooks" +entry-points.hatch.python-sbom = "python_sbom.hatch.hooks" [dependency-groups] dev = [ @@ -70,6 +73,7 @@ dev = [ test = [ "covdefaults>=2.3", "cyclonedx-python-lib[json-validation]>=11.12", + "hatchling>=1.28", "installer>=0.7", "pytest>=9.1.1", "pytest-cov>=7.1", @@ -83,9 +87,11 @@ type = [ ] docs = [ "furo>=2025.12.19", + "hatchling>=1.28", "sphinx>=9.0.4", "sphinx-argparse-cli>=1.23", "sphinx-autodoc-typehints>=3.17", + "sphinxcontrib-mermaid>=2.1.1", ] build = [ "hatch-fancy-pypi-readme>=25.1", @@ -168,6 +174,8 @@ environment.python-version = "3.15" [tool.pytest] ini_options.testpaths = [ "tests" ] +# the tests run the examples under docs/examples that the documentation includes +ini_options.pythonpath = [ "." ] ini_options.filterwarnings = [ "error", ] diff --git a/src/python_sbom/__init__.py b/src/python_sbom/__init__.py index 6a40641..f3ecbf4 100644 --- a/src/python_sbom/__init__.py +++ b/src/python_sbom/__init__.py @@ -2,46 +2,46 @@ from __future__ import annotations -from ._cli import Options, main -from ._cyclonedx import ( - Body, - Component, - Dependency, - Document, - OrganizationalEntity, - Property, - build_tools, - component_from_metadata, - document, - timestamp, -) -from ._embed import embed -from ._hook import SbomBuildHook -from ._project import BundledWheelProperties, Project, Toml -from ._spdx import Spdx, to_spdx -from ._validate import validate, validate_wheel +from ._backend import Backend +from ._cli._parser import Options, main +from ._collect._facts import BuildFacts, RevisionReader, git_revision +from ._collect._settings import Settings, Toml +from ._errors import SbomError, SettingsError, SourceRevisionError, WheelError +from ._extension import ContentProvider +from ._formats._json import dumps +from ._formats._spdx import Spdx, to_spdx +from ._model._components import component_from_metadata, document +from ._model._types import Body, Component, Dependency, Document, OrganizationalEntity, Property +from ._validate._checks import validate, validate_wheel +from ._wheel._embed import BuiltWheel from .version import version as __version__ __all__ = [ + "Backend", "Body", - "BundledWheelProperties", + "BuildFacts", + "BuiltWheel", "Component", + "ContentProvider", "Dependency", "Document", "Options", "OrganizationalEntity", - "Project", "Property", - "SbomBuildHook", + "RevisionReader", + "SbomError", + "Settings", + "SettingsError", + "SourceRevisionError", "Spdx", "Toml", + "WheelError", "__version__", - "build_tools", "component_from_metadata", "document", - "embed", + "dumps", + "git_revision", "main", - "timestamp", "to_spdx", "validate", "validate_wheel", diff --git a/src/python_sbom/__main__.py b/src/python_sbom/__main__.py index fd5e81a..dc323c4 100644 --- a/src/python_sbom/__main__.py +++ b/src/python_sbom/__main__.py @@ -4,7 +4,7 @@ import sys -from ._cli import main +from ._cli._parser import main if __name__ == "__main__": main(sys.argv[1:]) diff --git a/src/python_sbom/_backend.py b/src/python_sbom/_backend.py new file mode 100644 index 0000000..17e69fc --- /dev/null +++ b/src/python_sbom/_backend.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +from packaging.utils import canonicalize_name + +from ._collect._source import read_source_tree +from ._formats._json import dumps +from ._model._wheel import WheelInputs, describe + +if TYPE_CHECKING: + from email.message import Message + from pathlib import Path, PurePosixPath + + from ._collect._facts import BuildFacts + from ._collect._settings import Settings + from ._model._types import Property + + +class Backend(ABC): + """ + Adapts a build system to the core: it supplies the wheel's metadata and the project's settings. + + The hatchling hook and :class:`BuiltWheel` are the two backends shipped; another build system adds a subclass. + :meth:`sbom` reads the project tree, builds the document and serializes it. + + """ + + def bundled_wheel_properties(self, wheel: Path) -> list[Property]: # ruff: ignore[no-self-use] # extension seam + """ + Return extra properties for a wheel the project bundles as a data file; none by default. + + :param wheel: the bundled wheel, an absolute path under :attr:`root` + :returns: the properties to add after ``:bundled-wheel`` + + """ + del wheel + return [] + + def sbom(self, facts: BuildFacts) -> tuple[str, bytes]: + """ + Build the SBOM of the wheel. + + :param facts: what the environment says about the build, collected once per build + :returns: the SBOM's file name under ``.dist-info/sboms/`` and its JSON + + """ + metadata = self.metadata() + source = read_source_tree( + self.root, self.settings, metadata.get_all("License-File", []), self.build_script(), self + ) + inputs = WheelInputs( + metadata=metadata, wheel_name=self.wheel_name(), settings=self.settings, facts=facts, source=source + ) + return f"{canonicalize_name(metadata['Name'])}.cdx.json", dumps(describe(inputs)) + + @abstractmethod + def metadata(self) -> Message: + """ + Return the core metadata of the wheel. + + :returns: the parsed ``METADATA`` + + """ + + @property + @abstractmethod + def root(self) -> Path: + """The project directory.""" + + @property + @abstractmethod + def settings(self) -> Settings: + """The project's options.""" + + def build_script(self) -> PurePosixPath | None: # ruff: ignore[no-self-use] # backends override it + """ + Return a build script to list among the build tools, as hatchling's custom hook script. + + :returns: the script, relative to :attr:`root`, or ``None``, the default, for none + + """ + return None + + @abstractmethod + def wheel_name(self) -> str: + """ + Return the file name of the wheel, for its attestation link. + + :returns: the file name + + """ + + +__all__ = [ + "Backend", +] diff --git a/src/python_sbom/_cli/__init__.py b/src/python_sbom/_cli/__init__.py new file mode 100644 index 0000000..376611a --- /dev/null +++ b/src/python_sbom/_cli/__init__.py @@ -0,0 +1 @@ +"""The command line: parsing options and running one command.""" diff --git a/src/python_sbom/_cli.py b/src/python_sbom/_cli/_parser.py similarity index 89% rename from src/python_sbom/_cli.py rename to src/python_sbom/_cli/_parser.py index d4460f5..da657f5 100644 --- a/src/python_sbom/_cli.py +++ b/src/python_sbom/_cli/_parser.py @@ -1,11 +1,14 @@ from __future__ import annotations +import sys from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter from pathlib import Path from typing import TYPE_CHECKING +from python_sbom._errors import SbomError +from python_sbom.version import version + from ._run import run -from .version import version if TYPE_CHECKING: from collections.abc import Sequence @@ -19,7 +22,13 @@ def main(args: Sequence[str] | None = None) -> None: :raises SystemExit: with status 1 when a command fails """ - if not run(_build_parser().parse_args(args, namespace=Options())): + options = _build_parser().parse_args(args, namespace=Options()) + try: + succeeded = run(options) + except SbomError as error: + sys.stderr.write(f"python-sbom {options.command}: {error}\n") + succeeded = False + if not succeeded: raise SystemExit(1) diff --git a/src/python_sbom/_run.py b/src/python_sbom/_cli/_run.py similarity index 61% rename from src/python_sbom/_run.py rename to src/python_sbom/_cli/_run.py index b494965..f9f92c6 100644 --- a/src/python_sbom/_run.py +++ b/src/python_sbom/_cli/_run.py @@ -5,26 +5,28 @@ import tomllib from typing import TYPE_CHECKING -from ._embed import embed -from ._project import Project -from ._spdx import to_spdx -from ._validate import validate_wheel +from python_sbom._collect._facts import BuildFacts, git_revision +from python_sbom._collect._settings import Settings +from python_sbom._formats._json import dumps +from python_sbom._formats._spdx import to_spdx +from python_sbom._validate._checks import validate_wheel +from python_sbom._wheel._embed import BuiltWheel if TYPE_CHECKING: - from ._cli import Options + from ._parser import Options def run(options: Options) -> bool: if options.command == "embed": pyproject = options.root / "pyproject.toml" table = tomllib.loads(pyproject.read_text(encoding="utf-8")).get("tool", {}).get("python-sbom", {}) - project = Project.from_table(table, root=options.root, where=f"[tool.python-sbom] of {pyproject}") + settings = Settings.from_table(table, f"[tool.python-sbom] of {pyproject}") + facts = BuildFacts.collect(options.root, git_revision) for wheel in options.wheels: - sys.stdout.write(f"wrote {embed(wheel, project)} into {wheel.name}\n") + sys.stdout.write(f"wrote {BuiltWheel(wheel, options.root, settings).embed(facts)} into {wheel.name}\n") return True if options.command == "spdx": - bom = json.loads(options.source.read_text(encoding="utf-8")) - options.target.write_bytes(f"{json.dumps(to_spdx(bom), indent=2)}\n".encode()) + options.target.write_bytes(dumps(to_spdx(json.loads(options.source.read_text(encoding="utf-8"))))) return True allowed = None if options.allowed_licenses is None else frozenset(options.allowed_licenses) wheels = sorted(wheel for path in options.paths for wheel in (path.glob("*.whl") if path.is_dir() else [path])) diff --git a/src/python_sbom/_collect/__init__.py b/src/python_sbom/_collect/__init__.py new file mode 100644 index 0000000..c468933 --- /dev/null +++ b/src/python_sbom/_collect/__init__.py @@ -0,0 +1 @@ +"""Facts gathered at the edge: settings, the build environment, the source revision, bundled wheels.""" diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py new file mode 100644 index 0000000..4a7897c --- /dev/null +++ b/src/python_sbom/_collect/_facts.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import os +import platform +import shutil +import subprocess # ruff: ignore[suspicious-subprocess-import] # runs git to read the source revision +import sys +from dataclasses import dataclass +from datetime import UTC, datetime +from email.parser import Parser +from importlib.metadata import distributions +from typing import TYPE_CHECKING, Final, Protocol + +from packaging.requirements import Requirement + +from python_sbom._errors import SourceRevisionError +from python_sbom._model._components import component_from_metadata, file_component, purl + +if TYPE_CHECKING: + from importlib.metadata import Distribution + from pathlib import Path + + from python_sbom._model._types import Component, Dependency + + +class RevisionReader(Protocol): + """Reads the source revision a build comes from.""" + + def __call__(self, root: Path, /) -> str | None: + """ + Read the revision of a source tree. + + :param root: the project directory + :returns: the revision, or ``None`` when the directory is not under version control + :raises SourceRevisionError: when it is, but the revision cannot be read + + """ + + +@dataclass(frozen=True, slots=True, kw_only=True) +class BuildFacts: + """ + What the SBOM records about the build itself, gathered once from the environment that runs it. + + :param epoch: the build time as a Unix timestamp, from ``SOURCE_DATE_EPOCH`` + :param commit: the source revision of a git checkout, or ``None`` outside one + :param tools: the interpreter and every distribution of the build environment, interpreter first + :param tool_dependencies: the dependency graph between those distributions + + """ + + epoch: int + commit: str | None + tools: list[Component] + tool_dependencies: list[Dependency] + + @classmethod + def collect(cls, root: Path, revision: RevisionReader) -> BuildFacts: + """ + Read the build time, the source revision and the build environment. + + Nothing about the machine goes in: the same source, toolchain and ``SOURCE_DATE_EPOCH`` give the same facts + on any operating system. A pure-Python distribution lists its files with their hashes; a platform one does + not, because its files differ per machine. + + :param root: the project directory + :param revision: reads the source revision of ``root``, such as :func:`git_revision` + :returns: the facts + + """ + tools, tool_dependencies = _build_tools() + # hatchling's default when the variable is unset, so a hook build and an embed run agree + epoch = int(os.environ.get("SOURCE_DATE_EPOCH", "1580601600")) + return cls(epoch=epoch, commit=revision(root), tools=tools, tool_dependencies=tool_dependencies) + + @property + def timestamp(self) -> str: + """The build time in ISO 8601.""" + return datetime.fromtimestamp(self.epoch, tz=UTC).isoformat() + + +def _build_tools() -> tuple[list[Component], list[Dependency]]: + # nothing about the machine goes in, so the same toolchain yields the same record on any operating system + interpreter = f"pkg:generic/{sys.implementation.name}@{platform.python_version()}" + tools: Final[list[Component]] = [ + { + "type": "platform", + "bom-ref": f"tool:{interpreter}", + "name": sys.implementation.name, + "version": platform.python_version(), + "purl": interpreter, + "properties": [{"name": "python:implementation", "value": platform.python_implementation()}], + }, + ] + # bom-refs are prefixed because the same distribution can be both a build tool and a component of the artifact + installed = {purl(distribution.metadata["Name"]): distribution for distribution in distributions()} + tool_dependencies: Final[list[Dependency]] = [] + for distribution in (installed[key] for key in sorted(installed)): + component = component_from_metadata(distribution.metadata, "library") + component["bom-ref"] = f"tool:{component['bom-ref']}" + if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] == "true": + component["components"] = [ + file_component( + component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) + ) + for file in distribution.files or [] + # console-script launchers live outside site-packages and embed the environment's interpreter path in + # their shebang, so their hash differs on every build; the installer metadata names the frontend that + # set up the environment rather than the distribution + if file.hash is not None + and not file.as_posix().startswith("../") + and not ( + file.parent.suffix == ".dist-info" and file.name in {"INSTALLER", "REQUESTED", "direct_url.json"} + ) + ] + tools.append(component) + tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) + return tools, tool_dependencies + + +def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) -> list[str]: + refs = set() + for requirement in map(Requirement, distribution.requires or []): + if (requirement.marker is None or requirement.marker.evaluate()) and ( + dependency := installed.get(purl(requirement.name)) + ): + refs.add(f"tool:{purl(dependency.metadata['Name'], dependency.version)}") + return sorted(refs) + + +def git_revision(root: Path, /) -> str | None: + """ + Read the commit of a git checkout. + + :param root: the project directory + :returns: the commit, or ``None`` when ``root`` holds no ``.git``, as in a build from an sdist, or git is missing + :raises SourceRevisionError: when git cannot read the commit, such as in a checkout it considers unsafe + + """ + if not (root / ".git").exists() or (git := shutil.which("git")) is None: + return None + try: + result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] # fixed arguments + [git, "rev-parse", "HEAD"], check=True, capture_output=True, text=True, cwd=root + ) + except subprocess.CalledProcessError as error: + msg = f"git rev-parse HEAD failed in {root}: {error.stderr.strip()}" + raise SourceRevisionError(msg) from error + return result.stdout.strip() + + +__all__ = [ + "BuildFacts", + "RevisionReader", + "git_revision", +] diff --git a/src/python_sbom/_collect/_settings.py b/src/python_sbom/_collect/_settings.py new file mode 100644 index 0000000..9fefe96 --- /dev/null +++ b/src/python_sbom/_collect/_settings.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypeAlias + +from python_sbom._errors import SettingsError + +if TYPE_CHECKING: + from collections.abc import Mapping + from datetime import date, datetime, time + + from python_sbom._model._types import OrganizationalEntity + +#: a value of a TOML table, as :mod:`tomllib` or hatchling hand it over +Toml: TypeAlias = "str | int | float | bool | datetime | date | time | list[Toml] | dict[str, Toml]" + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Settings: + """ + The options a project sets, in a hatchling hook's table or in ``[tool.python-sbom]``. + + :param supplier: the organization that supplies the project, or ``None`` to name none + :param repository: the repository URL, or ``None`` to take the first ``Source``, ``Repository`` or ``GitHub`` + project URL of the metadata + :param default_branch: branch used in links to files in the repository + :param bundled_wheels: glob patterns, relative to the project root, of wheels the project ships as data files + + """ + + supplier: OrganizationalEntity | None + repository: str | None + default_branch: str + bundled_wheels: tuple[str, ...] + + @classmethod + def from_table(cls, table: Mapping[str, Toml], where: str) -> Settings: + """ + Read the options from a configuration table, ignoring keys it does not know. + + :param table: the table, with any of ``supplier``, ``repository``, ``default-branch`` and ``bundled-wheels`` + :param where: where the table lives, named in error messages, such as ``[tool.python-sbom]`` + :returns: the settings + :raises SettingsError: when an option has the wrong type + + """ + supplier = table.get("supplier") + if supplier is not None and not ( + isinstance(supplier, dict) + and supplier.keys() <= {"name", "url"} + and isinstance(name := supplier.get("name"), str) + and isinstance(urls := supplier.get("url", []), list) + and all(isinstance(url, str) for url in urls) + ): + msg = f"`supplier` in {where} must be a table with a string `name` and an optional `url` string array" + raise SettingsError(msg) + if not isinstance(repository := table.get("repository"), str | None): + msg = f"`repository` in {where} must be a string" + raise SettingsError(msg) + if not isinstance(default_branch := table.get("default-branch", "main"), str): + msg = f"`default-branch` in {where} must be a string" + raise SettingsError(msg) + if not ( + isinstance(patterns := table.get("bundled-wheels", []), list) + and all(isinstance(pattern, str) for pattern in patterns) + ): + msg = f"`bundled-wheels` in {where} must be an array of strings" + raise SettingsError(msg) + return cls( + supplier=None if supplier is None else {"name": name, "url": [str(url) for url in urls]}, + repository=None if repository is None else repository.rstrip("/"), + default_branch=default_branch, + bundled_wheels=tuple(str(pattern) for pattern in patterns), + ) + + +__all__ = [ + "Settings", + "Toml", +] diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py new file mode 100644 index 0000000..58c4e38 --- /dev/null +++ b/src/python_sbom/_collect/_source.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import hashlib +import zipfile +from dataclasses import dataclass +from email.parser import Parser +from typing import TYPE_CHECKING, Final + +from packaging.requirements import InvalidRequirement, Requirement + +from python_sbom._errors import WheelError + +if TYPE_CHECKING: + from email.message import Message + from pathlib import Path, PurePosixPath + + from python_sbom._extension import ContentProvider + from python_sbom._model._types import Property + + from ._settings import Settings + + +@dataclass(frozen=True, slots=True, kw_only=True) +class BundledWheel: + # A wheel the project ships as a data file, read from the project tree. + path: str + sha256: str + metadata_name: str + metadata: Message + record: str + vendored: list[tuple[str, Message]] + extra_properties: list[Property] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class BuildScript: + # A build script that shapes the document, so a rebuild has to match it. + path: PurePosixPath + sha256: str + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SourceTree: + # What the document takes from the project tree besides the wheel's metadata. + copyright: str | None + bundled: list[BundledWheel] + build_script: BuildScript | None + + +def read_source_tree( + root: Path, + settings: Settings, + license_files: list[str], + build_script: PurePosixPath | None, + content: ContentProvider, +) -> SourceTree: + return SourceTree( + copyright=next( + ( + line + for license_file in license_files + if (path := root / license_file).is_file() + for line in path.read_text(encoding="utf-8").splitlines() + if line.startswith("Copyright") + ), + None, + ), + bundled=[ + _read_bundled_wheel(root, wheel, content) + for wheel in sorted( + {path for pattern in settings.bundled_wheels for path in root.glob(pattern)}, + key=lambda path: path.relative_to(root).as_posix(), + ) + ], + # str, so a Windows drive in an absolute path survives the join with the native root + build_script=None + if build_script is None + else BuildScript(path=build_script, sha256=hashlib.sha256((root / str(build_script)).read_bytes()).hexdigest()), + ) + + +def _read_bundled_wheel(root: Path, wheel: Path, content: ContentProvider) -> BundledWheel: + with zipfile.ZipFile(wheel) as archive: + members: Final[list[str]] = sorted(archive.namelist()) + if ( + metadata_name := next( + (name for name in members if name.count("/") == 1 and name.endswith(".dist-info/METADATA")), None + ) + ) is None: + msg = f"bundled wheel {wheel} has no .dist-info/METADATA" + raise WheelError(msg) + metadata = Parser().parsestr(archive.read(metadata_name).decode("utf-8")) + record = archive.read(metadata_name.replace("METADATA", "RECORD")).decode("utf-8") + vendored: Final[list[tuple[str, Message]]] = [ + *_vendor_manifest(wheel, archive, members), + *( + (name, Parser().parsestr(archive.read(name).decode("utf-8"))) + for name in members + if "/_vendor/" in name and name.endswith(".dist-info/METADATA") + ), + ] + return BundledWheel( + path=wheel.relative_to(root).as_posix(), + sha256=hashlib.sha256(wheel.read_bytes()).hexdigest(), + metadata_name=metadata_name, + metadata=metadata, + record=record, + vendored=vendored, + extra_properties=content.bundled_wheel_properties(wheel), + ) + + +def _vendor_manifest(wheel: Path, archive: zipfile.ZipFile, members: list[str]) -> list[tuple[str, Message]]: + # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info + found: Final[list[tuple[str, Message]]] = [] + for name in members: + if not name.endswith("/_vendor/vendor.txt"): + continue + for line in archive.read(name).decode("utf-8").splitlines(): + if not (requirement := line.partition("#")[0].strip()): + continue + try: + pins = list((parsed := Requirement(requirement)).specifier) + except InvalidRequirement as error: + msg = f"vendored requirement in {wheel.name}:{name} does not parse: {requirement}" + raise WheelError(msg) from error + if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: + msg = f"Unpinned vendored dependency in {wheel.name}:{name}: {requirement}; pin one exact version" + raise WheelError(msg) + found.append((name, Parser().parsestr(f"Name: {parsed.name}\nVersion: {pins[0].version}\n"))) + return found + + +__all__ = [ + "BuildScript", + "BundledWheel", + "SourceTree", + "read_source_tree", +] diff --git a/src/python_sbom/_cyclonedx.py b/src/python_sbom/_cyclonedx.py deleted file mode 100644 index 1888094..0000000 --- a/src/python_sbom/_cyclonedx.py +++ /dev/null @@ -1,450 +0,0 @@ -from __future__ import annotations - -import base64 -import hashlib -import json -import platform -import re -import sys -import uuid -from datetime import UTC, datetime -from email.parser import Parser -from email.utils import getaddresses -from importlib.metadata import distributions -from typing import TYPE_CHECKING, Final, NotRequired, TypedDict - -from hatchling.builders.utils import get_reproducible_timestamp -from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression -from packaging.requirements import Requirement -from packaging.utils import canonicalize_name - -if TYPE_CHECKING: - from email.message import Message - from importlib.metadata import Distribution, PackageMetadata - - -class OrganizationalEntity(TypedDict): - """An organization that supplies or makes a component.""" - - name: str - url: list[str] - - -class Contact(TypedDict): - name: NotRequired[str] - email: NotRequired[str] - - -class _Hash(TypedDict): - alg: str - content: str - - -class Property(TypedDict): - """A name and value pair; names this package writes start with the project name or ``python:``.""" - - name: str - value: str - - -class ExternalReference(TypedDict): - type: str - url: str - comment: NotRequired[str] - - -class LicenseInfo(TypedDict): - id: NotRequired[str] - name: NotRequired[str] - acknowledgement: str - - -class License(TypedDict): - license: NotRequired[LicenseInfo] - expression: NotRequired[str] - acknowledgement: NotRequired[str] - - -class _Method(TypedDict): - technique: str - confidence: int - value: str - - -class _Identity(TypedDict): - field: str - confidence: int - methods: list[_Method] - - -class _Evidence(TypedDict): - identity: list[_Identity] - - -#: A CycloneDX component: a distribution, an application, the interpreter or a file. -Component = TypedDict( - "Component", - { - "type": str, - "bom-ref": str, - "name": str, - "version": NotRequired[str], - "purl": NotRequired[str], - "supplier": NotRequired[OrganizationalEntity], - "authors": NotRequired[list[Contact]], - "description": NotRequired[str], - "licenses": NotRequired[list[License]], - "copyright": NotRequired[str], - "hashes": NotRequired[list[_Hash]], - "externalReferences": NotRequired[list[ExternalReference]], - "properties": NotRequired[list[Property]], - "evidence": NotRequired[_Evidence], - "components": NotRequired[list["Component"]], - }, -) - - -class Dependency(TypedDict): - """The components one component depends on, by ``bom-ref``.""" - - ref: str - dependsOn: list[str] - - -class _Reference(TypedDict): - ref: str - - -class _Output(TypedDict): - type: str - resource: _Reference - - -class _Input(TypedDict): - environmentVars: list[Property] - - -_Workflow = TypedDict( - "_Workflow", - { - "bom-ref": str, - "uid": str, - "name": str, - "taskTypes": list[str], - "resourceReferences": list[_Reference], - "outputs": list[_Output], - "inputs": list[_Input], - }, -) - - -_Formula = TypedDict("_Formula", {"bom-ref": str, "workflows": list[_Workflow]}) - - -class Lifecycle(TypedDict): - phase: str - - -class Tools(TypedDict): - components: list[Component] - - -class Metadata(TypedDict): - timestamp: str - lifecycles: list[Lifecycle] - tools: Tools - manufacturer: NotRequired[OrganizationalEntity] - authors: NotRequired[list[Contact]] - supplier: NotRequired[OrganizationalEntity] - component: Component - licenses: NotRequired[list[License]] - properties: NotRequired[list[Property]] - - -class Composition(TypedDict): - aggregate: str - dependencies: list[str] - - -class Body(TypedDict): - """The content of a document, everything but the header :func:`document` adds.""" - - metadata: Metadata - components: list[Component] - dependencies: list[Dependency] - compositions: NotRequired[list[Composition]] - formulation: list[_Formula] - - -#: A complete CycloneDX 1.6 JSON document. -Document = TypedDict( - "Document", - { - "$schema": str, - "bomFormat": str, - "specVersion": str, - "serialNumber": str, - "version": int, - "metadata": Metadata, - "components": list[Component], - "dependencies": list[Dependency], - "compositions": NotRequired[list[Composition]], - "formulation": list[_Formula], - }, -) - -# the label normalization and mapping cyclonedx-py applies to Project-URL entries, plus "source", so the same metadata -# yields the same reference types whichever generator produced the document -_URL_LABEL_TO_REFERENCE_TYPE: Final[dict[str, str]] = { - "bugtracker": "issue-tracker", - "issuetracker": "issue-tracker", - "issues": "issue-tracker", - "bugreports": "issue-tracker", - "tracker": "issue-tracker", - "home": "website", - "homepage": "website", - "download": "distribution", - "documentation": "documentation", - "docs": "documentation", - "changelog": "release-notes", - "changes": "release-notes", - "source": "vcs", - "repository": "vcs", - "github": "vcs", - "chat": "chat", -} - - -def document(body: Body, namespace: uuid.UUID) -> Document: - """ - Add the CycloneDX 1.6 header to a document body. - - :param body: the metadata, components, dependencies and formulation - :param namespace: UUID namespace of the serial number, one per project - :returns: the document; its serial number derives from the body, so identical bodies share one - - """ - return { - "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", - "bomFormat": "CycloneDX", - "specVersion": "1.6", - "serialNumber": f"urn:uuid:{uuid.uuid5(namespace, json.dumps(body, sort_keys=True))}", - "version": 1, - **body, - } - - -def timestamp() -> str: - """ - Format the build time. - - :returns: ISO 8601 time from ``SOURCE_DATE_EPOCH``, or hatchling's fixed default when it is unset - - """ - return datetime.fromtimestamp(get_reproducible_timestamp(), tz=UTC).isoformat() - - -def build_tools() -> tuple[list[Component], list[Dependency]]: - """ - Describe the running environment: the interpreter and every installed distribution. - - Nothing about the machine goes in, so the same toolchain yields the same record on any operating system. A - pure-Python distribution lists its files with their hashes; a platform one does not, because its files differ per - machine. - - :returns: the tool components, interpreter first, and the dependency graph between the distributions - - """ - interpreter = f"pkg:generic/{sys.implementation.name}@{platform.python_version()}" - tools: Final[list[Component]] = [ - { - "type": "platform", - "bom-ref": f"tool:{interpreter}", - "name": sys.implementation.name, - "version": platform.python_version(), - "purl": interpreter, - "properties": [{"name": "python:implementation", "value": platform.python_implementation()}], - }, - ] - # bom-refs are prefixed because the same distribution can be both a build tool and a component of the artifact - installed = {purl(distribution.metadata["Name"]): distribution for distribution in distributions()} - tool_dependencies: Final[list[Dependency]] = [] - for distribution in (installed[key] for key in sorted(installed)): - component = component_from_metadata(distribution.metadata, "library") - component["bom-ref"] = f"tool:{component['bom-ref']}" - if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] == "true": - component["components"] = [ - file_component( - component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) - ) - for file in distribution.files or [] - # console-script launchers live outside site-packages and embed the environment's interpreter path in - # their shebang, so their hash differs on every build; the installer metadata names the frontend that - # set up the environment rather than the distribution - if file.hash is not None - and not file.as_posix().startswith("../") - and not ( - file.parent.suffix == ".dist-info" and file.name in {"INSTALLER", "REQUESTED", "direct_url.json"} - ) - ] - tools.append(component) - tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) - return tools, tool_dependencies - - -def purl(name: str, version: str | None = None) -> str: - return f"pkg:pypi/{canonicalize_name(name)}@{version}" if version else f"pkg:pypi/{canonicalize_name(name)}" - - -def component_from_metadata(metadata: PackageMetadata | Message, component_type: str) -> Component: - """ - Describe a distribution from its core metadata. - - :param metadata: the distribution's ``METADATA`` - :param component_type: the CycloneDX component type, such as ``library`` - :returns: the component, with its purl as ``bom-ref``, licenses, Project-URL references and metadata properties - - """ - name, version = metadata["Name"], metadata["Version"] - package = purl(name, version) - component: Final[Component] = {"type": component_type, "bom-ref": package, "name": name, "version": version} - component["purl"] = package - if summary := metadata.get("Summary"): - component["description"] = summary - if authors := contacts( - metadata.get_all("Author", []) + metadata.get_all("Maintainer", []), - metadata.get_all("Author-email", []) + metadata.get_all("Maintainer-email", []), - ): - component["authors"] = authors - if licenses := _licenses(metadata): - component["licenses"] = licenses - component["externalReferences"] = [ - external_reference(label.strip(), url.strip()) - for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) - ] - if home_page := metadata.get("Home-page"): - component["externalReferences"].insert(0, {"type": "website", "url": home_page}) - # core metadata headers copied verbatim onto the component - component["properties"] = [ - {"name": f"python:{header.lower()}", "value": value} - for header in ("Requires-Python", "Requires-Dist", "Provides-Extra", "Classifier", "Keywords") - for value in metadata.get_all(header, []) - ] - return component - - -def contacts(names: list[str], addresses: list[str]) -> list[Contact]: - found: Final[list[Contact]] = [{"name": name} for name in names] - for name, email in getaddresses(addresses): - contact: Contact = {} - if name: - contact["name"] = name - if email: - contact["email"] = email - found.append(contact) - return found - - -def _licenses(metadata: PackageMetadata | Message) -> list[License]: - if expression := metadata.get("License-Expression"): - return [{"expression": expression, "acknowledgement": "declared"}] - names = [ - entry.rsplit(" :: ", 1)[-1] for entry in metadata.get_all("Classifier", []) if entry.startswith("License :: ") - ] - if (declared := metadata.get("License")) and "\n" not in declared: - names.append(declared) - licenses: Final[list[License]] = [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] - # a license policy judges SPDX ids only, so a component keeps its bare names when any one of them does not resolve, - # and the policy reports it instead of judging a partial set - identifiers: Final[list[str]] = [identifier for name in names if (identifier := _spdx_license(name)) is not None] - if len(identifiers) == len(names): - licenses.extend( - {"license": {"id": identifier, "acknowledgement": "concluded"}} for identifier in dict.fromkeys(identifiers) - ) - return licenses - - -def _spdx_license(name: str) -> str | None: - # only the license classifiers that name exactly one SPDX license; "BSD License" or "Apache Software License" leave - # the version or variant open - if identifier := { - "ISC License (ISCL)": "ISC", - "MIT License": "MIT", - "MIT No Attribution License (MIT-0)": "MIT-0", - "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", - "Python Software Foundation License": "PSF-2.0", - "The Unlicense (Unlicense)": "Unlicense", - }.get(name): - return identifier - try: - canonical = canonicalize_license_expression(name) - except InvalidLicenseExpression: - return None - # a CycloneDX license id holds one entry of the SPDX license list, never an expression or a LicenseRef - return canonical if re.fullmatch(r"[A-Za-z0-9.-]+", canonical) and not canonical.startswith("LicenseRef-") else None - - -def external_reference(label: str, url: str) -> ExternalReference: - reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") - return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} - - -def file_component(parent_ref: str, path: str, digest: str, size: str) -> Component: - # RECORD stores "=", CycloneDX wants lowercase hex - algorithm, _, encoded = digest.partition("=") - algorithms: Final[dict[str, str]] = { - "md5": "MD5", - "sha1": "SHA-1", - "sha256": "SHA-256", - "sha384": "SHA-384", - "sha512": "SHA-512", - } - raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) - return { - "type": "file", - "bom-ref": f"{parent_ref}#{path}", - "name": path, - "hashes": [{"alg": algorithms[algorithm], "content": raw.hex()}], - "properties": [{"name": "size", "value": size}], - } - - -def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) -> list[str]: - refs = set() - for requirement in map(Requirement, distribution.requires or []): - if (requirement.marker is None or requirement.marker.evaluate()) and ( - dependency := installed.get(purl(requirement.name)) - ): - refs.add(f"tool:{purl(dependency.metadata['Name'], dependency.version)}") - return sorted(refs) - - -def record_digest(content: bytes) -> str: - return f"sha256={base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b'=').decode()}" - - -__all__ = [ - "Body", - "Component", - "Composition", - "Contact", - "Dependency", - "Document", - "ExternalReference", - "License", - "LicenseInfo", - "Lifecycle", - "Metadata", - "OrganizationalEntity", - "Property", - "Tools", - "build_tools", - "component_from_metadata", - "contacts", - "document", - "external_reference", - "file_component", - "purl", - "record_digest", - "timestamp", -] diff --git a/src/python_sbom/_embed.py b/src/python_sbom/_embed.py deleted file mode 100644 index c9da7c3..0000000 --- a/src/python_sbom/_embed.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -import json -import os -import tempfile -import time -import zipfile -from email.parser import Parser -from pathlib import Path -from typing import TYPE_CHECKING, Final - -from hatchling.builders.utils import get_reproducible_timestamp - -from ._cyclonedx import record_digest -from ._project import sbom_file_name - -if TYPE_CHECKING: - from ._project import Project - - -def embed(wheel: Path, project: Project) -> str: - """ - Write the SBOM into a wheel some other build backend already built, as the build hook would have. - - The SBOM goes under ``.dist-info/sboms/`` with a ``RECORD`` row, replacing an SBOM of the same name. Every other - entry keeps its name, order, timestamp and permissions, and the new entry takes its timestamp from - ``SOURCE_DATE_EPOCH``, so embedding into the same wheel twice gives the same bytes. - - :param wheel: the wheel to rewrite in place - :param project: the source tree the wheel was built from - :returns: the SBOM's path inside the wheel - - """ - with zipfile.ZipFile(wheel) as archive: - entries: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [ - (info, archive.read(info)) for info in archive.infolist() - ] - dist_info: Final[str] = next( - info.filename.split("/")[0] - for info, _ in entries - if info.filename.count("/") == 1 and info.filename.endswith(".dist-info/METADATA") - ) - contents: Final[dict[str, bytes]] = {info.filename: content for info, content in entries} - metadata = Parser().parsestr(contents[f"{dist_info}/METADATA"].decode("utf-8")) - sbom: Final[str] = f"{dist_info}/sboms/{sbom_file_name(metadata)}" - content: Final[bytes] = f"{json.dumps(project.describe(metadata, wheel.name), indent=2)}\n".encode() - record: Final[str] = f"{dist_info}/RECORD" - lines: Final[list[str]] = [ - line for line in contents[record].decode("utf-8").splitlines(keepends=True) if not line.startswith(f"{sbom},") - ] - newline = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" - # the RECORD row of RECORD itself stays last - at = next((index for index, line in enumerate(lines) if line.startswith(f"{record},")), len(lines)) - lines.insert(at, f"{sbom},{record_digest(content)},{len(content)}{newline}") - entry = zipfile.ZipInfo(sbom, time.gmtime(max(get_reproducible_timestamp(), 315532800))[:6]) # zip starts at 1980 - entry.compress_type = zipfile.ZIP_DEFLATED - entry.external_attr = 0o644 << 16 - # written next to the wheel and moved over it, so a failure leaves the wheel as it was - descriptor, temporary = tempfile.mkstemp(dir=wheel.parent, suffix=".whl") - os.close(descriptor) - with zipfile.ZipFile(temporary, "w") as archive: - for info, data in entries: - if info.filename == record: - archive.writestr(entry, content) - archive.writestr(info, "".join(lines).encode()) - elif info.filename != sbom: - archive.writestr(info, data) - Path(temporary).replace(wheel) - return sbom - - -__all__ = [ - "embed", -] diff --git a/src/python_sbom/_errors.py b/src/python_sbom/_errors.py new file mode 100644 index 0000000..5617ce8 --- /dev/null +++ b/src/python_sbom/_errors.py @@ -0,0 +1,25 @@ +from __future__ import annotations + + +class SbomError(Exception): + """Base of the errors this package raises; the command line reports them without a traceback.""" + + +class SettingsError(SbomError, TypeError): + """A setting has the wrong type.""" + + +class WheelError(SbomError, ValueError): + """A wheel, built or bundled, lacks what its SBOM needs to describe it.""" + + +class SourceRevisionError(SbomError): + """The project is a git checkout, but git cannot read its revision.""" + + +__all__ = [ + "SbomError", + "SettingsError", + "SourceRevisionError", + "WheelError", +] diff --git a/src/python_sbom/_extension.py b/src/python_sbom/_extension.py new file mode 100644 index 0000000..880de70 --- /dev/null +++ b/src/python_sbom/_extension.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from pathlib import Path + + from ._model._types import Property + + +class ContentProvider(Protocol): + """Adds project-specific content to the SBOM; the hatchling hook and every backend implement it.""" + + def bundled_wheel_properties(self, wheel: Path) -> list[Property]: + """ + Return extra properties for a wheel the project bundles as a data file. + + The properties follow ``:bundled-wheel`` and precede the ones taken from the wheel's metadata. + + :param wheel: the bundled wheel, an absolute path under the project root + :returns: the properties to add + + """ + ... + + +__all__ = [ + "ContentProvider", +] diff --git a/src/python_sbom/_formats/__init__.py b/src/python_sbom/_formats/__init__.py new file mode 100644 index 0000000..539b271 --- /dev/null +++ b/src/python_sbom/_formats/__init__.py @@ -0,0 +1 @@ +"""Serialization of documents: CycloneDX JSON and the SPDX 2.3 rendering.""" diff --git a/src/python_sbom/_formats/_json.py b/src/python_sbom/_formats/_json.py new file mode 100644 index 0000000..c65b888 --- /dev/null +++ b/src/python_sbom/_formats/_json.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from python_sbom._model._types import Document + + from ._spdx import Spdx + + +def dumps(document: Document | Spdx) -> bytes: + """ + Serialize a document the way python-sbom writes it: two-space indented JSON and a final newline. + + :param document: a CycloneDX or SPDX document + :returns: UTF-8 bytes, with the same line endings on every operating system + + """ + return f"{json.dumps(document, indent=2)}\n".encode() + + +__all__ = [ + "dumps", +] diff --git a/src/python_sbom/_spdx.py b/src/python_sbom/_formats/_spdx.py similarity index 92% rename from src/python_sbom/_spdx.py rename to src/python_sbom/_formats/_spdx.py index 665fd95..f4db7a8 100644 --- a/src/python_sbom/_spdx.py +++ b/src/python_sbom/_formats/_spdx.py @@ -1,15 +1,16 @@ from __future__ import annotations +import hashlib import re from datetime import UTC, datetime from typing import TYPE_CHECKING, Final, NotRequired, TypedDict -from .version import version +from python_sbom.version import version if TYPE_CHECKING: from collections.abc import Iterator - from ._cyclonedx import Component, Document, License + from python_sbom._model._types import Component, Document, License class _Checksum(TypedDict): @@ -81,7 +82,7 @@ def to_spdx(bom: Document) -> Spdx: """ root: Final[Component] = bom["metadata"]["component"] tools: Final[list[Component]] = bom["metadata"]["tools"]["components"] - name: Final[str] = f"{root['name']}-{root['version']}" + name: Final[str] = f"{root['name']}-{root['version']}" if "version" in root else root["name"] namespace: Final[str] = re.sub(r"[-_.]+", "-", root["name"]).lower() repository: Final[str] = next( ( @@ -126,7 +127,7 @@ def to_spdx(bom: Document) -> Spdx: *( _relationship(_spdx_id(entry["ref"]), "DEPENDS_ON", _spdx_id(dependency)) for entry in bom["dependencies"] - for dependency in entry["dependsOn"] + for dependency in entry.get("dependsOn", []) ), *(_relationship(_spdx_id(tool["bom-ref"]), "BUILD_TOOL_OF", _spdx_id(root["bom-ref"])) for tool in tools), ], @@ -196,7 +197,10 @@ def _relationship(element: str, kind: str, related: str) -> _Relationship: def _spdx_id(bom_ref: str) -> str: - return f"SPDXRef-{re.sub(r'[^A-Za-z0-9.]+', '-', bom_ref).strip('-')}" + # SPDX ids allow letters, digits, "." and "-"; the digest keeps two refs that differ only in other characters, + # such as the markers of two conditional requirements, apart + readable = re.sub(r"[^A-Za-z0-9.]+", "-", bom_ref).strip("-") + return f"SPDXRef-{readable}-{hashlib.sha256(bom_ref.encode()).hexdigest()[:8]}" __all__ = [ diff --git a/src/python_sbom/_model/__init__.py b/src/python_sbom/_model/__init__.py new file mode 100644 index 0000000..d137f14 --- /dev/null +++ b/src/python_sbom/_model/__init__.py @@ -0,0 +1 @@ +"""The CycloneDX document model: its types and the pure functions that build it.""" diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py new file mode 100644 index 0000000..d971396 --- /dev/null +++ b/src/python_sbom/_model/_components.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import re +import uuid +from email.utils import getaddresses +from typing import TYPE_CHECKING, Final + +from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression +from packaging.utils import canonicalize_name + +if TYPE_CHECKING: + from email.message import Message + from importlib.metadata import PackageMetadata + + from ._types import Body, Component, Contact, Document, ExternalReference, License + + +# the label normalization and mapping cyclonedx-py applies to Project-URL entries, plus "source", so the same metadata +# yields the same reference types whichever generator produced the document +_URL_LABEL_TO_REFERENCE_TYPE: Final[dict[str, str]] = { + "bugtracker": "issue-tracker", + "issuetracker": "issue-tracker", + "issues": "issue-tracker", + "bugreports": "issue-tracker", + "tracker": "issue-tracker", + "home": "website", + "homepage": "website", + "download": "distribution", + "documentation": "documentation", + "docs": "documentation", + "changelog": "release-notes", + "changes": "release-notes", + "source": "vcs", + "repository": "vcs", + "github": "vcs", + "chat": "chat", +} + + +def document(body: Body, namespace: uuid.UUID) -> Document: + """ + Add the CycloneDX 1.6 header to a document body. + + :param body: the metadata, components, dependencies and formulation + :param namespace: UUID namespace of the serial number, one per project + :returns: the document; its serial number derives from the body, so identical bodies share one + + """ + return { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": f"urn:uuid:{uuid.uuid5(namespace, json.dumps(body, sort_keys=True))}", + "version": 1, + **body, + } + + +def component_from_metadata(metadata: PackageMetadata | Message, component_type: str) -> Component: + """ + Describe a distribution from its core metadata. + + :param metadata: the distribution's ``METADATA`` + :param component_type: the CycloneDX component type, such as ``library`` + :returns: the component, with its purl as ``bom-ref``, licenses, Project-URL references and metadata properties + + """ + name, version = metadata["Name"], metadata["Version"] + package = purl(name, version) + component: Final[Component] = {"type": component_type, "bom-ref": package, "name": name, "version": version} + component["purl"] = package + if summary := metadata.get("Summary"): + component["description"] = summary + if authors := contacts( + metadata.get_all("Author", []) + metadata.get_all("Maintainer", []), + metadata.get_all("Author-email", []) + metadata.get_all("Maintainer-email", []), + ): + component["authors"] = authors + if licenses := _licenses(metadata): + component["licenses"] = licenses + component["externalReferences"] = [ + external_reference(label.strip(), url.strip()) + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) + ] + if home_page := metadata.get("Home-page"): + component["externalReferences"].insert(0, {"type": "website", "url": home_page}) + # core metadata headers copied verbatim onto the component + component["properties"] = [ + {"name": f"python:{header.lower()}", "value": value} + for header in ("Requires-Python", "Requires-Dist", "Provides-Extra", "Classifier", "Keywords") + for value in metadata.get_all(header, []) + ] + return component + + +def purl(name: str, version: str | None = None) -> str: + return f"pkg:pypi/{canonicalize_name(name)}@{version}" if version else f"pkg:pypi/{canonicalize_name(name)}" + + +def contacts(names: list[str], addresses: list[str]) -> list[Contact]: + found: Final[list[Contact]] = [{"name": name} for name in names] + for name, email in getaddresses(addresses): + contact: Contact = {} + if name: + contact["name"] = name + if email: + contact["email"] = email + found.append(contact) + return found + + +def _licenses(metadata: PackageMetadata | Message) -> list[License]: + if expression := metadata.get("License-Expression"): + return [{"expression": expression, "acknowledgement": "declared"}] + names = [ + entry.rsplit(" :: ", 1)[-1] for entry in metadata.get_all("Classifier", []) if entry.startswith("License :: ") + ] + if (declared := metadata.get("License")) and "\n" not in declared: + names.append(declared) + licenses: Final[list[License]] = [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] + # a license policy judges SPDX ids only, so a component keeps its bare names when any one of them does not resolve, + # and the policy reports it instead of judging a partial set + identifiers: Final[list[str]] = [identifier for name in names if (identifier := _spdx_license(name)) is not None] + if len(identifiers) == len(names): + licenses.extend( + {"license": {"id": identifier, "acknowledgement": "concluded"}} for identifier in dict.fromkeys(identifiers) + ) + return licenses + + +def _spdx_license(name: str) -> str | None: + # only the license classifiers that name exactly one SPDX license; "BSD License" or "Apache Software License" leave + # the version or variant open + if identifier := { + "ISC License (ISCL)": "ISC", + "MIT License": "MIT", + "MIT No Attribution License (MIT-0)": "MIT-0", + "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", + "Python Software Foundation License": "PSF-2.0", + "The Unlicense (Unlicense)": "Unlicense", + }.get(name): + return identifier + try: + canonical = canonicalize_license_expression(name) + except InvalidLicenseExpression: + return None + # a CycloneDX license id holds one entry of the SPDX license list, never an expression or a LicenseRef + return canonical if re.fullmatch(r"[A-Za-z0-9.-]+", canonical) and not canonical.startswith("LicenseRef-") else None + + +def external_reference(label: str, url: str) -> ExternalReference: + reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") + return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} + + +def file_component(parent_ref: str, path: str, digest: str, size: str) -> Component: + # RECORD stores "=", CycloneDX wants lowercase hex + algorithm, _, encoded = digest.partition("=") + algorithms: Final[dict[str, str]] = { + "md5": "MD5", + "sha1": "SHA-1", + "sha256": "SHA-256", + "sha384": "SHA-384", + "sha512": "SHA-512", + } + raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + return { + "type": "file", + "bom-ref": f"{parent_ref}#{path}", + "name": path, + "hashes": [{"alg": algorithms[algorithm], "content": raw.hex()}], + "properties": [{"name": "size", "value": size}], + } + + +def record_digest(content: bytes) -> str: + return f"sha256={base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b'=').decode()}" + + +__all__ = [ + "component_from_metadata", + "contacts", + "document", + "external_reference", + "file_component", + "purl", + "record_digest", +] diff --git a/src/python_sbom/_model/_types.py b/src/python_sbom/_model/_types.py new file mode 100644 index 0000000..7593199 --- /dev/null +++ b/src/python_sbom/_model/_types.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +from typing import NotRequired, TypedDict + + +class OrganizationalEntity(TypedDict): + """An organization that supplies or makes a component.""" + + name: str + url: list[str] + + +class Contact(TypedDict): + name: NotRequired[str] + email: NotRequired[str] + + +class _Hash(TypedDict): + alg: str + content: str + + +class Property(TypedDict): + """A name and value pair; names this package writes start with the project name or ``python:``.""" + + name: str + value: str + + +class ExternalReference(TypedDict): + type: str + url: str + comment: NotRequired[str] + + +class LicenseInfo(TypedDict): + id: NotRequired[str] + name: NotRequired[str] + acknowledgement: str + + +class License(TypedDict): + license: NotRequired[LicenseInfo] + expression: NotRequired[str] + acknowledgement: NotRequired[str] + + +class _Method(TypedDict): + technique: str + confidence: int + value: str + + +class _Identity(TypedDict): + field: str + confidence: int + methods: list[_Method] + + +class _Evidence(TypedDict): + identity: list[_Identity] + + +#: A CycloneDX component: a distribution, an application, the interpreter or a file. +Component = TypedDict( + "Component", + { + "type": str, + "bom-ref": str, + "name": str, + "version": NotRequired[str], + "purl": NotRequired[str], + "supplier": NotRequired[OrganizationalEntity], + "authors": NotRequired[list[Contact]], + "description": NotRequired[str], + "licenses": NotRequired[list[License]], + "copyright": NotRequired[str], + "hashes": NotRequired[list[_Hash]], + "externalReferences": NotRequired[list[ExternalReference]], + "properties": NotRequired[list[Property]], + "evidence": NotRequired[_Evidence], + "components": NotRequired[list["Component"]], + }, +) + + +class Dependency(TypedDict): + """The components one component depends on, by ``bom-ref``.""" + + ref: str + dependsOn: NotRequired[list[str]] + + +class _Reference(TypedDict): + ref: str + + +class _Output(TypedDict): + type: str + resource: _Reference + + +class _Input(TypedDict): + environmentVars: list[Property] + + +_Workflow = TypedDict( + "_Workflow", + { + "bom-ref": str, + "uid": str, + "name": str, + "taskTypes": list[str], + "resourceReferences": list[_Reference], + "outputs": list[_Output], + "inputs": list[_Input], + }, +) + + +_Formula = TypedDict("_Formula", {"bom-ref": str, "workflows": list[_Workflow]}) + + +class Lifecycle(TypedDict): + phase: str + + +class Tools(TypedDict): + components: list[Component] + + +class Metadata(TypedDict): + timestamp: str + lifecycles: list[Lifecycle] + tools: Tools + manufacturer: NotRequired[OrganizationalEntity] + authors: NotRequired[list[Contact]] + supplier: NotRequired[OrganizationalEntity] + component: Component + licenses: NotRequired[list[License]] + properties: NotRequired[list[Property]] + + +class Composition(TypedDict): + aggregate: str + dependencies: list[str] + + +class Body(TypedDict): + """The content of a document, everything but the header :func:`document` adds.""" + + metadata: Metadata + components: list[Component] + dependencies: list[Dependency] + compositions: NotRequired[list[Composition]] + formulation: NotRequired[list[_Formula]] + + +#: A complete CycloneDX 1.6 JSON document. +Document = TypedDict( + "Document", + { + "$schema": str, + "bomFormat": str, + "specVersion": str, + "serialNumber": str, + "version": int, + "metadata": Metadata, + "components": list[Component], + "dependencies": list[Dependency], + "compositions": NotRequired[list[Composition]], + "formulation": NotRequired[list[_Formula]], + }, +) + + +__all__ = [ + "Body", + "Component", + "Composition", + "Contact", + "Dependency", + "Document", + "ExternalReference", + "License", + "LicenseInfo", + "Lifecycle", + "Metadata", + "OrganizationalEntity", + "Property", + "Tools", +] diff --git a/src/python_sbom/_model/_wheel.py b/src/python_sbom/_model/_wheel.py new file mode 100644 index 0000000..97bffc4 --- /dev/null +++ b/src/python_sbom/_model/_wheel.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +import csv +import uuid +from dataclasses import dataclass +from io import StringIO +from pathlib import PurePosixPath +from typing import TYPE_CHECKING, Final +from urllib.parse import quote, urlsplit + +from packaging.markers import Marker +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +from ._components import component_from_metadata, contacts, document, external_reference, file_component, purl + +if TYPE_CHECKING: + from collections.abc import Sequence + from email.message import Message + + from python_sbom._collect._facts import BuildFacts + from python_sbom._collect._settings import Settings + from python_sbom._collect._source import BuildScript, BundledWheel, SourceTree + + from ._types import Component, Contact, Dependency, Document, ExternalReference, License, Property + + +@dataclass(frozen=True, slots=True, kw_only=True) +class WheelInputs: + # Everything the document of one wheel is built from, gathered beforehand. + metadata: Message + wheel_name: str + settings: Settings + facts: BuildFacts + source: SourceTree + + +def describe(inputs: WheelInputs) -> Document: + metadata, settings = inputs.metadata, inputs.settings + namespace = canonicalize_name(metadata["Name"]) + repository = settings.repository or next( + ( + url.strip().rstrip("/") + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) + if external_reference(label.strip(), url.strip())["type"] == "vcs" + ), + None, + ) + directories = " and ".join(sorted({str(PurePosixPath(pattern).parent) for pattern in settings.bundled_wheels})) + scope = ( + f"Components are the wheels bundled as data files under {directories} and the direct runtime dependencies" + " declared in the wheel's core metadata. The latter carry no version because they are resolved at install" + " time, not at build time." + if directories + else "Components are the direct runtime dependencies declared in the wheel's core metadata. They carry no" + " version because they are resolved at install time, not at build time." + ) + root = _root_component(inputs, repository) + bundled = [_bundled_component(wheel, namespace, settings, repository) for wheel in inputs.source.bundled] + declared = [_declared_dependency(requirement, namespace) for requirement in _runtime_requirements(metadata)] + bundled_dependencies: Final[list[Dependency]] = [ + {"ref": component["bom-ref"], "dependsOn": references} + for component in bundled + if (references := [child["bom-ref"] for child in component["components"] if child["type"] == "library"]) + ] + tools: Final[list[Component]] = list(inputs.facts.tools) + if (script := inputs.source.build_script) is not None: + tools.insert(0, _build_script_tool(script, metadata["Version"], settings, repository)) + authors = _authors(metadata) + licenses = _licenses(metadata) + return document( + { + "metadata": { + "timestamp": inputs.facts.timestamp, + "lifecycles": [{"phase": "build"}], + "tools": {"components": tools}, + **({"manufacturer": settings.supplier} if settings.supplier else {}), + **({"authors": authors} if authors else {}), + **({"supplier": settings.supplier} if settings.supplier else {}), + "component": root, + **({"licenses": licenses} if licenses else {}), + "properties": [{"name": f"{namespace}:sbom:scope", "value": scope}], + }, + "components": [*bundled, *declared], + "dependencies": [ + {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in [*bundled, *declared]]}, + *bundled_dependencies, + *inputs.facts.tool_dependencies, + ], + # transitive runtime dependencies are unknown until install time + "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], + "formulation": [ + { + "bom-ref": "formula:wheel-build", + "workflows": [ + { + "bom-ref": "workflow:wheel-build", + "uid": "wheel-build", + "name": "build the wheel and this SBOM", + "taskTypes": ["build"], + "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], + "outputs": [{"type": "artifact", "resource": {"ref": root["bom-ref"]}}], + "inputs": [ + {"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": str(inputs.facts.epoch)}]} + ], + } + ], + } + ], + }, + uuid.uuid5(uuid.NAMESPACE_URL, f"{repository or f'https://pypi.org/project/{namespace}'}/sboms"), + ) + + +def _root_component(inputs: WheelInputs, repository: str | None) -> Component: + metadata, settings = inputs.metadata, inputs.settings + name, version = metadata["Name"], metadata["Version"] + namespace = canonicalize_name(name) + references: Final[list[ExternalReference]] = [ + external_reference(label.strip(), url.strip()) + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) + ] + references.extend([ + {"type": "distribution", "url": f"https://pypi.org/project/{name}/{version}/"}, + { + "type": "attestation", + "url": f"https://pypi.org/integrity/{name}/{version}/{inputs.wheel_name}/provenance", + }, + ]) + if repository and urlsplit(repository).netloc == "github.com": + references.extend([ + {"type": "security-contact", "url": f"{repository}/security/policy"}, + {"type": "advisories", "url": f"{repository}/security/advisories"}, + ]) + license_files: Final[list[str]] = metadata.get_all("License-File", []) + if repository and license_files: + references.append({"type": "license", "url": f"{repository}/blob/{settings.default_branch}/{license_files[0]}"}) + properties: Final[list[Property]] = [] + if requires_python := metadata.get("Requires-Python"): + properties.append({"name": f"{namespace}:requires-python", "value": requires_python}) + properties.extend({"name": "python:classifier", "value": value} for value in metadata.get_all("Classifier", [])) + properties.extend( + {"name": "python:keyword", "value": keyword.strip()} + for keyword in (metadata.get("Keywords") or "").split(",") + if keyword.strip() + ) + if commit := inputs.facts.commit: + if repository: + references.append({"type": "vcs", "url": f"{repository}/tree/{commit}", "comment": "exact source revision"}) + properties.append({"name": f"{namespace}:vcs-commit", "value": commit}) + authors = _authors(metadata) + licenses = _licenses(metadata) + package = purl(name, version) + return { + "type": "application", + "bom-ref": package, + **({"supplier": settings.supplier} if settings.supplier else {}), + **({"authors": authors} if authors else {}), + "name": name, + "version": version, + **({"description": summary} if (summary := metadata.get("Summary")) else {}), + **({"licenses": licenses} if licenses else {}), + **({"copyright": inputs.source.copyright} if inputs.source.copyright else {}), + "purl": package, + "externalReferences": references, + "properties": properties, + } + + +def _bundled_component(wheel: BundledWheel, namespace: str, settings: Settings, repository: str | None) -> Component: + component = component_from_metadata(wheel.metadata, "library") + # the purl names the package; two variants of one release, such as per-platform wheels, are told apart by file + component["bom-ref"] = f"{component['bom-ref']}?file_name={quote(PurePosixPath(wheel.path).name)}" + # a bundled wheel published from the same forge account as this project shares its supplier + if ( + settings.supplier + and repository + and any(reference["url"].startswith(_owner(repository)) for reference in component["externalReferences"]) + ): + component["supplier"] = settings.supplier + component["hashes"] = [{"alg": "SHA-256", "content": wheel.sha256}] + component["externalReferences"].append({ + "type": "distribution", + "url": f"https://pypi.org/project/{wheel.metadata['Name']}/{wheel.metadata['Version']}/", + }) + component["properties"] = [ + {"name": f"{namespace}:bundled-wheel", "value": wheel.path}, + *wheel.extra_properties, + *component["properties"], + ] + component["evidence"] = { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": wheel.metadata_name}], + }, + { + "field": "hash", + "confidence": 1, + "methods": [{"technique": "hash-comparison", "confidence": 1, "value": wheel.sha256}], + }, + ], + } + component["components"] = [ + file_component(component["bom-ref"], path, digest, size) + for path, digest, size in (row for row in csv.reader(StringIO(wheel.record, newline="")) if row) + if digest + ] + children: Final[dict[str, Component]] = {} + for source, child_metadata in wheel.vendored: + child = component_from_metadata(child_metadata, "library") + child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['bom-ref']}" + child["properties"].append({"name": f"{namespace}:vendored-manifest", "value": source}) + child["evidence"] = { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], + } + ], + } + # a vendored .dist-info comes after the pip manifest line naming the same purl, so its metadata wins + children[purl(child["name"], child["version"])] = child + component["components"].extend(children[key] for key in sorted(children)) + return component + + +def _owner(repository: str) -> str: + parts = urlsplit(repository) + return f"{parts.scheme}://{parts.netloc}/{parts.path.strip('/').split('/')[0]}/" + + +def _declared_dependency(requirement: str, namespace: str) -> Component: + parsed = Requirement(requirement) + component: Final[Component] = { + "type": "library", + # the same distribution can be declared more than once with different markers, so the purl is not unique + "bom-ref": f"requires-dist:{requirement}", + "name": parsed.name, + "purl": purl(parsed.name), + "externalReferences": [{"type": "distribution", "url": f"https://pypi.org/project/{parsed.name}/"}], + "properties": [{"name": f"{namespace}:requires-dist", "value": requirement}], + } + if parsed.marker is not None: + component["properties"].append({"name": f"{namespace}:environment-marker", "value": str(parsed.marker)}) + return component + + +def _runtime_requirements(metadata: Message) -> list[str]: + # a requirement belongs to the wheel's runtime dependencies when it can apply with no extra requested + return [ + requirement + for requirement in metadata.get_all("Requires-Dist", []) + if (marker := Requirement(requirement).marker) is None or _without_extras(marker) is not False + ] + + +def _without_extras(marker: Marker) -> bool | None: + # three-valued: an extra condition is decided with no extra requested, any other stays unknown, so a requirement + # is kept unless its extra conditions alone rule it out; packaging exposes the parsed tree only privately + return _any_of(marker._markers) # ruff: ignore[private-member-access] # no public marker tree + + +def _any_of( + markers: Sequence[object], # simplint: ignore[SMP602] # packaging's private tree is untyped +) -> bool | None: + # "and" binds tighter than "or", as in packaging's own evaluation + groups: Final[list[list[bool | None]]] = [[]] + for item in markers: + if item == "or": + groups.append([]) + elif isinstance(item, list): + groups[-1].append(_any_of(item)) + elif isinstance(item, tuple): # skips "and", which grouping already applies + groups[-1].append(_atom(item)) + verdicts = [False if False in group else (None if None in group else True) for group in groups] + return True if True in verdicts else (None if None in verdicts else False) + + +def _atom(item: tuple[object, ...]) -> bool | None: # simplint: ignore[SMP602] # packaging's private tree + kinds = [type(part).__name__ for part in item] + left, operator, right = (str(part) for part in item) + if kinds[0] == "Variable" and left == "extra": + return Marker(f"extra {operator} {right!r}").evaluate({"extra": ""}) + if kinds[2] == "Variable" and right == "extra": + return Marker(f"{left!r} {operator} extra").evaluate({"extra": ""}) + return None + + +def _build_script_tool(script: BuildScript, version: str, settings: Settings, repository: str | None) -> Component: + tool: Final[Component] = { + "type": "application", + "bom-ref": f"tool:{script.path}", + "name": script.path.name, + "version": version, + "hashes": [{"alg": "SHA-256", "content": script.sha256}], + } + if repository: + tool["externalReferences"] = [ + {"type": "vcs", "url": f"{repository}/blob/{settings.default_branch}/{script.path}"} + ] + return tool + + +def _authors(metadata: Message) -> list[Contact]: + # core metadata joins several names into one header + if people := contacts( + [name for entry in metadata.get_all("Maintainer", []) for name in entry.split(", ")], + metadata.get_all("Maintainer-email", []), + ): + return people + return contacts( + [name for entry in metadata.get_all("Author", []) for name in entry.split(", ")], + metadata.get_all("Author-email", []), + ) + + +def _licenses(metadata: Message) -> list[License]: + if expression := metadata.get("License-Expression"): + return [{"expression": expression, "acknowledgement": "declared"}] + return [] + + +__all__ = [ + "WheelInputs", + "describe", +] diff --git a/src/python_sbom/_project.py b/src/python_sbom/_project.py deleted file mode 100644 index 13fe8d9..0000000 --- a/src/python_sbom/_project.py +++ /dev/null @@ -1,487 +0,0 @@ -from __future__ import annotations - -import csv -import hashlib -import shutil -import subprocess # ruff: ignore[suspicious-subprocess-import] # runs git to read the source revision -import uuid -import zipfile -from dataclasses import dataclass -from email.parser import Parser -from io import StringIO -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Final, Protocol, TypeAlias -from urllib.parse import urlsplit - -from hatchling.builders.utils import get_reproducible_timestamp -from packaging.requirements import Requirement -from packaging.utils import canonicalize_name - -from ._cyclonedx import ( - build_tools, - component_from_metadata, - contacts, - document, - external_reference, - file_component, - purl, - timestamp, -) - -if TYPE_CHECKING: - from collections.abc import Mapping - from datetime import date, datetime, time - from email.message import Message - - from ._cyclonedx import ( - Component, - Contact, - Dependency, - Document, - ExternalReference, - License, - OrganizationalEntity, - Property, - ) - -#: a value of a TOML table, as :mod:`tomllib` or hatchling hand it over -Toml: TypeAlias = "str | int | float | bool | datetime | date | time | list[Toml] | dict[str, Toml]" - - -class BundledWheelProperties(Protocol): - """Extra properties for a wheel the project bundles as a data file.""" - - def __call__(self, wheel: Path, /) -> list[Property]: - """ - Describe one bundled wheel. - - :param wheel: the bundled wheel, an absolute path under the project root - :returns: the properties to add after ``:bundled-wheel`` - - """ - - -def _no_bundled_wheel_properties(_wheel: Path, /) -> list[Property]: - return [] - - -@dataclass(frozen=True, slots=True, kw_only=True) -class Project: - """ - The source tree a wheel comes from, and what its SBOM should say about it. - - :param root: the project directory, where ``.git`` and the license files are - :param supplier: the organization that supplies the project, or ``None`` to leave the supplier out - :param repository: the repository URL, or ``None`` to take the first ``Source``, ``Repository`` or ``GitHub`` - project URL of the metadata - :param default_branch: branch used in links to files in the repository - :param bundled_wheels: glob patterns, relative to ``root``, of wheels the project ships as data files - :param build_script: path, relative to ``root``, of a build script to list among the build tools, or ``None`` - :param bundled_wheel_properties: extra properties for each bundled wheel - - """ - - root: Path - supplier: OrganizationalEntity | None - repository: str | None - default_branch: str - bundled_wheels: tuple[str, ...] - build_script: PurePosixPath | None - bundled_wheel_properties: BundledWheelProperties - - @classmethod - def from_table( - cls, - table: Mapping[str, Toml], - *, - root: Path, - where: str, - build_script: PurePosixPath | None = None, - bundled_wheel_properties: BundledWheelProperties = _no_bundled_wheel_properties, - ) -> Project: - """ - Read the settings from a configuration table, ignoring keys it does not know. - - :param table: the ``supplier``, ``repository``, ``default-branch`` and ``bundled-wheels`` settings - :param root: the project directory - :param where: where the table lives, for error messages, such as ``[tool.python-sbom]`` - :param build_script: path, relative to ``root``, of a build script to list among the build tools - :param bundled_wheel_properties: extra properties for each bundled wheel - :returns: the project - :raises TypeError: when a setting has the wrong type - - """ - supplier = table.get("supplier") - if supplier is not None and not ( - isinstance(supplier, dict) - and supplier.keys() <= {"name", "url"} - and isinstance(name := supplier.get("name"), str) - and isinstance(urls := supplier.get("url", []), list) - and all(isinstance(url, str) for url in urls) - ): - msg = f"`supplier` in {where} must be a table with a string `name` and an optional `url` string array" - raise TypeError(msg) - if not isinstance(repository := table.get("repository"), str | None): - msg = f"`repository` in {where} must be a string" - raise TypeError(msg) - if not isinstance(default_branch := table.get("default-branch", "main"), str): - msg = f"`default-branch` in {where} must be a string" - raise TypeError(msg) - if not ( - isinstance(patterns := table.get("bundled-wheels", []), list) - and all(isinstance(pattern, str) for pattern in patterns) - ): - msg = f"`bundled-wheels` in {where} must be an array of strings" - raise TypeError(msg) - return cls( - root=root, - supplier=None if supplier is None else {"name": name, "url": [str(url) for url in urls]}, - repository=None if repository is None else repository.rstrip("/"), - default_branch=default_branch, - bundled_wheels=tuple(str(pattern) for pattern in patterns), - build_script=build_script, - bundled_wheel_properties=bundled_wheel_properties, - ) - - def describe(self, metadata: Message, wheel_name: str) -> Document: - """ - Write the SBOM of a wheel. - - :param metadata: the wheel's ``METADATA`` - :param wheel_name: the wheel's file name, for its attestation link - :returns: the CycloneDX 1.6 document - - """ - namespace = canonicalize_name(metadata["Name"]) - repository = self.repository or self._project_url_repository(metadata) - root = self._root_component(metadata, wheel_name, repository) - bundled = [self._bundled_component(wheel, namespace, repository) for wheel in self._bundled_wheel_paths()] - declared = [_declared_dependency(requirement, namespace) for requirement in _runtime_requirements(metadata)] - bundled_dependencies: Final[list[Dependency]] = [ - {"ref": component["bom-ref"], "dependsOn": references} - for component in bundled - if (references := [child["bom-ref"] for child in component["components"] if child["type"] == "library"]) - ] - tools, tool_dependencies = build_tools() - if self.build_script is not None: - tools.insert(0, self._build_script_tool(self.build_script, metadata["Version"], repository)) - authors = _authors(metadata) - licenses = _licenses(metadata) - base = repository or f"https://pypi.org/project/{namespace}" - return document( - { - "metadata": { - "timestamp": timestamp(), - "lifecycles": [{"phase": "build"}], - "tools": {"components": tools}, - **({"manufacturer": self.supplier} if self.supplier else {}), - **({"authors": authors} if authors else {}), - **({"supplier": self.supplier} if self.supplier else {}), - "component": root, - **({"licenses": licenses} if licenses else {}), - "properties": [{"name": f"{namespace}:sbom:scope", "value": self._scope()}], - }, - "components": [*bundled, *declared], - "dependencies": [ - { - "ref": root["bom-ref"], - "dependsOn": [component["bom-ref"] for component in [*bundled, *declared]], - }, - *bundled_dependencies, - *tool_dependencies, - ], - # transitive runtime dependencies are unknown until install time - "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], - "formulation": [ - { - "bom-ref": "formula:wheel-build", - "workflows": [ - { - "bom-ref": "workflow:wheel-build", - "uid": "wheel-build", - "name": "build the wheel and this SBOM", - "taskTypes": ["build"], - "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], - "outputs": [{"type": "artifact", "resource": {"ref": root["bom-ref"]}}], - "inputs": [ - { - "environmentVars": [ - {"name": "SOURCE_DATE_EPOCH", "value": str(get_reproducible_timestamp())} - ] - } - ], - } - ], - } - ], - }, - uuid.uuid5(uuid.NAMESPACE_URL, f"{base}/sboms"), - ) - - @staticmethod - def _project_url_repository(metadata: Message) -> str | None: - return next( - ( - url.strip().rstrip("/") - for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) - if external_reference(label.strip(), url.strip())["type"] == "vcs" - ), - None, - ) - - def _root_component(self, metadata: Message, wheel_name: str, repository: str | None) -> Component: - name, version = metadata["Name"], metadata["Version"] - namespace = canonicalize_name(name) - references: Final[list[ExternalReference]] = [ - external_reference(label.strip(), url.strip()) - for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) - ] - references.extend([ - {"type": "distribution", "url": f"https://pypi.org/project/{name}/{version}/"}, - {"type": "attestation", "url": f"https://pypi.org/integrity/{name}/{version}/{wheel_name}/provenance"}, - ]) - if repository and urlsplit(repository).netloc == "github.com": - references.extend([ - {"type": "security-contact", "url": f"{repository}/security/policy"}, - {"type": "advisories", "url": f"{repository}/security/advisories"}, - ]) - license_files: Final[list[str]] = metadata.get_all("License-File", []) - if repository and license_files: - references.append({"type": "license", "url": f"{repository}/blob/{self.default_branch}/{license_files[0]}"}) - properties: Final[list[Property]] = [] - if requires_python := metadata.get("Requires-Python"): - properties.append({"name": f"{namespace}:requires-python", "value": requires_python}) - properties.extend({"name": "python:classifier", "value": value} for value in metadata.get_all("Classifier", [])) - properties.extend( - {"name": "python:keyword", "value": keyword.strip()} - for keyword in (metadata.get("Keywords") or "").split(",") - if keyword.strip() - ) - if commit := self._commit(): - if repository: - references.append({ - "type": "vcs", - "url": f"{repository}/tree/{commit}", - "comment": "exact source revision", - }) - properties.append({"name": f"{namespace}:vcs-commit", "value": commit}) - authors = _authors(metadata) - licenses = _licenses(metadata) - copyright_line = self._copyright(license_files) - package = purl(name, version) - return { - "type": "application", - "bom-ref": package, - **({"supplier": self.supplier} if self.supplier else {}), - **({"authors": authors} if authors else {}), - "name": name, - "version": version, - **({"description": summary} if (summary := metadata.get("Summary")) else {}), - **({"licenses": licenses} if licenses else {}), - **({"copyright": copyright_line} if copyright_line else {}), - "purl": package, - "externalReferences": references, - "properties": properties, - } - - def _commit(self) -> str | None: - if not (self.root / ".git").exists() or (git := shutil.which("git")) is None: # building from an sdist - return None - return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] # fixed arguments - [git, "rev-parse", "HEAD"], check=True, capture_output=True, text=True, cwd=self.root - ).stdout.strip() - - def _copyright(self, license_files: list[str]) -> str | None: - return next( - ( - line - for license_file in license_files - if (path := self.root / license_file).is_file() - for line in path.read_text(encoding="utf-8").splitlines() - if line.startswith("Copyright") - ), - None, - ) - - def _bundled_component(self, wheel: Path, namespace: str, repository: str | None) -> Component: - with zipfile.ZipFile(wheel) as archive: - members: Final[list[str]] = sorted(archive.namelist()) - metadata_name: Final[str] = next( - name for name in members if name.count("/") == 1 and name.endswith(".dist-info/METADATA") - ) - metadata = Parser().parsestr(archive.read(metadata_name).decode("utf-8")) - record = archive.read(metadata_name.replace("METADATA", "RECORD")).decode("utf-8") - vendored: Final[list[tuple[str, Message]]] = [ - *_vendor_manifest(wheel, archive, members), - *( - (name, Parser().parsestr(archive.read(name).decode("utf-8"))) - for name in members - if "/_vendor/" in name and name.endswith(".dist-info/METADATA") - ), - ] - component = component_from_metadata(metadata, "library") - # a bundled wheel published from the same forge account as this project shares its supplier - if ( - self.supplier - and repository - and any(reference["url"].startswith(_owner(repository)) for reference in component["externalReferences"]) - ): - component["supplier"] = self.supplier - sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() - component["hashes"] = [{"alg": "SHA-256", "content": sha256}] - component["externalReferences"].append( - {"type": "distribution", "url": f"https://pypi.org/project/{metadata['Name']}/{metadata['Version']}/"}, - ) - component["properties"] = [ - {"name": f"{namespace}:bundled-wheel", "value": wheel.relative_to(self.root).as_posix()}, - *self.bundled_wheel_properties(wheel), - *component["properties"], - ] - component["evidence"] = { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": metadata_name}], - }, - { - "field": "hash", - "confidence": 1, - "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], - }, - ], - } - component["components"] = [ - file_component(component["bom-ref"], path, digest, size) - for path, digest, size in (row for row in csv.reader(StringIO(record, newline="")) if row) - if digest - ] - children: Final[dict[str, Component]] = {} - for source, child_metadata in vendored: - child = component_from_metadata(child_metadata, "library") - child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['bom-ref']}" - child["properties"].append({"name": f"{namespace}:vendored-manifest", "value": source}) - child["evidence"] = { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], - } - ], - } - # a vendored .dist-info comes after the pip manifest line naming the same purl, so its metadata wins - children[purl(child["name"], child["version"])] = child - component["components"].extend(children[key] for key in sorted(children)) - return component - - def _bundled_wheel_paths(self) -> list[Path]: - found = {path for pattern in self.bundled_wheels for path in self.root.glob(pattern)} - return sorted(found, key=lambda path: path.relative_to(self.root).as_posix()) - - def _build_script_tool(self, script: PurePosixPath, version: str, repository: str | None) -> Component: - # a custom build hook shapes the document, so it is part of the toolchain a rebuild has to match - tool: Final[Component] = { - "type": "application", - "bom-ref": f"tool:{script}", - "name": script.name, - "version": version, - # str, so a Windows drive in an absolute path survives the join with the native root - "hashes": [ - {"alg": "SHA-256", "content": hashlib.sha256((self.root / str(script)).read_bytes()).hexdigest()} - ], - } - if repository: - tool["externalReferences"] = [{"type": "vcs", "url": f"{repository}/blob/{self.default_branch}/{script}"}] - return tool - - def _scope(self) -> str: - if directories := sorted({str(PurePosixPath(pattern).parent) for pattern in self.bundled_wheels}): - return ( - f"Components are the wheels bundled as data files under {' and '.join(directories)} and the direct" - " runtime dependencies declared in the wheel's core metadata. The latter carry no version because" - " they are resolved at install time, not at build time." - ) - return ( - "Components are the direct runtime dependencies declared in the wheel's core metadata. They carry no" - " version because they are resolved at install time, not at build time." - ) - - -def sbom_file_name(metadata: Message) -> str: - return f"{canonicalize_name(metadata['Name'])}.cdx.json" - - -def _runtime_requirements(metadata: Message) -> list[str]: - # an extra's requirements apply only when the extra is requested, so they are not runtime dependencies of the wheel - return [ - requirement - for requirement in metadata.get_all("Requires-Dist", []) - if (marker := Requirement(requirement).marker) is None or "extra" not in str(marker) - ] - - -def _declared_dependency(requirement: str, namespace: str) -> Component: - parsed = Requirement(requirement) - component: Final[Component] = { - "type": "library", - # the same distribution can be declared more than once with different markers, so the purl is not unique - "bom-ref": f"requires-dist:{requirement}", - "name": parsed.name, - "purl": purl(parsed.name), - "externalReferences": [{"type": "distribution", "url": f"https://pypi.org/project/{parsed.name}/"}], - "properties": [{"name": f"{namespace}:requires-dist", "value": requirement}], - } - if parsed.marker is not None: - component["properties"].append({"name": f"{namespace}:environment-marker", "value": str(parsed.marker)}) - return component - - -def _authors(metadata: Message) -> list[Contact]: - # core metadata joins several names into one header - if people := contacts( - [name for entry in metadata.get_all("Maintainer", []) for name in entry.split(", ")], - metadata.get_all("Maintainer-email", []), - ): - return people - return contacts( - [name for entry in metadata.get_all("Author", []) for name in entry.split(", ")], - metadata.get_all("Author-email", []), - ) - - -def _licenses(metadata: Message) -> list[License]: - if expression := metadata.get("License-Expression"): - return [{"expression": expression, "acknowledgement": "declared"}] - return [] - - -def _vendor_manifest(wheel: Path, archive: zipfile.ZipFile, members: list[str]) -> list[tuple[str, Message]]: - # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info - found: Final[list[tuple[str, Message]]] = [] - for name in members: - if not name.endswith("/_vendor/vendor.txt"): - continue - for line in archive.read(name).decode("utf-8").splitlines(): - if not (requirement := line.partition("#")[0].strip()): - continue - parsed = Requirement(requirement) - pins = list(parsed.specifier) - if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: - msg = f"Unpinned vendored dependency in {wheel.name}:{name}: {requirement}" - raise ValueError(msg) - found.append((name, Parser().parsestr(f"Name: {parsed.name}\nVersion: {pins[0].version}\n"))) - return found - - -def _owner(repository: str) -> str: - parts = urlsplit(repository) - return f"{parts.scheme}://{parts.netloc}/{parts.path.strip('/').split('/')[0]}/" - - -__all__ = [ - "BundledWheelProperties", - "Project", - "Toml", - "sbom_file_name", -] diff --git a/src/python_sbom/_validate/__init__.py b/src/python_sbom/_validate/__init__.py new file mode 100644 index 0000000..1496c0a --- /dev/null +++ b/src/python_sbom/_validate/__init__.py @@ -0,0 +1 @@ +"""Checks of a document against the schema, its own references, its SPDX rendering and a license policy.""" diff --git a/src/python_sbom/_validate.py b/src/python_sbom/_validate/_checks.py similarity index 84% rename from src/python_sbom/_validate.py rename to src/python_sbom/_validate/_checks.py index 51b1d6c..45e60fc 100644 --- a/src/python_sbom/_validate.py +++ b/src/python_sbom/_validate/_checks.py @@ -3,16 +3,18 @@ import json import re import zipfile -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, TypeAlias -from ._spdx import to_spdx +from python_sbom._formats._spdx import to_spdx if TYPE_CHECKING: from collections.abc import Iterator from pathlib import Path - from ._cyclonedx import Component, Document, License + from python_sbom._model._types import Component, Document, License +#: a JSON value, as :func:`json.loads` hands it over +Json: TypeAlias = "str | int | float | bool | list[Json] | dict[str, Json] | None" # SPDX matches operators only in upper case _OPERATORS: Final[frozenset[str]] = frozenset({"AND", "OR", "WITH", "(", ")"}) _TOKEN: Final[re.Pattern[str]] = re.compile(r"[()]|[^\s()]+") @@ -51,10 +53,22 @@ def validate(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: :returns: one message per problem, empty when the document is valid """ - from ._extra import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # optional extra + from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # optional extra if error := schema_error(raw): return [f"does not conform to the CycloneDX 1.6 schema: {error}"] + loaded: Json = json.loads(raw) + # the schema leaves these optional, yet the checks below and the SPDX rendering read them + required: tuple[tuple[str, ...], ...] = ( + ("metadata", "timestamp"), + ("metadata", "tools", "components"), + ("metadata", "component", "bom-ref"), + ("metadata", "component", "name"), + ("components",), + ("dependencies",), + ) + if missing := [".".join(path) for path in required if not _has(loaded, path)]: + return [f"lacks {', '.join(missing)}, which every document python-sbom writes carries"] document: Final[Document] = json.loads(raw) # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize the document # as CycloneDX at all, and silently rejects anything missing it as an unknown format @@ -76,13 +90,21 @@ def validate(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: return problems +def _has(value: Json, path: tuple[str, ...]) -> bool: + for key in path: + if not isinstance(value, dict) or key not in value: + return False + value = value[key] + return True + + def _reference_problems(document: Document) -> list[str]: root_ref = document["metadata"]["component"]["bom-ref"] tools = document["metadata"]["tools"]["components"] every_ref = list(_bom_refs([document["metadata"]["component"], *document["components"], *tools])) if duplicates := {ref for ref in every_ref if every_ref.count(ref) > 1}: return [f"bom-ref values are not unique: {sorted(duplicates)}"] - dependencies = {entry["ref"]: set(entry["dependsOn"]) for entry in document["dependencies"]} + dependencies = {entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"]} problems = [] if unknown := (set(dependencies) | set().union(*dependencies.values())) - set(every_ref): problems.append(f"dependencies reference bom-refs that do not exist: {sorted(unknown)}") @@ -93,7 +115,7 @@ def _reference_problems(document: Document) -> list[str]: tool_refs = {tool["bom-ref"] for tool in tools} problems.extend( f"workflow {workflow['uid']} references unknown tools: {sorted(unknown)}" - for workflow in (workflow for formula in document["formulation"] for workflow in formula["workflows"]) + for workflow in (workflow for formula in document.get("formulation", []) for workflow in formula["workflows"]) if (unknown := {reference["ref"] for reference in workflow["resourceReferences"]} - tool_refs) ) return problems diff --git a/src/python_sbom/_extra.py b/src/python_sbom/_validate/_schemas.py similarity index 96% rename from src/python_sbom/_extra.py rename to src/python_sbom/_validate/_schemas.py index 15536ad..b6eebdd 100644 --- a/src/python_sbom/_extra.py +++ b/src/python_sbom/_validate/_schemas.py @@ -10,7 +10,7 @@ from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document if TYPE_CHECKING: - from ._spdx import Spdx + from python_sbom._formats._spdx import Spdx def schema_error(raw: str) -> str | None: diff --git a/src/python_sbom/_wheel/__init__.py b/src/python_sbom/_wheel/__init__.py new file mode 100644 index 0000000..4c83645 --- /dev/null +++ b/src/python_sbom/_wheel/__init__.py @@ -0,0 +1 @@ +"""Writing an SBOM into a wheel some build backend already built.""" diff --git a/src/python_sbom/_wheel/_embed.py b/src/python_sbom/_wheel/_embed.py new file mode 100644 index 0000000..22c61ff --- /dev/null +++ b/src/python_sbom/_wheel/_embed.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import os +import tempfile +import time +import zipfile +from email.parser import Parser +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from python_sbom._backend import Backend +from python_sbom._errors import WheelError +from python_sbom._model._components import record_digest + +if TYPE_CHECKING: + from email.message import Message + + from python_sbom._collect._facts import BuildFacts + from python_sbom._collect._settings import Settings + + +class BuiltWheel(Backend): + """ + A wheel some other build backend already built, such as meson-python, and the project it came from. + + :param wheel: the wheel, rewritten in place by :meth:`embed` + :param root: the project directory the wheel was built from + :param settings: the project's options, usually from ``[tool.python-sbom]`` + + """ + + def __init__(self, wheel: Path, root: Path, settings: Settings) -> None: + self._wheel = wheel + self._root = root + self._settings = settings + with zipfile.ZipFile(wheel) as archive: + self._entries: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [ + (info, archive.read(info)) for info in archive.infolist() + ] + if ( + dist_info := next( + ( + info.filename.split("/")[0] + for info, _ in self._entries + if info.filename.count("/") == 1 and info.filename.endswith(".dist-info/METADATA") + ), + None, + ) + ) is None: + msg = f"{wheel} has no .dist-info/METADATA, so it is not a wheel" + raise WheelError(msg) + self._dist_info: Final[str] = dist_info + + @property + def root(self) -> Path: + """The project directory the wheel was built from.""" + return self._root + + @property + def settings(self) -> Settings: + """The project's options.""" + return self._settings + + def metadata(self) -> Message: + """ + Read the core metadata from the wheel. + + :returns: the parsed ``METADATA`` + + """ + return Parser().parsestr(self._read(f"{self._dist_info}/METADATA").decode("utf-8")) + + def _read(self, name: str) -> bytes: + return next(data for info, data in self._entries if info.filename == name) + + def wheel_name(self) -> str: + """ + Return the wheel's file name. + + :returns: the file name + + """ + return self._wheel.name + + def embed(self, facts: BuildFacts) -> str: + """ + Write the SBOM into the wheel under ``.dist-info/sboms/``, with a ``RECORD`` row. + + An SBOM of the same name is replaced. Every other entry keeps its name, order, timestamp and permissions, and + the new one takes its timestamp from ``SOURCE_DATE_EPOCH``, so embedding into the same wheel twice gives the + same bytes. + + :param facts: what the environment says about the build + :returns: the SBOM's path inside the wheel + + """ + name, content = self.sbom(facts) + sbom: Final[str] = f"{self._dist_info}/sboms/{name}" + record: Final[str] = f"{self._dist_info}/RECORD" + lines: Final[list[str]] = [ + line + for line in self._read(record).decode("utf-8").splitlines(keepends=True) + if not line.startswith(f"{sbom},") + ] + newline = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" + # the RECORD row of RECORD itself stays last + at = next((index for index, line in enumerate(lines) if line.startswith(f"{record},")), len(lines)) + lines.insert(at, f"{sbom},{record_digest(content)},{len(content)}{newline}") + entry = zipfile.ZipInfo(sbom, time.gmtime(max(facts.epoch, 315532800))[:6]) # zip timestamps start at 1980 + entry.compress_type = zipfile.ZIP_DEFLATED + entry.external_attr = 0o644 << 16 + # written next to the wheel and moved over it, so a failure leaves the wheel as it was + descriptor, temporary = tempfile.mkstemp(dir=self._wheel.parent, suffix=".whl") + os.close(descriptor) + with zipfile.ZipFile(temporary, "w") as archive: + for info, data in self._entries: + if info.filename == record: + archive.writestr(entry, content) + archive.writestr(info, "".join(lines).encode()) + elif info.filename != sbom: + archive.writestr(info, data) + Path(temporary).replace(self._wheel) + return sbom + + +__all__ = [ + "BuiltWheel", +] diff --git a/src/python_sbom/hatch/__init__.py b/src/python_sbom/hatch/__init__.py new file mode 100644 index 0000000..8c62fa0 --- /dev/null +++ b/src/python_sbom/hatch/__init__.py @@ -0,0 +1,9 @@ +"""The hatchling build hook: the backend that writes the SBOM while hatchling builds the wheel.""" + +from __future__ import annotations + +from ._hook import SbomBuildHook + +__all__ = [ + "SbomBuildHook", +] diff --git a/src/python_sbom/_hook.py b/src/python_sbom/hatch/_hook.py similarity index 61% rename from src/python_sbom/_hook.py rename to src/python_sbom/hatch/_hook.py index 86b3812..6f41961 100644 --- a/src/python_sbom/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import shutil import tempfile from email.parser import Parser @@ -11,15 +10,20 @@ from hatchling.builders.wheel import WheelBuilder from hatchling.utils.constants import DEFAULT_BUILD_SCRIPT -from ._project import Project, sbom_file_name +from python_sbom._backend import Backend +from python_sbom._collect._facts import BuildFacts, git_revision +from python_sbom._collect._settings import Settings +from python_sbom._extension import ContentProvider if TYPE_CHECKING: - from ._cyclonedx import Property + from email.message import Message + from python_sbom._model._types import Property -class SbomBuildHook(BuildHookInterface): + +class SbomBuildHook(BuildHookInterface, ContentProvider): """ - Hatchling build hook that writes a PEP 770 CycloneDX SBOM into the wheel's ``.dist-info/sboms/``. + Hatchling build hook that writes a CycloneDX SBOM into the wheel's ``.dist-info/sboms/``, as PEP 770 describes. Enable it with a ``[tool.hatch.build.hooks.python-sbom]`` table, or subclass it in a hatchling custom build hook to add project-specific detail through :meth:`bundled_wheel_properties`. @@ -43,28 +47,21 @@ def initialize( """ if not isinstance(builder := self.build_config.builder, WheelBuilder): return - metadata = Parser().parsestr( - builder.config.core_metadata_constructor(self.metadata, extra_dependencies=build_data["dependencies"]) - ) tag = build_data.get("tag") or ( builder.get_best_matching_tag() if build_data["infer_tag"] else builder.get_default_tag() ) - project = Project.from_table( - self.config, - root=Path(self.root), - where=f"build hook `{self.PLUGIN_NAME}`", - # a custom build hook script shapes the document, so the SBOM lists it among the build tools - build_script=PurePosixPath(self.config.get("path", DEFAULT_BUILD_SCRIPT)) - if self.PLUGIN_NAME == "custom" - else None, - bundled_wheel_properties=self.bundled_wheel_properties, + backend = _HatchBackend( + hook=self, + metadata=builder.config.core_metadata_constructor( + self.metadata, extra_dependencies=build_data["dependencies"] + ), + wheel_name=f"{builder.artifact_project_id}-{tag}.whl", ) - content = json.dumps(project.describe(metadata, f"{builder.artifact_project_id}-{tag}.whl"), indent=2) + name, content = backend.sbom(BuildFacts.collect(backend.root, git_revision)) # written outside self.directory, which is also the wheel's output directory, so no stray copy lands next to # the built artifacts self._sbom_directory = Path(tempfile.mkdtemp(prefix="python-sbom-")) - # bytes, so the line endings do not depend on the build machine's OS - (sbom := self._sbom_directory / sbom_file_name(metadata)).write_bytes(f"{content}\n".encode()) + (sbom := self._sbom_directory / name).write_bytes(content) build_data["sbom_files"].append(str(sbom)) def bundled_wheel_properties( # ruff: ignore[no-self-use] # subclasses override it @@ -102,6 +99,36 @@ def finalize( self._sbom_directory = None +class _HatchBackend(Backend): + def __init__(self, *, hook: SbomBuildHook, metadata: str, wheel_name: str) -> None: + self._hook = hook + self._metadata = metadata + self._wheel_name = wheel_name + + @property + def root(self) -> Path: + return Path(self._hook.root) + + @property + def settings(self) -> Settings: + return Settings.from_table(self._hook.config, f"build hook `{self._hook.PLUGIN_NAME}`") + + def metadata(self) -> Message: + return Parser().parsestr(self._metadata) + + def wheel_name(self) -> str: + return self._wheel_name + + def build_script(self) -> PurePosixPath | None: + # a custom build hook script shapes the document, so the SBOM lists it among the build tools + if self._hook.PLUGIN_NAME != "custom": + return None + return PurePosixPath(self._hook.config.get("path", DEFAULT_BUILD_SCRIPT)) + + def bundled_wheel_properties(self, wheel: Path) -> list[Property]: + return self._hook.bundled_wheel_properties(wheel) + + __all__ = [ "SbomBuildHook", ] diff --git a/src/python_sbom/hooks.py b/src/python_sbom/hatch/hooks.py similarity index 100% rename from src/python_sbom/hooks.py rename to src/python_sbom/hatch/hooks.py diff --git a/tests/conftest.py b/tests/conftest.py index 951c55b..9cc4e42 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ import pytest from hatchling.builders.wheel import WheelBuilder -from python_sbom import Project +from python_sbom import BuildFacts, BuiltWheel, Settings, git_revision if TYPE_CHECKING: from python_sbom import Document @@ -207,6 +207,16 @@ def install(site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> No return install +_PLAIN: Final[Settings] = Settings(supplier=None, repository=None, default_branch="main", bundled_wheels=()) + + +class EmbedInto(Protocol): + def __call__(self, wheel: Path, settings: Settings = _PLAIN) -> str: ... + + @pytest.fixture -def plain_project(tmp_path: Path) -> Project: - return Project.from_table({}, root=tmp_path, where="[tool.python-sbom]") +def embed_into(tmp_path: Path) -> EmbedInto: + def embed(wheel: Path, settings: Settings = _PLAIN) -> str: + return BuiltWheel(wheel, tmp_path, settings).embed(BuildFacts.collect(tmp_path, git_revision)) + + return embed diff --git a/tests/seeder/hatch_build.py b/tests/seeder/hatch_build.py index 38fcda9..3234b2e 100644 --- a/tests/seeder/hatch_build.py +++ b/tests/seeder/hatch_build.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import TYPE_CHECKING -from python_sbom import SbomBuildHook +from python_sbom.hatch import SbomBuildHook if TYPE_CHECKING: from python_sbom import Property diff --git a/tests/test_cli.py b/tests/test_cli.py index b17a2b7..f68bfb2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -69,3 +69,15 @@ def test_cli_validate_no_wheel(tmp_path: Path, capsys: pytest.CaptureFixture[str with pytest.raises(SystemExit) as error: main(["validate", str(tmp_path)]) assert (error.value.code, capsys.readouterr().err) == (1, f"no wheel found in {tmp_path}\n") + + +def test_cli_embed_not_a_wheel(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + with zipfile.ZipFile(archive := tmp_path / "broken-1.0-py3-none-any.whl", "w") as zipped: + zipped.writestr("broken/__init__.py", "") + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'broken'\n", encoding="utf-8") + with pytest.raises(SystemExit) as error: + main(["embed", str(archive), "--root", str(tmp_path)]) + assert (error.value.code, capsys.readouterr().err) == ( + 1, + f"python-sbom embed: {archive} has no .dist-info/METADATA, so it is not a wheel\n", + ) diff --git a/tests/test_embed.py b/tests/test_embed.py index 0fed65a..4cbb47b 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -1,29 +1,31 @@ from __future__ import annotations +import importlib import json import sys import zipfile from pathlib import Path from typing import TYPE_CHECKING, Final +import pytest from conftest import digest, write_seed_wheel from installer import install from installer.destinations import SchemeDictionaryDestination from installer.sources import WheelFile -from python_sbom import Project, embed, validate_wheel +from python_sbom import Settings, validate_wheel if TYPE_CHECKING: - from conftest import MakeWheel + from conftest import EmbedInto, MakeWheel from python_sbom import Document _METADATA: Final[str] = "License-Expression: MIT\nProject-URL: Source, https://github.com/example/native\n" -def test_embed_writes_sbom(make_wheel: MakeWheel, plain_project: Project) -> None: +def test_embed_writes_sbom(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: wheel: Final[Path] = make_wheel(_METADATA) - name: Final[str] = embed(wheel, plain_project) + name: Final[str] = embed_into(wheel) with zipfile.ZipFile(wheel) as archive: document: Final[Document] = json.loads(archive.read(name)) assert (name, document["metadata"]["component"]["purl"]) == ( @@ -32,9 +34,9 @@ def test_embed_writes_sbom(make_wheel: MakeWheel, plain_project: Project) -> Non ) -def test_embed_record(make_wheel: MakeWheel, plain_project: Project) -> None: +def test_embed_record(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: wheel: Final[Path] = make_wheel(_METADATA) - name: Final[str] = embed(wheel, plain_project) + name: Final[str] = embed_into(wheel) with zipfile.ZipFile(wheel) as archive: sbom: Final[bytes] = archive.read(name) rows: Final[list[str]] = archive.read("native-1.0.dist-info/RECORD").decode("utf-8").splitlines() @@ -45,21 +47,21 @@ def test_embed_record(make_wheel: MakeWheel, plain_project: Project) -> None: ) -def test_embed_twice_same_bytes(make_wheel: MakeWheel, plain_project: Project) -> None: +def test_embed_twice_same_bytes(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: wheel: Final[Path] = make_wheel(_METADATA) - embed(wheel, plain_project) + embed_into(wheel) first: Final[bytes] = wheel.read_bytes() - embed(wheel, plain_project) + embed_into(wheel) assert wheel.read_bytes() == first -def test_embed_keeps_entries(make_wheel: MakeWheel, plain_project: Project) -> None: +def test_embed_keeps_entries(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: wheel: Final[Path] = make_wheel(_METADATA) with zipfile.ZipFile(wheel) as archive: before: Final[list[tuple[str, tuple[int, ...]]]] = [ (info.filename, info.date_time) for info in archive.infolist() ] - embed(wheel, plain_project) + embed_into(wheel) with zipfile.ZipFile(wheel) as archive: after: Final[list[tuple[str, tuple[int, ...]]]] = [ (info.filename, info.date_time) for info in archive.infolist() @@ -67,7 +69,7 @@ def test_embed_keeps_entries(make_wheel: MakeWheel, plain_project: Project) -> N assert after == [*before[:-1], ("native-1.0.dist-info/sboms/native.cdx.json", (2020, 2, 2, 0, 0, 0)), before[-1]] -def test_embed_crlf_record(make_wheel: MakeWheel, plain_project: Project) -> None: +def test_embed_crlf_record(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: wheel: Final[Path] = make_wheel(_METADATA) with zipfile.ZipFile(wheel) as archive: entries: Final[dict[str, bytes]] = {name: archive.read(name) for name in archive.namelist()} @@ -75,21 +77,21 @@ def test_embed_crlf_record(make_wheel: MakeWheel, plain_project: Project) -> Non with zipfile.ZipFile(wheel, "w") as archive: for name, content in entries.items(): archive.writestr(name, content) - name: Final[str] = embed(wheel, plain_project) + name: Final[str] = embed_into(wheel) with zipfile.ZipFile(wheel) as archive: record: Final[bytes] = archive.read("native-1.0.dist-info/RECORD") assert (record.count(b"\r\n"), record.count(b"\n"), f"{name},".encode() in record) == (5, 5, True) -def test_embed_valid(make_wheel: MakeWheel, plain_project: Project) -> None: +def test_embed_valid(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: wheel: Final[Path] = make_wheel(_METADATA) - embed(wheel, plain_project) + embed_into(wheel) assert validate_wheel(wheel, frozenset({"MIT"})) == [] -def test_embed_survives_install(make_wheel: MakeWheel, plain_project: Project, tmp_path: Path) -> None: +def test_embed_survives_install(make_wheel: MakeWheel, embed_into: EmbedInto, tmp_path: Path) -> None: wheel: Final[Path] = make_wheel(_METADATA) - name: Final[str] = embed(wheel, plain_project) + name: Final[str] = embed_into(wheel) scheme: Final[dict[str, str]] = { key: str(tmp_path / "site" / key) for key in ("purelib", "platlib", "headers", "scripts", "data") } @@ -99,26 +101,42 @@ def test_embed_survives_install(make_wheel: MakeWheel, plain_project: Project, t assert (Path(scheme["platlib"]) / name).is_file() -def test_embed_uses_project_settings(make_wheel: MakeWheel, tmp_path: Path) -> None: +def test_embed_uses_project_settings(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: wheel: Final[Path] = make_wheel(_METADATA) - project: Final[Project] = Project.from_table( - {"supplier": {"name": "Org"}}, root=tmp_path, where="[tool.python-sbom]" - ) - name: Final[str] = embed(wheel, project) + settings: Final[Settings] = Settings.from_table({"supplier": {"name": "Org"}}, "[tool.python-sbom]") + name: Final[str] = embed_into(wheel, settings) with zipfile.ZipFile(wheel) as archive: document: Final[Document] = json.loads(archive.read(name)) assert document["metadata"].get("supplier") == {"name": "Org", "url": []} -def test_embed_bundled_wheel(make_wheel: MakeWheel, tmp_path: Path) -> None: +def test_embed_bundled_wheel(make_wheel: MakeWheel, embed_into: EmbedInto, tmp_path: Path) -> None: wheel: Final[Path] = make_wheel(_METADATA) write_seed_wheel(tmp_path) - project: Final[Project] = Project.from_table( - {"bundled-wheels": ["*.whl"]}, root=tmp_path, where="[tool.python-sbom]" - ) - name: Final[str] = embed(wheel, project) + settings: Final[Settings] = Settings.from_table({"bundled-wheels": ["*.whl"]}, "[tool.python-sbom]") + name: Final[str] = embed_into(wheel, settings) with zipfile.ZipFile(wheel) as archive: document: Final[Document] = json.loads(archive.read(name)) assert document["components"][0].get("properties") == [ {"name": "native:bundled-wheel", "value": "pip-1.0-py3-none-any.whl"} ] + + +@pytest.fixture +def without_hatchling(monkeypatch: pytest.MonkeyPatch) -> None: + # a fresh import of the package with hatchling unimportable, as for a project built with another backend + for name in [name for name in sys.modules if name.split(".")[0] in {"python_sbom", "hatchling"}]: + monkeypatch.delitem(sys.modules, name) + monkeypatch.setitem(sys.modules, "hatchling", None) + + +@pytest.mark.usefixtures("without_hatchling") +def test_embed_without_hatchling(make_wheel: MakeWheel, tmp_path: Path) -> None: + core = importlib.import_module("python_sbom") + wheel: Final[Path] = make_wheel(_METADATA) + facts = core.BuildFacts.collect(tmp_path, core.git_revision) + name: Final[str] = core.BuiltWheel(wheel, tmp_path, core.Settings.from_table({}, "[tool.python-sbom]")).embed(facts) + assert (name, "hatchling" in {module.split(".")[0] for module in sys.modules if sys.modules[module]}) == ( + "native-1.0.dist-info/sboms/native.cdx.json", + False, + ) diff --git a/tests/test_hook.py b/tests/test_hook.py index f61daf2..eba3342 100644 --- a/tests/test_hook.py +++ b/tests/test_hook.py @@ -99,7 +99,7 @@ def test_custom_build_script_recorded( root: Path, write_project: WriteProject, urls: str, references: dict[str, list[dict[str, str]]] ) -> None: (script := root / "build.py").write_text( - "from __future__ import annotations\n\nfrom python_sbom import SbomBuildHook\n\n\n" + "from __future__ import annotations\n\nfrom python_sbom.hatch import SbomBuildHook\n\n\n" "def get_build_hook() -> type[SbomBuildHook]:\n return SbomBuildHook\n", encoding="utf-8", ) diff --git a/tests/test_project.py b/tests/test_project.py index f50f14e..ed2416e 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -21,6 +21,8 @@ from conftest import BUNDLED, HOOK, build_hook_sbom +from python_sbom import SourceRevisionError, WheelError + _SHA256_ABC: Final[str] = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" @@ -153,7 +155,7 @@ def test_bundled_wheel(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None sha256: Final[str] = hashlib.sha256(bundle_wheel({}, "License-Expression: MIT\n").read_bytes()).hexdigest() assert build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0] == { "type": "library", - "bom-ref": "pkg:pypi/pip@1.0", + "bom-ref": "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl", "name": "pip", "version": "1.0", "purl": "pkg:pypi/pip@1.0", @@ -180,7 +182,7 @@ def test_bundled_wheel(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None "components": [ { "type": "file", - "bom-ref": "pkg:pypi/pip@1.0#pip/example.py", + "bom-ref": "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl#pip/example.py", "name": "pip/example.py", "hashes": [{"alg": "SHA-256", "content": _SHA256_ABC}], "properties": [{"name": "size", "value": "3"}], @@ -206,8 +208,8 @@ def test_bundled_wheels_sorted_once(build_sbom: BuildSbom, bundle_wheel: BundleW bundle_wheel({}, name="pip") hook = f'{HOOK}bundled-wheels = ["src/example/wheels/*.whl", "src/example/wheels/pip-*"]\n' assert build_sbom(hook=hook)["dependencies"][0]["dependsOn"] == [ - "pkg:pypi/pip@1.0", - "pkg:pypi/setuptools@1.0", + "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl", + "pkg:pypi/setuptools@1.0?file_name=setuptools-1.0-py3-none-any.whl", "requires-dist:packaging>=24", "requires-dist:tomli>=2; python_version < '3.11'", ] @@ -249,7 +251,7 @@ def test_bundled_wheel_record_paths(root: Path, build_sbom: BuildSbom, filename: assert build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0].get("components") == [ { "type": "file", - "bom-ref": f"pkg:pypi/pip@1.0#{filename}", + "bom-ref": f"pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl#{filename}", "name": filename, "hashes": [{"alg": "SHA-256", "content": _SHA256_ABC}], "properties": [{"name": "size", "value": "3"}], @@ -272,7 +274,7 @@ def test_vendored_identity(build_sbom: BuildSbom, bundle_wheel: BundleWheel, sou assert [child for child in children if child["type"] == "library"] == [ { "type": "library", - "bom-ref": "pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4", + "bom-ref": "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl#vendored/pkg:pypi/urllib3@1.26.4", "name": "urllib3", "version": "1.26.4", "purl": "pkg:pypi/urllib3@1.26.4", @@ -294,8 +296,8 @@ def test_vendored_identity(build_sbom: BuildSbom, bundle_wheel: BundleWheel, sou def test_vendored_relationship(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: bundle_wheel({"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}) assert build_sbom(hook=f"{HOOK}{BUNDLED}")["dependencies"][1] == { - "ref": "pkg:pypi/pip@1.0", - "dependsOn": ["pkg:pypi/pip@1.0#vendored/pkg:pypi/urllib3@1.26.4"], + "ref": "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl", + "dependsOn": ["pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl#vendored/pkg:pypi/urllib3@1.26.4"], } @@ -494,7 +496,9 @@ def test_source_revision_unreadable(root: Path, build_sbom: BuildSbom, mocker: M (root / ".git").mkdir() error = subprocess.CalledProcessError(128, ["git"], stderr="fatal: detected dubious ownership") mocker.patch("subprocess.run", autospec=True, side_effect=error) - with pytest.raises(subprocess.CalledProcessError): + with pytest.raises( + SourceRevisionError, match=r"git rev-parse HEAD failed in .*: fatal: detected dubious ownership" + ): build_sbom() @@ -548,3 +552,42 @@ def test_serial_number_per_project(build_sbom: BuildSbom) -> None: def test_invalid_option(build_sbom: BuildSbom, hook: str, message: str) -> None: with pytest.raises(TypeError, match=message): build_sbom(hook=hook) + + +def test_bundled_wheel_variants_unique_refs(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: + wheel = bundle_wheel({}) + shutil.copyfile(wheel, wheel.with_name("pip-1.0-py3-none-manylinux_2_17_x86_64.whl")) + assert [component["bom-ref"] for component in build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][:2]] == [ + "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl", + "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-manylinux_2_17_x86_64.whl", + ] + + +@pytest.mark.parametrize( + ("requirement", "expected"), + [ + pytest.param("foo; extra != 'fast'", ["foo"], id="not-an-extra"), + pytest.param("foo; extra == 'fast' or python_version < '3.12'", ["foo"], id="extra-or-platform"), + pytest.param("foo; python_version < '3.12' and extra == 'fast'", [], id="platform-and-extra"), + pytest.param("foo; (extra == 'a' or extra == 'b') and os_name == 'nt'", [], id="grouped-extras"), + pytest.param("foo; os_name == 'nt'", ["foo"], id="platform-only"), + pytest.param("foo; 'fast' == extra", [], id="extra-on-the-right"), + ], +) +def test_runtime_requirements_with_extra_markers(build_sbom: BuildSbom, requirement: str, expected: list[str]) -> None: + project = f'[project]\nname = "example"\nversion = "1.0"\ndependencies = ["{requirement}"]\n' + assert [component["name"] for component in build_sbom(project=project)["components"]] == expected + + +def test_bundled_wheel_without_metadata(root: Path, build_sbom: BuildSbom) -> None: + (wheels := root / "src" / "example" / "wheels").mkdir(parents=True) + with zipfile.ZipFile(wheels / "pip-1.0-py3-none-any.whl", "w") as archive: + archive.writestr("pip/__init__.py", "") + with pytest.raises(WheelError, match=r"pip-1\.0-py3-none-any\.whl has no \.dist-info/METADATA"): + build_sbom(hook=f"{HOOK}{BUNDLED}") + + +def test_vendored_requirement_unparsable(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: + bundle_wheel({"pip/_vendor/vendor.txt": "not a requirement!\n"}) + with pytest.raises(WheelError, match="does not parse: not a requirement!"): + build_sbom(hook=f"{HOOK}{BUNDLED}") diff --git a/tests/test_spdx.py b/tests/test_spdx.py index bface45..526e0d8 100644 --- a/tests/test_spdx.py +++ b/tests/test_spdx.py @@ -28,7 +28,7 @@ def bundled(bundle_wheel: BundleWheel) -> None: def test_spdx_bundled_wheel(build_sbom: BuildSbom) -> None: cyclonedx: Final[Document] = build_sbom(hook=_SUPPLIED) assert to_spdx(cyclonedx)["packages"][1] == { - "SPDXID": "SPDXRef-pkg-pypi-pip-1.0", + "SPDXID": "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-ccff93ba", "name": "pip", "downloadLocation": "NOASSERTION", "filesAnalyzed": False, @@ -49,7 +49,7 @@ def test_spdx_bundled_wheel(build_sbom: BuildSbom) -> None: def test_spdx_root_package(build_sbom: BuildSbom) -> None: assert to_spdx(build_sbom())["packages"][0] == { - "SPDXID": "SPDXRef-pkg-pypi-example-1.0", + "SPDXID": "SPDXRef-pkg-pypi-example-1.0-42410e0c", "name": "Example", "downloadLocation": "NOASSERTION", "filesAnalyzed": False, @@ -72,6 +72,9 @@ def test_spdx_root_package(build_sbom: BuildSbom) -> None: @pytest.mark.usefixtures("bundled") def test_spdx_relationships(build_sbom: BuildSbom) -> None: + urllib3: Final[str] = ( + "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-vendored-pkg-pypi-urllib3-1.26.4-82ad4d8f" + ) assert [ relationship for relationship in to_spdx(build_sbom(hook=_SUPPLIED))["relationships"] @@ -80,32 +83,32 @@ def test_spdx_relationships(build_sbom: BuildSbom) -> None: { "spdxElementId": "SPDXRef-DOCUMENT", "relationshipType": "DESCRIBES", - "relatedSpdxElement": "SPDXRef-pkg-pypi-example-1.0", + "relatedSpdxElement": "SPDXRef-pkg-pypi-example-1.0-42410e0c", }, { - "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0", + "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-ccff93ba", "relationshipType": "CONTAINS", - "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0-vendored-pkg-pypi-urllib3-1.26.4", + "relatedSpdxElement": urllib3, }, { - "spdxElementId": "SPDXRef-pkg-pypi-example-1.0", + "spdxElementId": "SPDXRef-pkg-pypi-example-1.0-42410e0c", "relationshipType": "DEPENDS_ON", - "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0", + "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-ccff93ba", }, { - "spdxElementId": "SPDXRef-pkg-pypi-example-1.0", + "spdxElementId": "SPDXRef-pkg-pypi-example-1.0-42410e0c", "relationshipType": "DEPENDS_ON", - "relatedSpdxElement": "SPDXRef-requires-dist-packaging-24", + "relatedSpdxElement": "SPDXRef-requires-dist-packaging-24-45a0ad48", }, { - "spdxElementId": "SPDXRef-pkg-pypi-example-1.0", + "spdxElementId": "SPDXRef-pkg-pypi-example-1.0-42410e0c", "relationshipType": "DEPENDS_ON", - "relatedSpdxElement": "SPDXRef-requires-dist-tomli-2-python-version-3.11", + "relatedSpdxElement": "SPDXRef-requires-dist-tomli-2-python-version-3.11-7c43c831", }, { - "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0", + "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-ccff93ba", "relationshipType": "DEPENDS_ON", - "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0-vendored-pkg-pypi-urllib3-1.26.4", + "relatedSpdxElement": urllib3, }, ] @@ -115,7 +118,7 @@ def test_spdx_build_tools(build_sbom: BuildSbom) -> None: relationship["relatedSpdxElement"] for relationship in to_spdx(build_sbom())["relationships"] if relationship["relationshipType"] == "BUILD_TOOL_OF" - } == {"SPDXRef-pkg-pypi-example-1.0"} + } == {"SPDXRef-pkg-pypi-example-1.0-42410e0c"} @pytest.mark.parametrize( @@ -163,7 +166,7 @@ def test_spdx_creation_info( ) -> None: monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") (root / "build.py").write_text( - "from python_sbom import SbomBuildHook\n\n\ndef get_build_hook():\n return SbomBuildHook\n", + "from python_sbom.hatch import SbomBuildHook\n\n\ndef get_build_hook():\n return SbomBuildHook\n", encoding="utf-8", ) cyclonedx: Final[Document] = build_sbom(hook=hook) @@ -184,3 +187,12 @@ def test_spdx_namespace_without_repository(build_sbom: BuildSbom) -> None: assert to_spdx(cyclonedx)["documentNamespace"] == ( f"https://spdx.org/spdxdocs/sboms/Example-1.0-{cyclonedx['serialNumber'].removeprefix('urn:uuid:')}" ) + + +def test_spdx_conditional_requirements_unique_ids(build_sbom: BuildSbom) -> None: + project = """[project]\nname = "example"\nversion = "1.0" +dependencies = ["foo; python_version < '3.12'", "foo; python_version >= '3.12'"]\n""" + ids = [ + package["SPDXID"] for package in to_spdx(build_sbom(project=project))["packages"] if package["name"] == "foo" + ] + assert (len(ids), len(set(ids))) == (2, 2) diff --git a/tests/test_validate.py b/tests/test_validate.py index 0948826..000cc4e 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -6,6 +6,7 @@ import pytest +from docs.examples.zipapp import describe_zipapp from python_sbom import validate, validate_wheel if TYPE_CHECKING: @@ -15,7 +16,7 @@ from pytest_mock import MockerFixture from python_sbom import Component, Document - from python_sbom._cyclonedx import License + from python_sbom._model._types import License _ALLOWED: Final[frozenset[str]] = frozenset({"Apache-2.0", "BSD-3-Clause", "MIT"}) @@ -39,7 +40,7 @@ def _denied(expression: str, bom_ref: str = "pkg:pypi/bundled@1.0") -> str: @pytest.fixture def sbom_document(build_sbom: BuildSbom, mocker: MockerFixture) -> Document: # schema validation time grows with the document, and the test environment's packages are irrelevant here - mocker.patch("python_sbom._cyclonedx.distributions", autospec=True, return_value=[]) + mocker.patch("python_sbom._collect._facts.distributions", autospec=True, return_value=[]) return build_sbom() @@ -234,3 +235,24 @@ def test_license_policy_judges_files_owner(sbom_document: Document) -> None: assert validate(_with_component(sbom_document, component), _ALLOWED) == [ _denied("LGPL-3.0-only", "pkg:pypi/distlib@0.4") ] + + +def test_document_missing_structure() -> None: + serial = f"urn:uuid:{'0' * 8}-0000-0000-0000-{'0' * 12}" + raw = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "serialNumber": serial}) + assert validate(raw, None) == [ + ( + "lacks metadata.timestamp, metadata.tools.components, metadata.component.bom-ref, metadata.component.name," + " components, dependencies, which every document python-sbom writes carries" + ) + ] + + +def test_documented_zipapp_example(tmp_path: Path) -> None: + assert validate(json.dumps(describe_zipapp(tmp_path, "21.0")), None) == [] + + +def test_versionless_root(tmp_path: Path) -> None: + document = describe_zipapp(tmp_path, "21.0") + del document["metadata"]["component"]["version"] + assert validate(json.dumps(document), None) == [] From e9efe48e7bdebefd6899bfbac5cb53aefad1e30a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 21:12:07 -0800 Subject: [PATCH 013/115] =?UTF-8?q?=F0=9F=94=92=20build:=20relock=20for=20?= =?UTF-8?q?the=20hatch=20extra=20and=20docs=20deps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pylock.toml | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/pylock.toml b/pylock.toml index a7f3b90..df5ccfc 100644 --- a/pylock.toml +++ b/pylock.toml @@ -4,6 +4,7 @@ environments = [ ] requires-python = ">=3.11" extras = [ + "hatch", "validate", ] dependency-groups = [ @@ -848,7 +849,7 @@ sha256 = "b49677dbdc597460cc22d01b27ab3696f5e16a21ecf2700fb01bc28e1f2a04a7" [[packages]] name = "hatchling" version = "1.33.0" -marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups" +marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" requires-python = ">=3.10" dependencies = [ { name = "packaging" }, @@ -1638,7 +1639,7 @@ sha256 = "31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9" [[packages]] name = "packaging" version = "26.3" -marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"lint\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"project-runtime\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"hatch\" in extras or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"lint\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"project-runtime\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1663,7 +1664,7 @@ sha256 = "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c" [[packages]] name = "pathspec" version = "1.1.1" -marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups" +marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1713,7 +1714,7 @@ sha256 = "1dadfc59a8a5d7565b3d9270ae8fa71c065595352c4277b41c11ee6c222c4b91" [[packages]] name = "pluggy" version = "1.7.0" -marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -2156,7 +2157,7 @@ sha256 = "fb65edfff3541b7ceb3a39be334a0f69f4cb5148287a053e1874748742fda93b" [[packages]] name = "pyyaml" version = "6.0.3" -marker = "\"dev\" in dependency_groups or \"lint\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"lint\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -3004,6 +3005,36 @@ size = 78095 [packages.wheels.hashes] sha256 = "15ca92fe0eec82f0baf91a7efcc51ffacdb605924be6327683a52ebdacb6744c" +[[packages]] +name = "sphinxcontrib-mermaid" +version = "2.1.1" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "jinja2" }, + { name = "pyyaml" }, + { name = "sphinx" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "sphinxcontrib_mermaid-2.1.1.tar.gz" +upload-time = 2026-09-01 16:10:23.671939+00:00 +url = "https://files.pythonhosted.org/packages/8c/87/06537dff20aa57df2d0f2afa87edff4afa85dec6f57f03f7acc24f83ff88/sphinxcontrib_mermaid-2.1.1.tar.gz" +size = 22353 + +[packages.sdist.hashes] +sha256 = "f8d7f1b2f0ad69b7ebdd5cbf532fb4ee107893b5373bc2d0e9634089c0b2cfee" + +[[packages.wheels]] +name = "sphinxcontrib_mermaid-2.1.1-py3-none-any.whl" +upload-time = 2026-09-01 16:10:22.377054+00:00 +url = "https://files.pythonhosted.org/packages/50/6a/5bf2222b8084c7a7bc3ec23d470c91e3e03e82e4ce0bbacaa61f456ce7c1/sphinxcontrib_mermaid-2.1.1-py3-none-any.whl" +size = 16368 + +[packages.wheels.hashes] +sha256 = "3b9ac1a8e0b5154dec3c0acf9b5c54dd9ba58540e416f20858c300814c25648f" + [[packages]] name = "sphinxcontrib-qthelp" version = "2.0.0" @@ -3057,7 +3088,7 @@ sha256 = "6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331" [[packages]] name = "tomlkit" version = "0.15.2" -marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups" +marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -3111,7 +3142,7 @@ sha256 = "ae4d223e6aadaff98d29b7f09e58d9c9ccf4cd3b455a2d2e0486d432cd8bf660" [[packages]] name = "trove-classifiers" version = "2026.9.21.13" -marker = "\"build\" in dependency_groups or \"project-runtime\" in dependency_groups" +marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" index = "https://pypi.org/simple/" [packages.sdist] @@ -3542,7 +3573,7 @@ sha256 = "a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a" [tool.nab] nab-version = "0.0.23" -created-at = 2026-11-07 03:47:52.490850+00:00 +created-at = 2026-11-07 05:12:05.951651+00:00 command-line = [ "nab", "lock", From 9f55e4ae2f81c993fd458e2ba9dad696c0505159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 22:08:31 -0800 Subject: [PATCH 014/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20rebuil?= =?UTF-8?q?d=20around=20one=20SBOM=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split settings and build facts, the document model, wheel archives, formats and validation into subpackages that import in one direction, with build_sbom as the only pipeline. A Backend now just returns the request, project content comes from a ContentProvider passed in, and the hatch hook writes the SBOM in finalize from the built wheel, so a later hook can no longer make it stale. Address the first review's correctness and security findings: bundled wheels are checked against their RECORD, embedded entries are the same on every OS and keep the wheel's mode, repeated requirements collapse, other tools' SBOMs get the schema check alone, and bad input raises an error the CLI reports instead of a traceback. --- docs/examples/zipapp.py | 54 +- pyproject.toml | 17 +- src/python_sbom/__init__.py | 73 ++- src/python_sbom/__main__.py | 4 +- src/python_sbom/_backend.py | 98 --- src/python_sbom/_cli/__init__.py | 2 +- src/python_sbom/_cli/_commands.py | 75 +++ src/python_sbom/_cli/_main.py | 34 + src/python_sbom/_cli/_parser.py | 114 ++-- src/python_sbom/_cli/_run.py | 49 -- src/python_sbom/_collect/__init__.py | 2 +- src/python_sbom/_collect/_facts.py | 249 ++++---- src/python_sbom/_collect/_settings.py | 131 ++-- src/python_sbom/_collect/_source.py | 142 ++--- src/python_sbom/_errors.py | 5 + src/python_sbom/_extension.py | 29 - src/python_sbom/_formats/_spdx.py | 108 ++-- src/python_sbom/_generate/__init__.py | 1 + src/python_sbom/_generate/_backend.py | 115 ++++ src/python_sbom/_generate/_content.py | 84 +++ src/python_sbom/_generate/_pipeline.py | 62 ++ src/python_sbom/_model/__init__.py | 2 +- src/python_sbom/_model/_bom.py | 189 ++++++ src/python_sbom/_model/_components.py | 263 +++++--- src/python_sbom/_model/_document.py | 118 ++++ src/python_sbom/_model/_inputs.py | 235 +++++++ src/python_sbom/_model/_markers.py | 43 ++ src/python_sbom/_model/_tools.py | 56 ++ src/python_sbom/_model/_types.py | 59 +- src/python_sbom/_model/_wheel.py | 329 ---------- src/python_sbom/_validate/_checks.py | 218 ------- src/python_sbom/_validate/_document.py | 114 ++++ src/python_sbom/_validate/_licenses.py | 116 ++++ src/python_sbom/_validate/_schemas.py | 33 +- src/python_sbom/_validate/_wheel.py | 39 ++ src/python_sbom/_wheel/__init__.py | 2 +- src/python_sbom/_wheel/_archive.py | 121 ++++ src/python_sbom/_wheel/_embed.py | 128 ---- src/python_sbom/_wheel/_record.py | 61 ++ src/python_sbom/hatch/__init__.py | 2 +- src/python_sbom/hatch/_hook.py | 160 ++--- tests/_cli/test_cli.py | 149 +++++ tests/_collect/test_collect_facts.py | 221 +++++++ tests/_collect/test_collect_settings.py | 79 +++ tests/_collect/test_collect_source.py | 185 ++++++ tests/_formats/test_formats_spdx.py | 192 ++++++ tests/_generate/test_generate_backend.py | 43 ++ tests/_generate/test_generate_content.py | 80 +++ tests/_model/test_model_bom.py | 379 +++++++++++ tests/_model/test_model_components.py | 306 +++++++++ tests/_model/test_model_document.py | 66 ++ tests/_model/test_model_tools.py | 52 ++ tests/{seeder => _seeder}/LICENSE | 0 tests/_seeder/hatch_build.py | 53 ++ tests/{seeder => _seeder}/pyproject.toml | 0 .../src/seeder/__init__.py | 0 tests/_seeder/src/seeder/embed/__init__.py | 10 + tests/_validate/test_validate_document.py | 184 ++++++ tests/_validate/test_validate_extra.py | 15 + tests/_validate/test_validate_wheel.py | 40 ++ tests/_wheel/test_wheel_embed.py | 137 ++++ tests/conftest.py | 281 ++++----- tests/hatch/conftest.py | 79 +++ tests/hatch/test_hatch_hook.py | 141 +++++ tests/seeder/hatch_build.py | 29 - tests/seeder/src/seeder/embed/__init__.py | 0 tests/seeder/src/seeder/embed/bundle.py | 9 - tests/test_cli.py | 83 --- tests/test_embed.py | 142 ----- tests/test_examples.py | 24 + tests/test_hook.py | 144 ----- tests/test_main.py | 21 - tests/test_no_hatchling.py | 14 + tests/test_project.py | 593 ------------------ tests/test_spdx.py | 198 ------ tests/test_validate.py | 258 -------- tox.toml | 1 - 77 files changed, 4863 insertions(+), 3081 deletions(-) delete mode 100644 src/python_sbom/_backend.py create mode 100644 src/python_sbom/_cli/_commands.py create mode 100644 src/python_sbom/_cli/_main.py delete mode 100644 src/python_sbom/_cli/_run.py delete mode 100644 src/python_sbom/_extension.py create mode 100644 src/python_sbom/_generate/__init__.py create mode 100644 src/python_sbom/_generate/_backend.py create mode 100644 src/python_sbom/_generate/_content.py create mode 100644 src/python_sbom/_generate/_pipeline.py create mode 100644 src/python_sbom/_model/_bom.py create mode 100644 src/python_sbom/_model/_document.py create mode 100644 src/python_sbom/_model/_inputs.py create mode 100644 src/python_sbom/_model/_markers.py create mode 100644 src/python_sbom/_model/_tools.py delete mode 100644 src/python_sbom/_model/_wheel.py delete mode 100644 src/python_sbom/_validate/_checks.py create mode 100644 src/python_sbom/_validate/_document.py create mode 100644 src/python_sbom/_validate/_licenses.py create mode 100644 src/python_sbom/_validate/_wheel.py create mode 100644 src/python_sbom/_wheel/_archive.py delete mode 100644 src/python_sbom/_wheel/_embed.py create mode 100644 src/python_sbom/_wheel/_record.py create mode 100644 tests/_cli/test_cli.py create mode 100644 tests/_collect/test_collect_facts.py create mode 100644 tests/_collect/test_collect_settings.py create mode 100644 tests/_collect/test_collect_source.py create mode 100644 tests/_formats/test_formats_spdx.py create mode 100644 tests/_generate/test_generate_backend.py create mode 100644 tests/_generate/test_generate_content.py create mode 100644 tests/_model/test_model_bom.py create mode 100644 tests/_model/test_model_components.py create mode 100644 tests/_model/test_model_document.py create mode 100644 tests/_model/test_model_tools.py rename tests/{seeder => _seeder}/LICENSE (100%) create mode 100644 tests/_seeder/hatch_build.py rename tests/{seeder => _seeder}/pyproject.toml (100%) rename tests/{seeder => _seeder}/src/seeder/__init__.py (100%) create mode 100644 tests/_seeder/src/seeder/embed/__init__.py create mode 100644 tests/_validate/test_validate_document.py create mode 100644 tests/_validate/test_validate_extra.py create mode 100644 tests/_validate/test_validate_wheel.py create mode 100644 tests/_wheel/test_wheel_embed.py create mode 100644 tests/hatch/conftest.py create mode 100644 tests/hatch/test_hatch_hook.py delete mode 100644 tests/seeder/hatch_build.py delete mode 100644 tests/seeder/src/seeder/embed/__init__.py delete mode 100644 tests/seeder/src/seeder/embed/bundle.py delete mode 100644 tests/test_cli.py delete mode 100644 tests/test_embed.py create mode 100644 tests/test_examples.py delete mode 100644 tests/test_hook.py delete mode 100644 tests/test_main.py create mode 100644 tests/test_no_hatchling.py delete mode 100644 tests/test_project.py delete mode 100644 tests/test_spdx.py delete mode 100644 tests/test_validate.py diff --git a/docs/examples/zipapp.py b/docs/examples/zipapp.py index f1ed45f..495dba2 100644 --- a/docs/examples/zipapp.py +++ b/docs/examples/zipapp.py @@ -1,9 +1,20 @@ from __future__ import annotations -import uuid +import zipfile +from datetime import UTC, datetime +from email.parser import Parser from typing import TYPE_CHECKING -from python_sbom import BuildFacts, document, git_revision +from python_sbom import ( + collect_facts, + default_revision, + document, + file_component_from_bytes, + formulation, + project_identity, + settings_from_pyproject, + tool_components, +) if TYPE_CHECKING: from pathlib import Path @@ -11,26 +22,37 @@ from python_sbom import Body, Component, Document -def describe_zipapp(root: Path, version: str) -> Document: # simplint: ignore[SMP408] # the docs include it - # the build toolchain and time, recorded the same way the wheel SBOM records them - facts = BuildFacts.collect(root, git_revision) - zipapp: Component = { - "type": "application", - "bom-ref": f"pkg:generic/virtualenv.pyz@{version}", - "name": "virtualenv.pyz", - "version": version, - } +def describe_zipapp(root: Path, zipapp: Path) -> Document: # simplint: ignore[SMP408] # the docs include it + with zipfile.ZipFile(zipapp) as archive: + members = sorted(name for name in archive.namelist() if not name.endswith("/")) + metadata = Parser().parsestr( + archive.read(next(name for name in members if name.endswith(".dist-info/METADATA"))).decode() + ) + app: Component = { + "type": "application", + "bom-ref": f"pkg:generic/{zipapp.name}@{metadata['Version']}", + "name": zipapp.name, + "version": metadata["Version"], + } + app["components"] = [file_component_from_bytes(app["bom-ref"], name, archive.read(name)) for name in members] + # the build toolchain, time and serial namespace, taken the way the wheel SBOM takes them + facts = collect_facts(root, default_revision, None, None) + identity = project_identity(metadata, settings_from_pyproject(root, {})) + tools, tool_dependencies = tool_components(facts) body: Body = { "metadata": { - "timestamp": facts.timestamp, + "timestamp": datetime.fromtimestamp(facts.epoch, UTC).isoformat(), "lifecycles": [{"phase": "build"}], - "tools": {"components": facts.tools}, - "component": zipapp, + "tools": {"components": tools}, + "component": app, + # naming the document's file marks it as built the python-sbom way, so validate checks it in full + "properties": [{"name": f"{identity.namespace}:sbom:file", "value": f"{zipapp.name}.cdx.json"}], }, "components": [], - "dependencies": [{"ref": zipapp["bom-ref"], "dependsOn": []}, *facts.tool_dependencies], + "dependencies": [{"ref": app["bom-ref"], "dependsOn": []}, *tool_dependencies], + "formulation": [formulation("zipapp-build", "build the zipapp", tools, app["bom-ref"], facts)], } - return document(body, uuid.uuid5(uuid.NAMESPACE_URL, "https://github.com/pypa/virtualenv/sboms")) + return document(body, identity.serial_namespace) __all__ = [ diff --git a/pyproject.toml b/pyproject.toml index a11b283..b451fa1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ urls.Documentation = "https://python-sbom.readthedocs.io" urls.Homepage = "https://github.com/tox-dev/python-sbom" urls.Source = "https://github.com/tox-dev/python-sbom" urls.Tracker = "https://github.com/tox-dev/python-sbom/issues" -scripts.python-sbom = "python_sbom.__main__:main" +scripts.python-sbom = "python_sbom._cli._main:main" entry-points.hatch.python-sbom = "python_sbom.hatch.hooks" [dependency-groups] @@ -140,20 +140,18 @@ lint.ignore = [ "incorrect-blank-line-before-class", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible "missing-trailing-comma", # Conflict with formatter "multi-line-summary-first-line", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible - "non-empty-init-module", # `__init__` module should only contain docstrings and re-exports "single-line-implicit-string-concatenation", # Conflict with formatter ] lint.per-file-ignores."tests/**/*.py" = [ "assert", # Tests use assertions - "D", # Test names document behavior "FBT", # Boolean parameters keep parameterized cases compact "implicit-namespace-package", # The tests directory is not a package - "import-private-name", # Tests cover private parsing boundaries directly "magic-value-comparison", # Literal expected values keep assertions local "subprocess-without-shell-equals-true", # Entry-point arguments are fixed by the tests "suspicious-subprocess-import", # Entry-point tests invoke a local Python executable - "too-many-positional-arguments", # Test inputs favor explicit parameters over setup objects ] +# the fixture mirrors virtualenv's seed/wheels/embed/__init__.py, which holds the bundle table +lint.per-file-ignores."tests/_seeder/src/**/__init__.py" = [ "non-empty-init-module" ] lint.isort = { known-first-party = [ "python_sbom", ], required-imports = [ @@ -219,6 +217,15 @@ type = [ { directory = "packaging", name = "Packaging", showcontent = true }, ] +[tool.simplint] +docstring-style = "rst" +# the fixture mirrors virtualenv's seed/wheels/embed/__init__.py, which holds the bundle table +per-file-ignores."tests/_seeder/src/**/__init__.py" = [ + "init-module-content", + "missing-final", + "module-variable-prefix" +] + [tool.nab] mode = "universal" base-group = "project-runtime" diff --git a/src/python_sbom/__init__.py b/src/python_sbom/__init__.py index f3ecbf4..783bd91 100644 --- a/src/python_sbom/__init__.py +++ b/src/python_sbom/__init__.py @@ -2,47 +2,96 @@ from __future__ import annotations -from ._backend import Backend -from ._cli._parser import Options, main -from ._collect._facts import BuildFacts, RevisionReader, git_revision -from ._collect._settings import Settings, Toml -from ._errors import SbomError, SettingsError, SourceRevisionError, WheelError -from ._extension import ContentProvider +from ._cli._main import main +from ._collect._facts import RevisionReader, collect_facts, default_revision, git_revision +from ._collect._settings import Toml, settings_from_pyproject, settings_from_table +from ._collect._source import read_bundled_wheel, read_source_tree +from ._errors import MissingExtraError, SbomError, SettingsError, SourceRevisionError, WheelError from ._formats._json import dumps from ._formats._spdx import Spdx, to_spdx -from ._model._components import component_from_metadata, document -from ._model._types import Body, Component, Dependency, Document, OrganizationalEntity, Property -from ._validate._checks import validate, validate_wheel -from ._wheel._embed import BuiltWheel -from .version import version as __version__ +from ._generate._backend import Backend, BuiltWheel, MetadataBackend +from ._generate._content import ContentProvider, load_content_provider +from ._generate._pipeline import Sbom, build_sbom +from ._model._components import bundled_wheel_component, component_from_metadata, file_component_from_bytes, script_tool +from ._model._document import ProjectIdentity, document, formulation, project_identity +from ._model._inputs import ( + BuildFacts, + BuildScript, + BundledWheel, + BundledWheelInfo, + Contributions, + InstalledDistribution, + Interpreter, + RecordRow, + Revision, + SbomRequest, + Settings, + SourceTree, +) +from ._model._tools import tool_components +from ._model._types import Body, Component, Dependency, Document, Formula, License, OrganizationalEntity, Property +from ._validate._document import validate +from ._validate._wheel import validate_wheel +from ._wheel._archive import embed_sbom +from .version import __version__ __all__ = [ "Backend", "Body", "BuildFacts", + "BuildScript", "BuiltWheel", + "BundledWheel", + "BundledWheelInfo", "Component", "ContentProvider", + "Contributions", "Dependency", "Document", - "Options", + "Formula", + "InstalledDistribution", + "Interpreter", + "License", + "MetadataBackend", + "MissingExtraError", "OrganizationalEntity", + "ProjectIdentity", "Property", + "RecordRow", + "Revision", "RevisionReader", + "Sbom", "SbomError", + "SbomRequest", "Settings", "SettingsError", "SourceRevisionError", + "SourceTree", "Spdx", "Toml", "WheelError", "__version__", + "build_sbom", + "bundled_wheel_component", + "collect_facts", "component_from_metadata", + "default_revision", "document", "dumps", + "embed_sbom", + "file_component_from_bytes", + "formulation", "git_revision", + "load_content_provider", "main", + "project_identity", + "read_bundled_wheel", + "read_source_tree", + "script_tool", + "settings_from_pyproject", + "settings_from_table", "to_spdx", + "tool_components", "validate", "validate_wheel", ] diff --git a/src/python_sbom/__main__.py b/src/python_sbom/__main__.py index dc323c4..2c7a692 100644 --- a/src/python_sbom/__main__.py +++ b/src/python_sbom/__main__.py @@ -1,10 +1,8 @@ -"""Main entry point.""" - from __future__ import annotations import sys -from ._cli._parser import main +from ._cli._main import main if __name__ == "__main__": main(sys.argv[1:]) diff --git a/src/python_sbom/_backend.py b/src/python_sbom/_backend.py deleted file mode 100644 index 17e69fc..0000000 --- a/src/python_sbom/_backend.py +++ /dev/null @@ -1,98 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING - -from packaging.utils import canonicalize_name - -from ._collect._source import read_source_tree -from ._formats._json import dumps -from ._model._wheel import WheelInputs, describe - -if TYPE_CHECKING: - from email.message import Message - from pathlib import Path, PurePosixPath - - from ._collect._facts import BuildFacts - from ._collect._settings import Settings - from ._model._types import Property - - -class Backend(ABC): - """ - Adapts a build system to the core: it supplies the wheel's metadata and the project's settings. - - The hatchling hook and :class:`BuiltWheel` are the two backends shipped; another build system adds a subclass. - :meth:`sbom` reads the project tree, builds the document and serializes it. - - """ - - def bundled_wheel_properties(self, wheel: Path) -> list[Property]: # ruff: ignore[no-self-use] # extension seam - """ - Return extra properties for a wheel the project bundles as a data file; none by default. - - :param wheel: the bundled wheel, an absolute path under :attr:`root` - :returns: the properties to add after ``:bundled-wheel`` - - """ - del wheel - return [] - - def sbom(self, facts: BuildFacts) -> tuple[str, bytes]: - """ - Build the SBOM of the wheel. - - :param facts: what the environment says about the build, collected once per build - :returns: the SBOM's file name under ``.dist-info/sboms/`` and its JSON - - """ - metadata = self.metadata() - source = read_source_tree( - self.root, self.settings, metadata.get_all("License-File", []), self.build_script(), self - ) - inputs = WheelInputs( - metadata=metadata, wheel_name=self.wheel_name(), settings=self.settings, facts=facts, source=source - ) - return f"{canonicalize_name(metadata['Name'])}.cdx.json", dumps(describe(inputs)) - - @abstractmethod - def metadata(self) -> Message: - """ - Return the core metadata of the wheel. - - :returns: the parsed ``METADATA`` - - """ - - @property - @abstractmethod - def root(self) -> Path: - """The project directory.""" - - @property - @abstractmethod - def settings(self) -> Settings: - """The project's options.""" - - def build_script(self) -> PurePosixPath | None: # ruff: ignore[no-self-use] # backends override it - """ - Return a build script to list among the build tools, as hatchling's custom hook script. - - :returns: the script, relative to :attr:`root`, or ``None``, the default, for none - - """ - return None - - @abstractmethod - def wheel_name(self) -> str: - """ - Return the file name of the wheel, for its attestation link. - - :returns: the file name - - """ - - -__all__ = [ - "Backend", -] diff --git a/src/python_sbom/_cli/__init__.py b/src/python_sbom/_cli/__init__.py index 376611a..f60ec27 100644 --- a/src/python_sbom/_cli/__init__.py +++ b/src/python_sbom/_cli/__init__.py @@ -1 +1 @@ -"""The command line: parsing options and running one command.""" +"""The command line: parsing options and running one command per subcommand.""" diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py new file mode 100644 index 0000000..9115121 --- /dev/null +++ b/src/python_sbom/_cli/_commands.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +from python_sbom._collect._facts import collect_facts, default_revision +from python_sbom._collect._settings import settings_from_pyproject +from python_sbom._collect._source import read_source_tree +from python_sbom._formats._json import dumps +from python_sbom._formats._spdx import to_spdx +from python_sbom._generate._backend import BuiltWheel +from python_sbom._generate._content import ContentProvider, load_content_provider +from python_sbom._generate._pipeline import build_sbom +from python_sbom._model._inputs import Revision +from python_sbom._validate._wheel import validate_wheel +from python_sbom._wheel._archive import embed_sbom +from python_sbom.version import __version__ + +if TYPE_CHECKING: + from python_sbom._collect._facts import RevisionReader + + from ._parser import Options + + +def embed(options: Options) -> bool: + root = Path.cwd() if options.root is None else options.root + settings = settings_from_pyproject(root, {}) + content, script = ( + (ContentProvider(), None) + if (spec := options.content_provider or settings.content_provider) is None + else load_content_provider(spec, root) + ) + revision: RevisionReader = ( + default_revision if (commit := options.commit) is None else lambda _root: Revision(commit=commit, dirty=False) + ) + facts = collect_facts(root, revision, options.build_env, None) + for wheel in options.wheels: + request = BuiltWheel( + wheel=wheel, root=root, settings=settings, build_scripts=() if script is None else (script,) + ).request() + sbom = build_sbom(request, read_source_tree(request), facts, content) + sys.stdout.write(f"wrote {embed_sbom(wheel, sbom.file_name, sbom.content, facts.epoch)} into {wheel.name}\n") + return True + + +def spdx(options: Options) -> bool: + options.target.write_bytes(dumps(to_spdx(json.loads(options.source.read_text(encoding="utf-8")), __version__))) + return True + + +def validate(options: Options) -> bool: + allowed = None if options.allowed_licenses is None else frozenset(options.allowed_licenses) + if not ( + wheels := sorted(wheel for path in options.paths for wheel in (path.glob("*.whl") if path.is_dir() else [path])) + ): + sys.stderr.write(f"no wheel found in {', '.join(map(str, options.paths))}\n") + return False + valid = True + for wheel in wheels: + for problem in (problems := validate_wheel(wheel, allowed)): + sys.stderr.write(f"SBOM invalid: {wheel.name}: {problem}\n") + if problems: + valid = False + else: + sys.stdout.write(f"SBOM in {wheel.name} is valid\n") + return valid + + +__all__ = [ + "embed", + "spdx", + "validate", +] diff --git a/src/python_sbom/_cli/_main.py b/src/python_sbom/_cli/_main.py new file mode 100644 index 0000000..f489ae5 --- /dev/null +++ b/src/python_sbom/_cli/_main.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING + +from python_sbom._errors import SbomError + +from ._parser import Options, build_parser + +if TYPE_CHECKING: + from collections.abc import Sequence + + +def main(args: Sequence[str] | None = None) -> None: + """ + Run the command-line interface. + + :param args: command-line arguments, or ``sys.argv[1:]`` when ``None`` + :raises SystemExit: with status 1 when a command fails + + """ + options = build_parser().parse_args(args, namespace=Options()) + try: + succeeded = options.handler(options) + except SbomError as error: + sys.stderr.write(f"python-sbom: {error}\n") + succeeded = False + if not succeeded: + raise SystemExit(1) + + +__all__ = [ + "main", +] diff --git a/src/python_sbom/_cli/_parser.py b/src/python_sbom/_cli/_parser.py index da657f5..f5923ac 100644 --- a/src/python_sbom/_cli/_parser.py +++ b/src/python_sbom/_cli/_parser.py @@ -1,66 +1,67 @@ from __future__ import annotations -import sys -from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter +from argparse import ArgumentParser, Namespace from pathlib import Path from typing import TYPE_CHECKING -from python_sbom._errors import SbomError -from python_sbom.version import version - -from ._run import run +from ._commands import embed, spdx, validate if TYPE_CHECKING: - from collections.abc import Sequence - - -def main(args: Sequence[str] | None = None) -> None: - """ - Run the command-line interface. + from collections.abc import Callable - :param args: command-line arguments, or ``sys.argv[1:]`` when None - :raises SystemExit: with status 1 when a command fails - """ - options = _build_parser().parse_args(args, namespace=Options()) - try: - succeeded = run(options) - except SbomError as error: - sys.stderr.write(f"python-sbom {options.command}: {error}\n") - succeeded = False - if not succeeded: - raise SystemExit(1) +class Options(Namespace): + handler: Callable[[Options], bool] + wheels: list[Path] + root: Path | None + commit: str | None + build_env: list[str] | None + content_provider: str | None + source: Path + target: Path + paths: list[Path] + allowed_licenses: list[str] | None -def _build_parser() -> ArgumentParser: - parser = ArgumentParser( - prog="python-sbom", - description="SBOMs for Python distributions: write, convert and check the PEP 770 CycloneDX SBOMs of wheels.", - formatter_class=RawDescriptionHelpFormatter, - epilog=f"running {version} at {Path(__file__).parent}", - ) - commands = parser.add_subparsers(dest="command", required=True) - embed = commands.add_parser( +def build_parser() -> ArgumentParser: + parser = ArgumentParser(prog="python-sbom", description="Write, convert and check the CycloneDX SBOMs of wheels.") + commands = parser.add_subparsers(required=True) + embed_parser = commands.add_parser( "embed", help="write the SBOM into wheels another build backend built", - description="Write the SBOM the hatchling hook would write into wheels another build backend built, such as" - " meson-python. Settings come from the [tool.python-sbom] table of the project's pyproject.toml.", + description="Write the SBOM the hatchling hook writes into wheels another build backend built, such as" + " meson-python. Options come from the [tool.python-sbom] table of the project's pyproject.toml.", + ) + embed_parser.set_defaults(handler=embed) + embed_parser.add_argument("wheels", type=Path, nargs="+", metavar="wheel", help="wheel to rewrite in place") + embed_parser.add_argument( + "--root", type=Path, help="project directory the wheels were built from (default: current directory)" ) - embed.add_argument("wheels", type=Path, nargs="+", metavar="wheel", help="wheel to rewrite in place") - embed.add_argument( - "--root", - type=Path, - default=Path.cwd(), - help="project directory the wheels were built from (default: %(default)s)", + embed_parser.add_argument("--commit", help="source revision to record, for a wheel built from an sdist") + embed_parser.add_argument( + "--build-env", + action="append", + metavar="dir", + help="site-packages directory of the environment that built the wheels; repeat it per directory" + " (default: this interpreter's)", ) - spdx = commands.add_parser("spdx", help="render a CycloneDX SBOM as SPDX 2.3 JSON") - spdx.add_argument("source", type=Path, help="CycloneDX JSON file to read") - spdx.add_argument("target", type=Path, help="SPDX JSON file to write") - validate = commands.add_parser( + embed_parser.add_argument( + "--content-provider", + metavar="module:attribute", + help="content provider, overriding the content-provider option", + ) + spdx_parser = commands.add_parser("spdx", help="render a CycloneDX SBOM as SPDX 2.3 JSON") + spdx_parser.set_defaults(handler=spdx) + spdx_parser.add_argument("source", type=Path, help="CycloneDX JSON file to read") + spdx_parser.add_argument("target", type=Path, help="SPDX JSON file to write") + validate_parser = commands.add_parser( "validate", help="validate the SBOMs in wheels", description="Needs the validate extra: python-sbom[validate]." ) - validate.add_argument("paths", type=Path, nargs="+", metavar="path", help="wheel, or directory holding wheels") - validate.add_argument( + validate_parser.set_defaults(handler=validate) + validate_parser.add_argument( + "paths", type=Path, nargs="+", metavar="path", help="wheel, or directory holding wheels" + ) + validate_parser.add_argument( "--allow-license", dest="allowed_licenses", action="append", @@ -70,26 +71,7 @@ def _build_parser() -> ArgumentParser: return parser -class Options(Namespace): - """The parsed command line.""" - - command: str - """The subcommand: ``embed``, ``spdx`` or ``validate``.""" - wheels: list[Path] - """Wheels to write an SBOM into, for ``embed``.""" - root: Path - """Project directory the wheels were built from, for ``embed``.""" - source: Path - """CycloneDX document to read, for ``spdx``.""" - target: Path - """SPDX document to write, for ``spdx``.""" - paths: list[Path] - """Wheels, or directories holding them, for ``validate``.""" - allowed_licenses: list[str] | None - """SPDX license ids bundled components may use, for ``validate``; None skips the license policy.""" - - __all__ = [ "Options", - "main", + "build_parser", ] diff --git a/src/python_sbom/_cli/_run.py b/src/python_sbom/_cli/_run.py deleted file mode 100644 index f9f92c6..0000000 --- a/src/python_sbom/_cli/_run.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import annotations - -import json -import sys -import tomllib -from typing import TYPE_CHECKING - -from python_sbom._collect._facts import BuildFacts, git_revision -from python_sbom._collect._settings import Settings -from python_sbom._formats._json import dumps -from python_sbom._formats._spdx import to_spdx -from python_sbom._validate._checks import validate_wheel -from python_sbom._wheel._embed import BuiltWheel - -if TYPE_CHECKING: - from ._parser import Options - - -def run(options: Options) -> bool: - if options.command == "embed": - pyproject = options.root / "pyproject.toml" - table = tomllib.loads(pyproject.read_text(encoding="utf-8")).get("tool", {}).get("python-sbom", {}) - settings = Settings.from_table(table, f"[tool.python-sbom] of {pyproject}") - facts = BuildFacts.collect(options.root, git_revision) - for wheel in options.wheels: - sys.stdout.write(f"wrote {BuiltWheel(wheel, options.root, settings).embed(facts)} into {wheel.name}\n") - return True - if options.command == "spdx": - options.target.write_bytes(dumps(to_spdx(json.loads(options.source.read_text(encoding="utf-8"))))) - return True - allowed = None if options.allowed_licenses is None else frozenset(options.allowed_licenses) - wheels = sorted(wheel for path in options.paths for wheel in (path.glob("*.whl") if path.is_dir() else [path])) - if not wheels: - sys.stderr.write(f"no wheel found in {', '.join(map(str, options.paths))}\n") - return False - valid = True - for wheel in wheels: - for problem in (problems := validate_wheel(wheel, allowed)): - sys.stderr.write(f"SBOM invalid: {wheel.name}: {problem}\n") - if problems: - valid = False - else: - sys.stdout.write(f"SBOM in {wheel.name} is valid\n") - return valid - - -__all__ = [ - "run", -] diff --git a/src/python_sbom/_collect/__init__.py b/src/python_sbom/_collect/__init__.py index c468933..8083326 100644 --- a/src/python_sbom/_collect/__init__.py +++ b/src/python_sbom/_collect/__init__.py @@ -1 +1 @@ -"""Facts gathered at the edge: settings, the build environment, the source revision, bundled wheels.""" +"""Facts gathered at the edge: settings, the build environment, the source revision and the project tree.""" diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index 4a7897c..cb6f1dc 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -1,156 +1,187 @@ from __future__ import annotations +import json import os import platform +import re import shutil -import subprocess # ruff: ignore[suspicious-subprocess-import] # runs git to read the source revision +import subprocess # ruff: ignore[suspicious-subprocess-import] # runs git to read the source revision import sys -from dataclasses import dataclass -from datetime import UTC, datetime +import time from email.parser import Parser from importlib.metadata import distributions from typing import TYPE_CHECKING, Final, Protocol from packaging.requirements import Requirement +from packaging.utils import canonicalize_name -from python_sbom._errors import SourceRevisionError -from python_sbom._model._components import component_from_metadata, file_component, purl +from python_sbom._errors import SettingsError, SourceRevisionError +from python_sbom._model._inputs import BuildFacts, InstalledDistribution, Interpreter, RecordRow, Revision +from python_sbom._wheel._record import decode_digest if TYPE_CHECKING: + from collections.abc import Sequence from importlib.metadata import Distribution from pathlib import Path - from python_sbom._model._types import Component, Dependency - class RevisionReader(Protocol): - """Reads the source revision a build comes from.""" + """Reads the source revision of a project directory.""" - def __call__(self, root: Path, /) -> str | None: + def __call__(self, root: Path, /) -> Revision | None: """ - Read the revision of a source tree. + Read the revision. :param root: the project directory - :returns: the revision, or ``None`` when the directory is not under version control - :raises SourceRevisionError: when it is, but the revision cannot be read + :returns: the revision, or ``None`` when it is unknown + :raises SourceRevisionError: when version control holds the directory but cannot read its revision """ -@dataclass(frozen=True, slots=True, kw_only=True) -class BuildFacts: +def collect_facts( + root: Path, revision: RevisionReader, path: Sequence[str] | None, default_epoch: int | None +) -> BuildFacts: """ - What the SBOM records about the build itself, gathered once from the environment that runs it. + Gather what the environment says about a build. + + Nothing about the machine goes in: the same toolchain, revision and build time give the same facts on any + operating system. - :param epoch: the build time as a Unix timestamp, from ``SOURCE_DATE_EPOCH`` - :param commit: the source revision of a git checkout, or ``None`` outside one - :param tools: the interpreter and every distribution of the build environment, interpreter first - :param tool_dependencies: the dependency graph between those distributions + :param root: the project directory + :param revision: reads the source revision, such as :func:`default_revision` + :param path: the directories to find the build environment's distributions in, or ``None`` for :data:`sys.path` + :param default_epoch: the build time when ``SOURCE_DATE_EPOCH`` is unset, such as the fixed time of a reproducible + hatchling build, or ``None`` for the current time + :returns: the facts + :raises SettingsError: when ``SOURCE_DATE_EPOCH`` is not an integer """ + if from_environment := bool(raw := os.environ.get("SOURCE_DATE_EPOCH", "")): + try: + epoch = int(raw) + except ValueError as error: + msg = f"SOURCE_DATE_EPOCH must be a Unix timestamp, got {raw!r}" + raise SettingsError(msg) from error + else: + epoch = int(time.time()) if default_epoch is None else default_epoch + return BuildFacts( + epoch=epoch, + epoch_from_environment=from_environment, + revision=revision(root), + interpreter=Interpreter( + name=sys.implementation.name, + version=platform.python_version(), + implementation=platform.python_implementation(), + ), + distributions=_installed(path), + ) + + +def _installed(path: Sequence[str] | None) -> list[InstalledDistribution]: + found: Final[dict[str, Distribution]] = {} + for distribution in distributions(path=list(path)) if path is not None else distributions(): + # the first on the path is the one imports load, so a shadowed copy further down is left out + found.setdefault(canonicalize_name(distribution.metadata["Name"]), distribution) + return [ + InstalledDistribution( + metadata=Parser().parsestr(distribution.read_text("METADATA") or ""), + files=_recorded_files(distribution), + depends_on=sorted({ + name + for requirement in map(Requirement, distribution.requires or []) + if (name := canonicalize_name(requirement.name)) in found + and (requirement.marker is None or requirement.marker.evaluate()) + }), + ) + for distribution in found.values() + ] - epoch: int - commit: str | None - tools: list[Component] - tool_dependencies: list[Dependency] - @classmethod - def collect(cls, root: Path, revision: RevisionReader) -> BuildFacts: - """ - Read the build time, the source revision and the build environment. +def _recorded_files(distribution: Distribution) -> list[RecordRow]: + # a platform wheel's files are built for this machine's OS and architecture, and an editable install's point at a + # checkout, so listing either would tie the document to one machine + if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] != "true" or ( + json.loads(distribution.read_text("direct_url.json") or "{}").get("dir_info", {}).get("editable") + ): + return [] + rows: Final[list[RecordRow]] = [] + for file in distribution.files or []: + if file.hash is None or not _describes_distribution(file.as_posix(), file.parent.suffix, file.name): + continue + algorithm, digest = decode_digest(f"{file.hash.mode}={file.hash.value}", file.as_posix()) + rows.append(RecordRow(path=file.as_posix(), algorithm=algorithm, digest=digest, size=str(file.size))) + return rows + + +def _describes_distribution(path: str, parent_suffix: str, name: str) -> bool: + # console-script launchers live outside site-packages and embed the environment's interpreter path; the installer + # metadata names the frontend; uv's cache files hold paths and times; none of them says anything about the package + return not ( + path.startswith("../") + or (parent_suffix == ".dist-info" and name in {"INSTALLER", "REQUESTED", "direct_url.json"}) + or re.fullmatch(r"uv_.*\.json", name) + ) + + +def default_revision(root: Path, /) -> Revision | None: + """ + Read the source revision from the first source that knows it. - Nothing about the machine goes in: the same source, toolchain and ``SOURCE_DATE_EPOCH`` give the same facts - on any operating system. A pure-Python distribution lists its files with their hashes; a platform one does - not, because its files differ per machine. + The sources, in order: the ``PYTHON_SBOM_COMMIT`` environment variable, for builds from an sdist; git, for a + checkout or a directory inside one; and the ``node`` of an expanded ``.git_archival.txt``, for a ``git archive``. - :param root: the project directory - :param revision: reads the source revision of ``root``, such as :func:`git_revision` - :returns: the facts + :param root: the project directory + :returns: the revision, or ``None`` when no source knows it + :raises SourceRevisionError: when a git checkout holds the directory but git cannot read its revision - """ - tools, tool_dependencies = _build_tools() - # hatchling's default when the variable is unset, so a hook build and an embed run agree - epoch = int(os.environ.get("SOURCE_DATE_EPOCH", "1580601600")) - return cls(epoch=epoch, commit=revision(root), tools=tools, tool_dependencies=tool_dependencies) - - @property - def timestamp(self) -> str: - """The build time in ISO 8601.""" - return datetime.fromtimestamp(self.epoch, tz=UTC).isoformat() - - -def _build_tools() -> tuple[list[Component], list[Dependency]]: - # nothing about the machine goes in, so the same toolchain yields the same record on any operating system - interpreter = f"pkg:generic/{sys.implementation.name}@{platform.python_version()}" - tools: Final[list[Component]] = [ - { - "type": "platform", - "bom-ref": f"tool:{interpreter}", - "name": sys.implementation.name, - "version": platform.python_version(), - "purl": interpreter, - "properties": [{"name": "python:implementation", "value": platform.python_implementation()}], - }, - ] - # bom-refs are prefixed because the same distribution can be both a build tool and a component of the artifact - installed = {purl(distribution.metadata["Name"]): distribution for distribution in distributions()} - tool_dependencies: Final[list[Dependency]] = [] - for distribution in (installed[key] for key in sorted(installed)): - component = component_from_metadata(distribution.metadata, "library") - component["bom-ref"] = f"tool:{component['bom-ref']}" - if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] == "true": - component["components"] = [ - file_component( - component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) - ) - for file in distribution.files or [] - # console-script launchers live outside site-packages and embed the environment's interpreter path in - # their shebang, so their hash differs on every build; the installer metadata names the frontend that - # set up the environment rather than the distribution - if file.hash is not None - and not file.as_posix().startswith("../") - and not ( - file.parent.suffix == ".dist-info" and file.name in {"INSTALLER", "REQUESTED", "direct_url.json"} - ) - ] - tools.append(component) - tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) - return tools, tool_dependencies - - -def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) -> list[str]: - refs = set() - for requirement in map(Requirement, distribution.requires or []): - if (requirement.marker is None or requirement.marker.evaluate()) and ( - dependency := installed.get(purl(requirement.name)) - ): - refs.add(f"tool:{purl(dependency.metadata['Name'], dependency.version)}") - return sorted(refs) - - -def git_revision(root: Path, /) -> str | None: """ - Read the commit of a git checkout. + if commit := os.environ.get("PYTHON_SBOM_COMMIT"): + return Revision(commit=commit, dirty=False) + if (found := git_revision(root)) is not None: + return found + archival = root / ".git_archival.txt" + if archival.is_file() and ( + match := re.search(r"^node: ([0-9a-f]{40,64})$", archival.read_text(encoding="utf-8"), re.MULTILINE) + ): + return Revision(commit=match[1], dirty=False) + return None + + +def git_revision(root: Path, /) -> Revision | None: + """ + Read the commit of the git checkout that holds a directory. - :param root: the project directory - :returns: the commit, or ``None`` when ``root`` holds no ``.git``, as in a build from an sdist, or git is missing - :raises SourceRevisionError: when git cannot read the commit, such as in a checkout it considers unsafe + :param root: the project directory, or any directory inside a checkout + :returns: the commit and whether tracked files changed, or ``None`` outside a checkout, without git, or before the + first commit + :raises SourceRevisionError: when git cannot read the checkout, such as one it considers unsafe """ - if not (root / ".git").exists() or (git := shutil.which("git")) is None: + resolved = root.resolve() + if (git := shutil.which("git")) is None or not any( + (directory / ".git").exists() for directory in (resolved, *resolved.parents) + ): return None - try: - result = subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] # fixed arguments - [git, "rev-parse", "HEAD"], check=True, capture_output=True, text=True, cwd=root - ) - except subprocess.CalledProcessError as error: - msg = f"git rev-parse HEAD failed in {root}: {error.stderr.strip()}" - raise SourceRevisionError(msg) from error - return result.stdout.strip() + head = _git(git, root, "rev-parse", "--verify", "--quiet", "HEAD") + if head.returncode == 1: # a checkout before its first commit + return None + if head.returncode: + msg = f"git rev-parse HEAD failed in {root}: {head.stderr.strip()}" + raise SourceRevisionError(msg) + status = _git(git, root, "status", "--porcelain", "--untracked-files=no") + return Revision(commit=head.stdout.strip(), dirty=bool(status.stdout.strip())) + + +def _git(git: str, root: Path, *arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] # git from PATH with fixed arguments + [git, "-C", str(root), *arguments], check=False, capture_output=True, text=True + ) __all__ = [ - "BuildFacts", "RevisionReader", + "collect_facts", + "default_revision", "git_revision", ] diff --git a/src/python_sbom/_collect/_settings.py b/src/python_sbom/_collect/_settings.py index 9fefe96..340abf9 100644 --- a/src/python_sbom/_collect/_settings.py +++ b/src/python_sbom/_collect/_settings.py @@ -1,13 +1,15 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import TYPE_CHECKING, TypeAlias +import tomllib +from typing import TYPE_CHECKING, TypeAlias, TypeIs from python_sbom._errors import SettingsError +from python_sbom._model._inputs import Settings if TYPE_CHECKING: from collections.abc import Mapping from datetime import date, datetime, time + from pathlib import Path from python_sbom._model._types import OrganizationalEntity @@ -15,66 +17,85 @@ Toml: TypeAlias = "str | int | float | bool | datetime | date | time | list[Toml] | dict[str, Toml]" -@dataclass(frozen=True, slots=True, kw_only=True) -class Settings: +def settings_from_pyproject(root: Path, overrides: Mapping[str, Toml]) -> Settings: """ - The options a project sets, in a hatchling hook's table or in ``[tool.python-sbom]``. + Read the options from ``[tool.python-sbom]`` of a project's ``pyproject.toml``. - :param supplier: the organization that supplies the project, or ``None`` to name none - :param repository: the repository URL, or ``None`` to take the first ``Source``, ``Repository`` or ``GitHub`` - project URL of the metadata - :param default_branch: branch used in links to files in the repository - :param bundled_wheels: glob patterns, relative to the project root, of wheels the project ships as data files + :param root: the project directory + :param overrides: options that win over the table, such as a hatchling hook's own table + :returns: the settings + :raises SettingsError: when ``pyproject.toml`` is missing or not TOML, or an option is unknown or has the wrong type """ + pyproject = root / "pyproject.toml" + try: + loaded = tomllib.loads(pyproject.read_text(encoding="utf-8")) + except FileNotFoundError as error: + msg = f"{root} has no pyproject.toml to read [tool.python-sbom] from" + raise SettingsError(msg) from error + except tomllib.TOMLDecodeError as error: + msg = f"{pyproject} is not valid TOML: {error}" + raise SettingsError(msg) from error + table = loaded.get("tool", {}).get("python-sbom", {}) + return settings_from_table({**table, **overrides}, f"[tool.python-sbom] of {pyproject}") - supplier: OrganizationalEntity | None - repository: str | None - default_branch: str - bundled_wheels: tuple[str, ...] - - @classmethod - def from_table(cls, table: Mapping[str, Toml], where: str) -> Settings: - """ - Read the options from a configuration table, ignoring keys it does not know. - - :param table: the table, with any of ``supplier``, ``repository``, ``default-branch`` and ``bundled-wheels`` - :param where: where the table lives, named in error messages, such as ``[tool.python-sbom]`` - :returns: the settings - :raises SettingsError: when an option has the wrong type - - """ - supplier = table.get("supplier") - if supplier is not None and not ( - isinstance(supplier, dict) - and supplier.keys() <= {"name", "url"} - and isinstance(name := supplier.get("name"), str) - and isinstance(urls := supplier.get("url", []), list) - and all(isinstance(url, str) for url in urls) - ): - msg = f"`supplier` in {where} must be a table with a string `name` and an optional `url` string array" - raise SettingsError(msg) - if not isinstance(repository := table.get("repository"), str | None): - msg = f"`repository` in {where} must be a string" - raise SettingsError(msg) - if not isinstance(default_branch := table.get("default-branch", "main"), str): - msg = f"`default-branch` in {where} must be a string" - raise SettingsError(msg) - if not ( - isinstance(patterns := table.get("bundled-wheels", []), list) - and all(isinstance(pattern, str) for pattern in patterns) - ): - msg = f"`bundled-wheels` in {where} must be an array of strings" - raise SettingsError(msg) - return cls( - supplier=None if supplier is None else {"name": name, "url": [str(url) for url in urls]}, - repository=None if repository is None else repository.rstrip("/"), - default_branch=default_branch, - bundled_wheels=tuple(str(pattern) for pattern in patterns), - ) + +def settings_from_table(table: Mapping[str, Toml], where: str) -> Settings: + """ + Read the options from a configuration table. + + :param table: the table, with any of ``supplier``, ``repository``, ``default-branch``, ``bundled-wheels`` and + ``content-provider`` + :param where: where the table lives, named in error messages + :returns: the settings + :raises SettingsError: when a key is unknown or an option has the wrong type + + """ + known = {"bundled-wheels", "content-provider", "default-branch", "repository", "supplier"} + if unknown := sorted(table.keys() - known): + msg = f"{where} has unknown option(s) {', '.join(unknown)}; known: {', '.join(sorted(known))}" + raise SettingsError(msg) + if not isinstance(repository := table.get("repository"), str | None): + msg = f"`repository` in {where} must be a string" + raise SettingsError(msg) + if not isinstance(default_branch := table.get("default-branch", "main"), str): + msg = f"`default-branch` in {where} must be a string" + raise SettingsError(msg) + if not _strings(patterns := table.get("bundled-wheels", [])): + msg = f"`bundled-wheels` in {where} must be an array of strings" + raise SettingsError(msg) + if not isinstance(content_provider := table.get("content-provider"), str | None): + msg = f"`content-provider` in {where} must be a string, `module:attribute`" + raise SettingsError(msg) + return Settings( + supplier=_supplier(table.get("supplier"), where), + repository=None if repository is None else repository.rstrip("/"), + default_branch=default_branch, + bundled_wheels=tuple(patterns), + content_provider=content_provider, + ) + + +def _strings(values: object) -> TypeIs[list[str]]: + return isinstance(values, list) and all(isinstance(value, str) for value in values) + + +def _supplier(value: Toml | None, where: str) -> OrganizationalEntity | None: + if value is None: + return None + if ( + isinstance(value, dict) + and value.keys() <= {"name", "url"} + and isinstance(name := value.get("name"), str) + and _strings(urls := value.get("url", [])) + ): + return {"name": name, "url": urls} + msg = f"`supplier` in {where} must be a table with a string `name` and an optional `url` string array" + raise SettingsError(msg) __all__ = [ - "Settings", "Toml", + "settings_from_pyproject", + "settings_from_table", ] diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index 58c4e38..431760b 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -2,63 +2,38 @@ import hashlib import zipfile -from dataclasses import dataclass from email.parser import Parser from typing import TYPE_CHECKING, Final from packaging.requirements import InvalidRequirement, Requirement from python_sbom._errors import WheelError +from python_sbom._model._inputs import BuildScript, BundledWheel, SourceTree +from python_sbom._wheel._archive import find_dist_info, member_sha256, read_metadata_member +from python_sbom._wheel._record import parse_record if TYPE_CHECKING: from email.message import Message - from pathlib import Path, PurePosixPath + from pathlib import Path - from python_sbom._extension import ContentProvider - from python_sbom._model._types import Property + from python_sbom._model._inputs import SbomRequest - from ._settings import Settings +def read_source_tree(request: SbomRequest) -> SourceTree: + """ + Read what the document takes from the project tree: the copyright line, the bundled wheels and the build scripts. -@dataclass(frozen=True, slots=True, kw_only=True) -class BundledWheel: - # A wheel the project ships as a data file, read from the project tree. - path: str - sha256: str - metadata_name: str - metadata: Message - record: str - vendored: list[tuple[str, Message]] - extra_properties: list[Property] + :param request: what the backend knows about the wheel + :returns: the source tree + :raises WheelError: when a bundled wheel is malformed - -@dataclass(frozen=True, slots=True, kw_only=True) -class BuildScript: - # A build script that shapes the document, so a rebuild has to match it. - path: PurePosixPath - sha256: str - - -@dataclass(frozen=True, slots=True, kw_only=True) -class SourceTree: - # What the document takes from the project tree besides the wheel's metadata. - copyright: str | None - bundled: list[BundledWheel] - build_script: BuildScript | None - - -def read_source_tree( - root: Path, - settings: Settings, - license_files: list[str], - build_script: PurePosixPath | None, - content: ContentProvider, -) -> SourceTree: + """ + root = request.root return SourceTree( copyright=next( ( line - for license_file in license_files + for license_file in request.metadata.get_all("License-File", []) if (path := root / license_file).is_file() for line in path.read_text(encoding="utf-8").splitlines() if line.startswith("Copyright") @@ -66,74 +41,95 @@ def read_source_tree( None, ), bundled=[ - _read_bundled_wheel(root, wheel, content) + read_bundled_wheel(root, wheel) for wheel in sorted( - {path for pattern in settings.bundled_wheels for path in root.glob(pattern)}, + {path for pattern in request.settings.bundled_wheels for path in root.glob(pattern)}, key=lambda path: path.relative_to(root).as_posix(), ) ], # str, so a Windows drive in an absolute path survives the join with the native root - build_script=None - if build_script is None - else BuildScript(path=build_script, sha256=hashlib.sha256((root / str(build_script)).read_bytes()).hexdigest()), + build_scripts=[ + BuildScript(path=script, sha256=hashlib.sha256((root / str(script)).read_bytes()).hexdigest()) + for script in request.build_scripts + ], ) -def _read_bundled_wheel(root: Path, wheel: Path, content: ContentProvider) -> BundledWheel: +def read_bundled_wheel(root: Path, wheel: Path) -> BundledWheel: + """ + Read a wheel a project ships as a data file, checking every member against its ``RECORD``. + + :param root: the project directory + :param wheel: the wheel, under ``root`` + :returns: the wheel's metadata, files and vendored packages + :raises WheelError: when the wheel lacks metadata, its ``RECORD`` disagrees with its members, or it vendors a + package without one exact version + + """ + where = wheel.relative_to(root).as_posix() with zipfile.ZipFile(wheel) as archive: - members: Final[list[str]] = sorted(archive.namelist()) - if ( - metadata_name := next( - (name for name in members if name.count("/") == 1 and name.endswith(".dist-info/METADATA")), None - ) - ) is None: - msg = f"bundled wheel {wheel} has no .dist-info/METADATA" + members: Final[list[str]] = sorted(name for name in archive.namelist() if not name.endswith("/")) + if (dist_info := find_dist_info(members)) is None: + msg = f"bundled wheel {where} has no .dist-info/METADATA" raise WheelError(msg) - metadata = Parser().parsestr(archive.read(metadata_name).decode("utf-8")) - record = archive.read(metadata_name.replace("METADATA", "RECORD")).decode("utf-8") + metadata = Parser().parsestr(read_metadata_member(archive, f"{dist_info}/METADATA", where).decode("utf-8")) + if not (metadata["Name"] and metadata["Version"]): + msg = f"bundled wheel {where}: {dist_info}/METADATA lacks Name or Version" + raise WheelError(msg) + files = parse_record(read_metadata_member(archive, f"{dist_info}/RECORD", where).decode("utf-8"), where) + _check_record(archive, where, {row.path: row.digest for row in files if row.algorithm == "SHA-256"}, members) vendored: Final[list[tuple[str, Message]]] = [ - *_vendor_manifest(wheel, archive, members), + *_vendor_manifest(archive, where, members), *( - (name, Parser().parsestr(archive.read(name).decode("utf-8"))) + (name, Parser().parsestr(read_metadata_member(archive, name, where).decode("utf-8"))) for name in members if "/_vendor/" in name and name.endswith(".dist-info/METADATA") ), ] return BundledWheel( - path=wheel.relative_to(root).as_posix(), + path=where, sha256=hashlib.sha256(wheel.read_bytes()).hexdigest(), - metadata_name=metadata_name, + metadata_name=f"{dist_info}/METADATA", metadata=metadata, - record=record, + files=files, vendored=vendored, - extra_properties=content.bundled_wheel_properties(wheel), ) -def _vendor_manifest(wheel: Path, archive: zipfile.ZipFile, members: list[str]) -> list[tuple[str, Message]]: +def _check_record(archive: zipfile.ZipFile, where: str, digests: dict[str, str], members: list[str]) -> None: + # the SBOM copies these digests, so a RECORD that misstates the wheel would make the SBOM misstate it too + unhashed = {member for member in members if not member.endswith(("/RECORD", "/RECORD.jws", "/RECORD.p7s"))} + if missing := sorted(unhashed - digests.keys()): + msg = f"bundled wheel {where}: RECORD lacks a SHA-256 for {', '.join(missing)}" + raise WheelError(msg) + if extra := sorted(digests.keys() - set(members)): + msg = f"bundled wheel {where}: RECORD lists {', '.join(extra)}, which the wheel does not hold" + raise WheelError(msg) + if changed := [name for name in sorted(digests) if member_sha256(archive, name) != digests[name]]: + msg = f"bundled wheel {where}: the SHA-256 in RECORD differs from the content of {', '.join(changed)}" + raise WheelError(msg) + + +def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) -> list[tuple[str, Message]]: # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info found: Final[list[tuple[str, Message]]] = [] - for name in members: - if not name.endswith("/_vendor/vendor.txt"): - continue - for line in archive.read(name).decode("utf-8").splitlines(): - if not (requirement := line.partition("#")[0].strip()): + for name in (member for member in members if member.endswith("/_vendor/vendor.txt")): + for line in read_metadata_member(archive, name, where).decode("utf-8").splitlines(): + if not (text := line.partition("#")[0].strip()): continue try: - pins = list((parsed := Requirement(requirement)).specifier) + pins = list((requirement := Requirement(text)).specifier) except InvalidRequirement as error: - msg = f"vendored requirement in {wheel.name}:{name} does not parse: {requirement}" + msg = f"bundled wheel {where}: vendored requirement in {name} does not parse: {text}" raise WheelError(msg) from error if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: - msg = f"Unpinned vendored dependency in {wheel.name}:{name}: {requirement}; pin one exact version" + msg = f"bundled wheel {where}: vendored requirement in {name} pins no one exact version: {text}" raise WheelError(msg) - found.append((name, Parser().parsestr(f"Name: {parsed.name}\nVersion: {pins[0].version}\n"))) + found.append((name, Parser().parsestr(f"Name: {requirement.name}\nVersion: {pins[0].version}\n"))) return found __all__ = [ - "BuildScript", - "BundledWheel", - "SourceTree", + "read_bundled_wheel", "read_source_tree", ] diff --git a/src/python_sbom/_errors.py b/src/python_sbom/_errors.py index 5617ce8..7d27a42 100644 --- a/src/python_sbom/_errors.py +++ b/src/python_sbom/_errors.py @@ -17,7 +17,12 @@ class SourceRevisionError(SbomError): """The project is a git checkout, but git cannot read its revision.""" +class MissingExtraError(SbomError, ImportError): + """A feature needs an optional extra that is not installed.""" + + __all__ = [ + "MissingExtraError", "SbomError", "SettingsError", "SourceRevisionError", diff --git a/src/python_sbom/_extension.py b/src/python_sbom/_extension.py deleted file mode 100644 index 880de70..0000000 --- a/src/python_sbom/_extension.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Protocol - -if TYPE_CHECKING: - from pathlib import Path - - from ._model._types import Property - - -class ContentProvider(Protocol): - """Adds project-specific content to the SBOM; the hatchling hook and every backend implement it.""" - - def bundled_wheel_properties(self, wheel: Path) -> list[Property]: - """ - Return extra properties for a wheel the project bundles as a data file. - - The properties follow ``:bundled-wheel`` and precede the ones taken from the wheel's metadata. - - :param wheel: the bundled wheel, an absolute path under the project root - :returns: the properties to add - - """ - ... - - -__all__ = [ - "ContentProvider", -] diff --git a/src/python_sbom/_formats/_spdx.py b/src/python_sbom/_formats/_spdx.py index f4db7a8..7173b5b 100644 --- a/src/python_sbom/_formats/_spdx.py +++ b/src/python_sbom/_formats/_spdx.py @@ -5,11 +5,9 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Final, NotRequired, TypedDict -from python_sbom.version import version +from packaging.utils import canonicalize_name if TYPE_CHECKING: - from collections.abc import Iterator - from python_sbom._model._types import Component, Document, License @@ -54,8 +52,28 @@ class _CreationInfo(TypedDict): creators: list[str] +class _ExtractedLicense(TypedDict): + licenseId: str + extractedText: str + name: str + + class Spdx(TypedDict): - """An SPDX 2.3 JSON document.""" + """ + An SPDX 2.3 JSON document, as :func:`to_spdx` renders it. + + :ivar spdxVersion: ``SPDX-2.3`` + :ivar dataLicense: ``CC0-1.0`` + :ivar SPDXID: ``SPDXRef-DOCUMENT`` + :ivar name: the root component's name and version + :ivar documentNamespace: a URI that ends in the CycloneDX serial number + :ivar comment: which CycloneDX document it was rendered from + :ivar creationInfo: the build time and the creators + :ivar packages: one package per component, files aside + :ivar relationships: containment, dependency and build tool relationships + :ivar hasExtractedLicensingInfos: one entry per ``LicenseRef-`` a package declares + + """ spdxVersion: str dataLicense: str @@ -66,9 +84,10 @@ class Spdx(TypedDict): creationInfo: _CreationInfo packages: list[_Package] relationships: list[_Relationship] + hasExtractedLicensingInfos: NotRequired[list[_ExtractedLicense]] -def to_spdx(bom: Document) -> Spdx: +def to_spdx(bom: Document, tool_version: str) -> Spdx: """ Render a CycloneDX document as SPDX 2.3. @@ -76,46 +95,48 @@ def to_spdx(bom: Document) -> Spdx: over. Per-file hashes stay behind, because SPDX 2.3 requires a SHA-1 for every file and wheel ``RECORD`` files hold SHA-256, and so does the build formulation, which SPDX 2.3 has no field for. - :param bom: a CycloneDX document as this package writes it + :param bom: a CycloneDX document; python-sbom's own documents name their file and repository, others fall back + :param tool_version: the python-sbom version to name as a creator :returns: the SPDX document, whose namespace ends in the CycloneDX serial number so both name the same build """ root: Final[Component] = bom["metadata"]["component"] tools: Final[list[Component]] = bom["metadata"]["tools"]["components"] + prefix = f"{canonicalize_name(root['name'])}:" + recorded = {prop["name"]: prop["value"] for prop in bom["metadata"].get("properties", [])} name: Final[str] = f"{root['name']}-{root['version']}" if "version" in root else root["name"] - namespace: Final[str] = re.sub(r"[-_.]+", "-", root["name"]).lower() - repository: Final[str] = next( - ( - reference["url"] - for reference in root.get("externalReferences", []) - if reference["type"] == "vcs" and "Project-URL" in reference.get("comment", "") - ), - "https://spdx.org/spdxdocs", - ) - components: Final[list[Component]] = list(_walk([root, *bom["components"], *tools])) - supplier: Final[list[str]] = ( - [f"Organization: {bom['metadata']['supplier']['name']}"] if "supplier" in bom["metadata"] else [] - ) - return { + serial = bom["serialNumber"].removeprefix("urn:uuid:") + components: Final[list[Component]] = _walk([root, *bom["components"], *tools]) + packages = [_package(component, prefix) for component in components] + spdx: Final[Spdx] = { "spdxVersion": "SPDX-2.3", "dataLicense": "CC0-1.0", "SPDXID": "SPDXRef-DOCUMENT", "name": name, - "documentNamespace": f"{repository}/sboms/{name}-{bom['serialNumber'].removeprefix('urn:uuid:')}", - "comment": f"Rendered from the CycloneDX SBOM {bom['serialNumber']} ({namespace}.cdx.json), which also carries" - " per-file hashes and the build formulation.", + "documentNamespace": f"{recorded.get(f'{prefix}sbom:repository', 'https://spdx.org/spdxdocs')}/sboms/{name}-{serial}", + "comment": f"Rendered from the CycloneDX SBOM {bom['serialNumber']}" + f" ({recorded.get(f'{prefix}sbom:file', 'CycloneDX JSON')}), which also carries per-file hashes and the build" + " formulation.", "creationInfo": { "created": datetime .fromisoformat(bom["metadata"]["timestamp"]) .astimezone(UTC) .strftime("%Y-%m-%dT%H:%M:%SZ"), "creators": [ - *supplier, - *(f"Tool: {tool['name']}-{tool['version']}" for tool in tools if tool["type"] == "application"), - f"Tool: python-sbom-{version}", + *( + [f"Organization: {supplier['name']}"] + if (supplier := bom["metadata"].get("supplier")) is not None + else [] + ), + *( + f"Tool: {tool['name']}-{tool['version']}" if "version" in tool else f"Tool: {tool['name']}" + for tool in tools + if tool["type"] == "application" + ), + f"Tool: python-sbom-{tool_version}", ], }, - "packages": [_package(component, f"{namespace}:") for component in components], + "packages": packages, "relationships": [ _relationship("SPDXRef-DOCUMENT", "DESCRIBES", _spdx_id(root["bom-ref"])), *( @@ -132,19 +153,32 @@ def to_spdx(bom: Document) -> Spdx: *(_relationship(_spdx_id(tool["bom-ref"]), "BUILD_TOOL_OF", _spdx_id(root["bom-ref"])) for tool in tools), ], } + # SPDX requires the text of every LicenseRef a package declares; the metadata names them, it holds no text + if references := sorted({ + token for package in packages for token in re.findall(r"LicenseRef-[A-Za-z0-9.-]+", package["licenseDeclared"]) + }): + spdx["hasExtractedLicensingInfos"] = [ + {"licenseId": token, "extractedText": f"{token}, as named in package metadata", "name": token} + for token in references + ] + return spdx -def _walk(components: list[Component]) -> Iterator[Component]: - # file components mirror wheel RECORD entries, which carry no SHA-1 and so cannot become SPDX files - for component in components: - if component["type"] != "file": - yield component - yield from _walk(component.get("components", [])) +def _walk(components: list[Component]) -> list[Component]: + # iterative, so a deeply nested hostile document cannot exhaust the stack; file components mirror wheel RECORD + # entries, which carry no SHA-1 and so cannot become SPDX files + found: Final[list[Component]] = [] + pending = list(reversed(components)) + while pending: + if (component := pending.pop())["type"] != "file": + found.append(component) + pending.extend(reversed(component.get("components", []))) + return found def _package(component: Component, property_prefix: str) -> _Package: - # the hook writes either one SPDX expression, or the free-form names from License/classifier metadata plus the SPDX - # ids it concluded from them + # a component carries either one SPDX expression, or the free-form names from License/classifier metadata plus the + # SPDX ids concluded from them licenses: Final[list[License]] = component.get("licenses", []) concluded: Final[list[str]] = [ entry["license"]["id"] for entry in licenses if "license" in entry and "id" in entry["license"] @@ -156,9 +190,7 @@ def _package(component: Component, property_prefix: str) -> _Package: "filesAnalyzed": False, "licenseConcluded": " AND ".join(concluded) or "NOASSERTION", "licenseDeclared": next((entry["expression"] for entry in licenses if "expression" in entry), "NOASSERTION"), - "primaryPackagePurpose": {"application": "APPLICATION", "library": "LIBRARY", "platform": "OTHER"}[ - component["type"] - ], + "primaryPackagePurpose": {"application": "APPLICATION", "library": "LIBRARY"}.get(component["type"], "OTHER"), } if version := component.get("version"): package["versionInfo"] = version diff --git a/src/python_sbom/_generate/__init__.py b/src/python_sbom/_generate/__init__.py new file mode 100644 index 0000000..8f1ddab --- /dev/null +++ b/src/python_sbom/_generate/__init__.py @@ -0,0 +1 @@ +"""The build pipeline: backends that adapt a build system, the content seam, and assembling the SBOM.""" diff --git a/src/python_sbom/_generate/_backend.py b/src/python_sbom/_generate/_backend.py new file mode 100644 index 0000000..4879e5b --- /dev/null +++ b/src/python_sbom/_generate/_backend.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import zipfile +from abc import ABC, abstractmethod +from email.parser import Parser +from typing import TYPE_CHECKING + +from python_sbom._errors import WheelError +from python_sbom._model._inputs import SbomRequest +from python_sbom._wheel._archive import find_dist_info, read_metadata_member + +if TYPE_CHECKING: + from pathlib import Path, PurePosixPath + + from python_sbom._model._inputs import Settings + + +class Backend(ABC): + """Adapts a build system to python-sbom: it says what it knows about the wheel it builds or rewrites.""" + + @abstractmethod + def request(self) -> SbomRequest: + """ + Describe the wheel. + + :returns: the project directory, the wheel's metadata and file name, the settings and the build scripts + + """ + + +class MetadataBackend(Backend): + """ + A backend for build systems that hand over the core metadata before the wheel exists, such as a build hook. + + :param root: the project directory + :param metadata: the wheel's ``METADATA`` text + :param wheel_name: the wheel's file name + :param settings: the project's options + :param build_scripts: scripts, relative to ``root``, that shape the document + + """ + + def __init__( + self, + *, + root: Path, + metadata: str, + wheel_name: str, + settings: Settings, + build_scripts: tuple[PurePosixPath, ...], + ) -> None: + self._request = SbomRequest( + root=root, + metadata=Parser().parsestr(metadata), + wheel_name=wheel_name, + settings=settings, + build_scripts=build_scripts, + ) + + def request(self) -> SbomRequest: + """ + Return the request built from the metadata. + + :returns: the request + + """ + return self._request + + +class BuiltWheel(Backend): + """ + A wheel some build backend already built, such as meson-python, and the project it came from. + + :param wheel: the built wheel file + :param root: the project directory the wheel was built from + :param settings: the project's options + :param build_scripts: scripts, relative to ``root``, that shape the document + + """ + + def __init__( + self, *, wheel: Path, root: Path, settings: Settings, build_scripts: tuple[PurePosixPath, ...] + ) -> None: + self._wheel = wheel + self._root = root + self._settings = settings + self._build_scripts = build_scripts + + def request(self) -> SbomRequest: + """ + Read the wheel's core metadata. + + :returns: the request + :raises WheelError: when the archive is not a wheel + + """ + with zipfile.ZipFile(self._wheel) as archive: + if (dist_info := find_dist_info(archive.namelist())) is None: + msg = f"{self._wheel} has no .dist-info/METADATA, so it is not a wheel" + raise WheelError(msg) + metadata = read_metadata_member(archive, f"{dist_info}/METADATA", str(self._wheel)).decode("utf-8") + return SbomRequest( + root=self._root, + metadata=Parser().parsestr(metadata), + wheel_name=self._wheel.name, + settings=self._settings, + build_scripts=self._build_scripts, + ) + + +__all__ = [ + "Backend", + "BuiltWheel", + "MetadataBackend", +] diff --git a/src/python_sbom/_generate/_content.py b/src/python_sbom/_generate/_content.py new file mode 100644 index 0000000..f02ddd6 --- /dev/null +++ b/src/python_sbom/_generate/_content.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import importlib +import sys +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +from python_sbom._errors import SettingsError +from python_sbom._model._inputs import BundledWheelInfo, Contributions + +if TYPE_CHECKING: + from python_sbom._model._inputs import BundledWheel, SbomRequest + + +def load_content_provider(spec: str, root: Path) -> tuple[ContentProvider, PurePosixPath | None]: + """ + Import the content provider a ``content-provider`` option names. + + :param spec: ``module:attribute``, importable from ``root``; the attribute is a :class:`ContentProvider` subclass or + instance + :param root: the project directory, searched first for the module + :returns: the provider, and the module's path relative to ``root`` when it lives there, which the SBOM lists among + the build tools since it shapes the document + :raises SettingsError: when the spec is malformed, the import fails, or the attribute is no content provider + + """ + module_name, _, attribute = spec.partition(":") + if not (module_name and attribute): + msg = f"content provider {spec!r} must be `module:attribute`" + raise SettingsError(msg) + sys.path.insert(0, str(root)) + try: + module = importlib.import_module(module_name) + except ImportError as error: + msg = f"content provider {spec!r} does not import from {root}: {error}" + raise SettingsError(msg) from error + finally: + sys.path.remove(str(root)) + provider = getattr(module, attribute, None) + if isinstance(provider, type) and issubclass(provider, ContentProvider): + provider = provider() + if not isinstance(provider, ContentProvider): + msg = f"content provider {spec!r} is not a ContentProvider subclass or instance" + raise SettingsError(msg) + source = Path(module.__file__ or "").resolve() + script = ( + PurePosixPath(source.relative_to(root.resolve()).as_posix()) if source.is_relative_to(root.resolve()) else None + ) + return provider, script + + +class ContentProvider: + """ + Adds project-specific content to an SBOM; subclass it and override what the project needs. + + Name a subclass in the ``content-provider`` option, or pass an instance to :func:`build_sbom`. + + """ + + def bundled_wheel(self, wheel: BundledWheel) -> BundledWheelInfo: # ruff: ignore[unused-method-argument, no-self-use] # the no-op default + """ + Describe what the project does with a wheel it bundles. + + :param wheel: the bundled wheel + :returns: its extra properties and supplier; none by default + + """ + return BundledWheelInfo(properties=[], supplier=None) + + def contributions(self, request: SbomRequest) -> Contributions: # ruff: ignore[unused-method-argument, no-self-use] # the no-op default + """ + Add components, dependencies, root properties or tools, before references and the serial are derived. + + :param request: what the backend knows about the wheel + :returns: the contributions; none by default + + """ + return Contributions(components=[], dependencies=[], root_properties=[], tools=[]) + + +__all__ = [ + "ContentProvider", + "load_content_provider", +] diff --git a/src/python_sbom/_generate/_pipeline.py b/src/python_sbom/_generate/_pipeline.py new file mode 100644 index 0000000..7395583 --- /dev/null +++ b/src/python_sbom/_generate/_pipeline.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from python_sbom._formats._json import dumps +from python_sbom._model._bom import describe +from python_sbom._model._components import sbom_file_name + +if TYPE_CHECKING: + from python_sbom._model._inputs import BuildFacts, SbomRequest, SourceTree + from python_sbom._model._types import Document + + from ._content import ContentProvider + + +def build_sbom(request: SbomRequest, source: SourceTree, facts: BuildFacts, content: ContentProvider) -> Sbom: + """ + Build the SBOM of a wheel from what the edge gathered. + + :param request: what the backend knows about the wheel, from :meth:`Backend.request` + :param source: the project tree, from :func:`read_source_tree` + :param facts: the build facts, from :func:`collect_facts` + :param content: what the project adds + :returns: the SBOM + + """ + return Sbom( + file_name=sbom_file_name(request.metadata["Name"]), + document=describe( + request, + source, + facts, + {wheel.path: content.bundled_wheel(wheel) for wheel in source.bundled}, + content.contributions(request), + ), + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Sbom: + """ + An SBOM ready to write. + + :param file_name: its file name under ``.dist-info/sboms/`` + :param document: the CycloneDX document + + """ + + file_name: str + document: Document + + @property + def content(self) -> bytes: + """The document as python-sbom writes it: indented JSON with a final newline.""" + return dumps(self.document) + + +__all__ = [ + "Sbom", + "build_sbom", +] diff --git a/src/python_sbom/_model/__init__.py b/src/python_sbom/_model/__init__.py index d137f14..40aeade 100644 --- a/src/python_sbom/_model/__init__.py +++ b/src/python_sbom/_model/__init__.py @@ -1 +1 @@ -"""The CycloneDX document model: its types and the pure functions that build it.""" +"""The CycloneDX document model: its types, its inputs and the pure functions that build it.""" diff --git a/src/python_sbom/_model/_bom.py b/src/python_sbom/_model/_bom.py new file mode 100644 index 0000000..a2840f5 --- /dev/null +++ b/src/python_sbom/_model/_bom.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from pathlib import PurePosixPath +from typing import TYPE_CHECKING, Final +from urllib.parse import urlsplit + +from packaging.requirements import Requirement + +from ._components import ( + bundled_wheel_component, + component_from_metadata, + purl, + pypi_url, + sbom_file_name, + script_tool, +) +from ._document import document, formulation, project_identity, timestamp +from ._markers import applies_without_extras +from ._tools import tool_components +from ._types import Dependency + +if TYPE_CHECKING: + from collections.abc import Mapping + + from ._document import ProjectIdentity + from ._inputs import BuildFacts, BundledWheelInfo, Contributions, SbomRequest, Settings, SourceTree + from ._types import Component, Document, ExternalReference, Metadata, Property + + +def describe( + request: SbomRequest, + source: SourceTree, + facts: BuildFacts, + bundled_info: Mapping[str, BundledWheelInfo], + contributions: Contributions, +) -> Document: + project = project_identity(request.metadata, request.settings) + root = _root_component(request, source, facts, project, contributions) + bundled = [bundled_wheel_component(wheel, project.namespace, bundled_info[wheel.path]) for wheel in source.bundled] + components = [ + *bundled, + # hatchling repeats a Requires-Dist when a hook re-adds a dependency, and each bom-ref must be unique + *( + _declared_dependency(text, requirement, project.namespace) + for text in dict.fromkeys(request.metadata.get_all("Requires-Dist", [])) + if applies_without_extras(requirement := Requirement(text)) + ), + *contributions.components, + ] + environment, tool_dependencies = tool_components(facts) + tools = [ + *( + script_tool( + script.path, + script.sha256, + request.metadata["Version"], + _file_link(project, request.settings, str(script.path)), + ) + for script in source.build_scripts + ), + *environment, + *contributions.tools, + ] + return document( + { + "metadata": _metadata(root, tools, request, project, facts), + "components": components, + "dependencies": [ + {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in components]}, + *( + Dependency(ref=component["bom-ref"], dependsOn=vendored) + for component in bundled + if ( + vendored := [ + child["bom-ref"] for child in component.get("components", []) if child["type"] == "library" + ] + ) + ), + *tool_dependencies, + *contributions.dependencies, + ], + # transitive runtime dependencies are unknown until install time + "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], + "formulation": [formulation("wheel-build", "build the wheel and this SBOM", tools, root["bom-ref"], facts)], + }, + project.serial_namespace, + ) + + +def _root_component( + request: SbomRequest, source: SourceTree, facts: BuildFacts, project: ProjectIdentity, contributions: Contributions +) -> Component: + metadata, settings = request.metadata, request.settings + name, version = metadata["Name"], metadata["Version"] + root = component_from_metadata(metadata) + root["type"] = "application" + if settings.supplier is not None: + root["supplier"] = settings.supplier + if source.copyright is not None: + root["copyright"] = source.copyright + references: Final[list[ExternalReference]] = [ + *root["externalReferences"], + {"type": "distribution", "url": pypi_url(name, version)}, + {"type": "attestation", "url": f"https://pypi.org/integrity/{name}/{version}/{request.wheel_name}/provenance"}, + ] + if project.repository and urlsplit(project.repository).netloc == "github.com": + references.extend([ + {"type": "security-contact", "url": f"{project.repository}/security/policy"}, + {"type": "advisories", "url": f"{project.repository}/security/advisories"}, + ]) + if (license_files := metadata.get_all("License-File", [])) and ( + link := _file_link(project, settings, license_files[0]) + ): + references.append({"type": "license", "url": link}) + properties: Final[list[Property]] = [*root["properties"], *contributions.root_properties] + if (revision := facts.revision) is not None: + if project.repository: + comment = "source revision, with uncommitted changes" if revision.dirty else "exact source revision" + references.append({ + "type": "vcs", + "url": f"{project.repository}/tree/{revision.commit}", + "comment": comment, + }) + properties.append({"name": f"{project.namespace}:vcs-commit", "value": revision.commit}) + if revision.dirty: + properties.append({"name": f"{project.namespace}:vcs-dirty", "value": "true"}) + root["externalReferences"] = references + root["properties"] = properties + return root + + +def _declared_dependency(text: str, requirement: Requirement, namespace: str) -> Component: + component: Final[Component] = { + "type": "library", + # the same distribution can be declared more than once with different markers, so the purl is not unique + "bom-ref": f"requires-dist:{text}", + "name": requirement.name, + "purl": purl(requirement.name), + "externalReferences": [{"type": "distribution", "url": pypi_url(requirement.name)}], + "properties": [{"name": f"{namespace}:requires-dist", "value": text}], + } + if requirement.marker is not None: + component["properties"].append({"name": f"{namespace}:environment-marker", "value": str(requirement.marker)}) + return component + + +def _file_link(project: ProjectIdentity, settings: Settings, path: str) -> str | None: + return None if project.repository is None else f"{project.repository}/blob/{settings.default_branch}/{path}" + + +def _metadata( + root: Component, tools: list[Component], request: SbomRequest, project: ProjectIdentity, facts: BuildFacts +) -> Metadata: + settings = request.settings + metadata: Final[Metadata] = { + "timestamp": timestamp(facts), + "lifecycles": [{"phase": "build"}], + "tools": {"components": tools}, + "component": root, + } + if settings.supplier is not None: + metadata["manufacturer"] = metadata["supplier"] = settings.supplier + if "authors" in root: + metadata["authors"] = root["authors"] + if "licenses" in root: + metadata["licenses"] = root["licenses"] + properties: Final[list[Property]] = [ + {"name": f"{project.namespace}:sbom:scope", "value": _scope(settings)}, + {"name": f"{project.namespace}:sbom:file", "value": sbom_file_name(request.metadata["Name"])}, + ] + if project.repository is not None: + properties.append({"name": f"{project.namespace}:sbom:repository", "value": project.repository}) + metadata["properties"] = properties + return metadata + + +def _scope(settings: Settings) -> str: + declared = ( + "the direct runtime dependencies declared in the wheel's core metadata, which carry no version because the" + " installer resolves them" + ) + if directories := " and ".join(sorted({str(PurePosixPath(pattern).parent) for pattern in settings.bundled_wheels})): + return f"Components are the wheels bundled as data files under {directories} and {declared}." + return f"Components are {declared}." + + +__all__ = [ + "describe", +] diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index d971396..f72bef3 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -1,12 +1,11 @@ from __future__ import annotations -import base64 import hashlib -import json import re -import uuid from email.utils import getaddresses +from itertools import starmap from typing import TYPE_CHECKING, Final +from urllib.parse import urlsplit from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression from packaging.utils import canonicalize_name @@ -14,9 +13,10 @@ if TYPE_CHECKING: from email.message import Message from importlib.metadata import PackageMetadata + from pathlib import PurePosixPath - from ._types import Body, Component, Contact, Document, ExternalReference, License - + from ._inputs import BundledWheel, BundledWheelInfo, RecordRow + from ._types import Component, Contact, ExternalReference, Identity, License # the label normalization and mapping cyclonedx-py applies to Project-URL entries, plus "source", so the same metadata # yields the same reference types whichever generator produced the document @@ -40,54 +40,88 @@ } -def document(body: Body, namespace: uuid.UUID) -> Document: +def bundled_wheel_component(wheel: BundledWheel, namespace: str, info: BundledWheelInfo) -> Component: """ - Add the CycloneDX 1.6 header to a document body. + Describe a wheel a project ships as a data file. - :param body: the metadata, components, dependencies and formulation - :param namespace: UUID namespace of the serial number, one per project - :returns: the document; its serial number derives from the body, so identical bodies share one + :param wheel: the wheel, as :func:`~python_sbom.read_bundled_wheel` read it + :param namespace: the project's normalized name, which prefixes the properties this package writes + :param info: what the project adds to the component + :returns: the component, with a file component per ``RECORD`` row and a component per vendored package """ - return { - "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", - "bomFormat": "CycloneDX", - "specVersion": "1.6", - "serialNumber": f"urn:uuid:{uuid.uuid5(namespace, json.dumps(body, sort_keys=True))}", - "version": 1, - **body, + component = component_from_metadata(wheel.metadata) + # the purl names the package; the path tells apart copies of one release bundled in two places + component["bom-ref"] = f"bundled:{wheel.path}" + if info.supplier is not None: + component["supplier"] = info.supplier + component["hashes"] = [{"alg": "SHA-256", "content": wheel.sha256}] + component["externalReferences"] = [ + *component["externalReferences"], + {"type": "distribution", "url": pypi_url(wheel.metadata["Name"], wheel.metadata["Version"])}, + ] + component["properties"] = [ + {"name": f"{namespace}:bundled-wheel", "value": wheel.path}, + *info.properties, + *component["properties"], + ] + component["evidence"] = { + "identity": [ + _identity("purl", "manifest-analysis", wheel.metadata_name), + _identity("hash", "hash-comparison", wheel.sha256), + ] } + vendored: Final[dict[str, Component]] = {} + for source, metadata in wheel.vendored: + child = component_from_metadata(metadata) + child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['bom-ref']}" + child["properties"] = [*child["properties"], {"name": f"{namespace}:vendored-manifest", "value": source}] + child["evidence"] = {"identity": [_identity("purl", "manifest-analysis", source)]} + # a vendored .dist-info comes after pip's manifest line naming the same package, so its metadata wins + vendored[purl(child["name"], child["version"])] = child + component["components"] = [ + *(file_component(component["bom-ref"], row) for row in wheel.files), + *(vendored[key] for key in sorted(vendored)), + ] + return component -def component_from_metadata(metadata: PackageMetadata | Message, component_type: str) -> Component: +def component_from_metadata(metadata: PackageMetadata | Message) -> Component: """ - Describe a distribution from its core metadata. + Describe a distribution from its core metadata, as a ``library`` component. :param metadata: the distribution's ``METADATA`` - :param component_type: the CycloneDX component type, such as ``library`` - :returns: the component, with its purl as ``bom-ref``, licenses, Project-URL references and metadata properties + :returns: the component, with its purl as ``bom-ref``, authors, licenses, project URLs and metadata properties """ name, version = metadata["Name"], metadata["Version"] - package = purl(name, version) - component: Final[Component] = {"type": component_type, "bom-ref": package, "name": name, "version": version} - component["purl"] = package + component: Final[Component] = {"type": "library", "bom-ref": purl(name, version), "name": name, "version": version} + component["purl"] = component["bom-ref"] if summary := metadata.get("Summary"): component["description"] = summary - if authors := contacts( - metadata.get_all("Author", []) + metadata.get_all("Maintainer", []), - metadata.get_all("Author-email", []) + metadata.get_all("Maintainer-email", []), + # maintainers are who users reach today, so they win over the original authors when both are named + if people := next( + ( + found + for names, addresses in (("Maintainer", "Maintainer-email"), ("Author", "Author-email")) + if ( + found := [ + contact + for contact in ( + *(_contact(name, "") for name in metadata.get_all(names, [])), + *starmap(_contact, getaddresses(metadata.get_all(addresses, []))), + ) + if contact + ] + ) + ), + None, ): - component["authors"] = authors - if licenses := _licenses(metadata): - component["licenses"] = licenses - component["externalReferences"] = [ - external_reference(label.strip(), url.strip()) - for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) - ] - if home_page := metadata.get("Home-page"): - component["externalReferences"].insert(0, {"type": "website", "url": home_page}) - # core metadata headers copied verbatim onto the component + component["authors"] = people + if declared := _licenses(metadata): + component["licenses"] = declared + component["externalReferences"] = _project_references(metadata) + # core metadata headers copied verbatim, so a reader sees what the distribution declared component["properties"] = [ {"name": f"python:{header.lower()}", "value": value} for header in ("Requires-Python", "Requires-Dist", "Provides-Extra", "Classifier", "Keywords") @@ -96,23 +130,21 @@ def component_from_metadata(metadata: PackageMetadata | Message, component_type: return component -def purl(name: str, version: str | None = None) -> str: - return f"pkg:pypi/{canonicalize_name(name)}@{version}" if version else f"pkg:pypi/{canonicalize_name(name)}" - - -def contacts(names: list[str], addresses: list[str]) -> list[Contact]: - found: Final[list[Contact]] = [{"name": name} for name in names] - for name, email in getaddresses(addresses): - contact: Contact = {} - if name: - contact["name"] = name - if email: - contact["email"] = email - found.append(contact) - return found +def _contact(name: str, email: str) -> Contact: + if email and not re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]+", email): + # an address the schema rejects as an email, such as "jane at example.com", is kept as the name + return {"name": " ".join(part for part in (name, email) if part)} + contact: Final[Contact] = {} + if name: + contact["name"] = name + if email: + contact["email"] = email + return contact def _licenses(metadata: PackageMetadata | Message) -> list[License]: + # a License-Expression is taken as declared; otherwise each License field and License classifier name is + # declared, and the SPDX ids they map to are added as concluded if expression := metadata.get("License-Expression"): return [{"expression": expression, "acknowledgement": "declared"}] names = [ @@ -120,15 +152,15 @@ def _licenses(metadata: PackageMetadata | Message) -> list[License]: ] if (declared := metadata.get("License")) and "\n" not in declared: names.append(declared) - licenses: Final[list[License]] = [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] - # a license policy judges SPDX ids only, so a component keeps its bare names when any one of them does not resolve, - # and the policy reports it instead of judging a partial set - identifiers: Final[list[str]] = [identifier for name in names if (identifier := _spdx_license(name)) is not None] + found: Final[list[License]] = [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] + # a license policy judges SPDX ids only, so the names stay bare when any one of them does not resolve, and the + # policy reports that instead of judging a partial set + identifiers = [identifier for name in names if (identifier := _spdx_license(name)) is not None] if len(identifiers) == len(names): - licenses.extend( + found.extend( {"license": {"id": identifier, "acknowledgement": "concluded"}} for identifier in dict.fromkeys(identifiers) ) - return licenses + return found def _spdx_license(name: str) -> str | None: @@ -151,41 +183,112 @@ def _spdx_license(name: str) -> str | None: return canonical if re.fullmatch(r"[A-Za-z0-9.-]+", canonical) and not canonical.startswith("LicenseRef-") else None -def external_reference(label: str, url: str) -> ExternalReference: - reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") - return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} +def _project_references(metadata: PackageMetadata | Message) -> list[ExternalReference]: + # a home page that is not an http(s) URL, such as UNKNOWN, is left out + references: Final[list[ExternalReference]] = [] + if (home_page := metadata.get("Home-page")) and urlsplit(home_page).scheme in {"http", "https"}: + references.append({"type": "website", "url": home_page}) + references.extend( + {"type": reference_type(label), "url": url, "comment": f"Project-URL: {label}"} + for label, url in project_urls(metadata) + ) + return references + + +def reference_type(label: str) -> str: + return _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") + + +def project_urls(metadata: PackageMetadata | Message) -> list[tuple[str, str]]: + return [ + (label.strip(), url.strip()) + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", []) if "," in entry) + ] + + +def pypi_url(name: str, version: str | None = None) -> str: + return f"https://pypi.org/project/{name}/{version}/" if version else f"https://pypi.org/project/{name}/" -def file_component(parent_ref: str, path: str, digest: str, size: str) -> Component: - # RECORD stores "=", CycloneDX wants lowercase hex - algorithm, _, encoded = digest.partition("=") - algorithms: Final[dict[str, str]] = { - "md5": "MD5", - "sha1": "SHA-1", - "sha256": "SHA-256", - "sha384": "SHA-384", - "sha512": "SHA-512", +def _identity(field: str, technique: str, value: str) -> Identity: + return {"field": field, "confidence": 1, "methods": [{"technique": technique, "confidence": 1, "value": value}]} + + +def purl(name: str, version: str | None = None) -> str: + return f"pkg:pypi/{canonicalize_name(name)}@{version}" if version else f"pkg:pypi/{canonicalize_name(name)}" + + +def file_component(parent_ref: str, row: RecordRow) -> Component: + return { + "type": "file", + "bom-ref": f"{parent_ref}#{row.path}", + "name": row.path, + "hashes": [{"alg": row.algorithm, "content": row.digest}], + "properties": [{"name": "size", "value": row.size}], } - raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + + +def file_component_from_bytes(parent_ref: str, name: str, content: bytes) -> Component: + """ + Describe one file from its bytes, such as a member of a zipapp. + + :param parent_ref: the ``bom-ref`` of the component that holds the file + :param name: the file's path + :param content: the file's bytes + :returns: a ``file`` component with the file's SHA-256 and size + + """ return { "type": "file", - "bom-ref": f"{parent_ref}#{path}", - "name": path, - "hashes": [{"alg": algorithms[algorithm], "content": raw.hex()}], - "properties": [{"name": "size", "value": size}], + "bom-ref": f"{parent_ref}#{name}", + "name": name, + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(content).hexdigest()}], + "properties": [{"name": "size", "value": str(len(content))}], } -def record_digest(content: bytes) -> str: - return f"sha256={base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b'=').decode()}" +def script_tool(path: PurePosixPath, sha256: str, version: str, link: str | None) -> Component: + """ + Describe a build script as a build tool, since a rebuild has to match the script that shaped the document. + + :param path: the script's path relative to the project root + :param sha256: the SHA-256 of the script + :param version: the version of the artifact the script builds + :param link: a URL of the script in the repository, or ``None`` + :returns: an ``application`` component + + """ + tool: Final[Component] = { + "type": "application", + "bom-ref": tool_ref(str(path)), + "name": path.name, + "version": version, + "hashes": [{"alg": "SHA-256", "content": sha256}], + } + if link is not None: + tool["externalReferences"] = [{"type": "vcs", "url": link}] + return tool + + +def tool_ref(ref: str) -> str: + # prefixed because the same distribution can be both a build tool and a component of the artifact + return f"tool:{ref}" + + +def sbom_file_name(name: str) -> str: + return f"{canonicalize_name(name)}.cdx.json" __all__ = [ + "bundled_wheel_component", "component_from_metadata", - "contacts", - "document", - "external_reference", "file_component", + "file_component_from_bytes", + "project_urls", "purl", - "record_digest", + "pypi_url", + "reference_type", + "sbom_file_name", + "script_tool", + "tool_ref", ] diff --git a/src/python_sbom/_model/_document.py b/src/python_sbom/_model/_document.py new file mode 100644 index 0000000..0de9c39 --- /dev/null +++ b/src/python_sbom/_model/_document.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import json +import uuid +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from packaging.utils import canonicalize_name + +from ._components import project_urls, reference_type + +if TYPE_CHECKING: + from email.message import Message + + from ._inputs import BuildFacts, Settings + from ._types import Body, Component, Document, Formula + + +def project_identity(metadata: Message, settings: Settings) -> ProjectIdentity: + """ + Name a project the way its documents do, so other documents about it, such as a zipapp's, agree. + + :param metadata: the project's core metadata + :param settings: the project's options, whose ``repository`` wins over the project URLs + :returns: the identity + + """ + namespace = canonicalize_name(metadata["Name"]) + repository = settings.repository or next( + (url.rstrip("/") for label, url in project_urls(metadata) if reference_type(label) == "vcs"), None + ) + base = repository or f"https://pypi.org/project/{namespace}" + return ProjectIdentity( + namespace=namespace, + repository=repository, + serial_namespace=uuid.uuid5(uuid.NAMESPACE_URL, f"{base}/sboms"), + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ProjectIdentity: + """ + How documents name a project. + + :param namespace: the normalized project name, which prefixes the properties this package writes + :param repository: the repository URL, or ``None`` + :param serial_namespace: the UUID namespace of the serial numbers of the project's documents + + """ + + namespace: str + repository: str | None + serial_namespace: uuid.UUID + + +def document(body: Body, namespace: uuid.UUID) -> Document: + """ + Add the CycloneDX 1.6 header to a document body. + + :param body: the metadata, components, dependencies and formulation + :param namespace: the UUID namespace of the serial number, see :func:`project_identity` + :returns: the document; its serial number derives from the body, so identical bodies share one + + """ + return { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": f"urn:uuid:{uuid.uuid5(namespace, json.dumps(body, sort_keys=True))}", + "version": 1, + **body, + } + + +def formulation(uid: str, name: str, tools: list[Component], output_ref: str, facts: BuildFacts) -> Formula: + """ + Record how an artifact was built: one workflow with its tools, its output and the build time it read. + + :param uid: the workflow's identifier, such as ``wheel-build`` + :param name: what the workflow does + :param tools: the build tools it used + :param output_ref: the ``bom-ref`` of the artifact it produced + :param facts: the build facts; ``SOURCE_DATE_EPOCH`` is listed as an input only when it set the build time + :returns: the formula + + """ + workflow: Formula = { + "bom-ref": f"formula:{uid}", + "workflows": [ + { + "bom-ref": f"workflow:{uid}", + "uid": uid, + "name": name, + "taskTypes": ["build"], + "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], + "outputs": [{"type": "artifact", "resource": {"ref": output_ref}}], + } + ], + } + if facts.epoch_from_environment: + workflow["workflows"][0]["inputs"] = [ + {"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": str(facts.epoch)}]} + ] + return workflow + + +def timestamp(facts: BuildFacts) -> str: + return datetime.fromtimestamp(facts.epoch, tz=UTC).isoformat() + + +__all__ = [ + "ProjectIdentity", + "document", + "formulation", + "project_identity", + "timestamp", +] diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py new file mode 100644 index 0000000..b47190d --- /dev/null +++ b/src/python_sbom/_model/_inputs.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from email.message import Message + from pathlib import Path, PurePosixPath + + from ._types import Component, Dependency, OrganizationalEntity, Property + + +@dataclass(frozen=True, slots=True, kw_only=True) +class RecordRow: + """ + One file of a wheel, as its ``RECORD`` lists it. + + :param path: the file's path inside the wheel + :param algorithm: the hash algorithm as CycloneDX names it, such as ``SHA-256`` + :param digest: the file's digest as lowercase hex + :param size: the file's size in bytes, as ``RECORD`` writes it + + """ + + path: str + algorithm: str + digest: str + size: str + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Interpreter: + """ + The Python that runs the build. + + :param name: the implementation name, such as ``cpython`` + :param version: the full version, such as ``3.15.0`` + :param implementation: the implementation as :func:`platform.python_implementation` names it, such as ``CPython`` + + """ + + name: str + version: str + implementation: str + + +@dataclass(frozen=True, slots=True, kw_only=True) +class InstalledDistribution: + """ + One distribution of the build environment. + + :param metadata: its core metadata + :param files: its files, for a pure-Python distribution; empty for a platform one, whose files differ per machine + :param depends_on: the normalized names of the installed distributions it requires in this environment + + """ + + metadata: Message + files: list[RecordRow] + depends_on: list[str] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Revision: + """ + The source revision a build comes from. + + :param commit: the commit identifier + :param dirty: whether tracked files differed from the commit when the build ran + + """ + + commit: str + dirty: bool + + +@dataclass(frozen=True, slots=True, kw_only=True) +class BuildFacts: + """ + What the environment says about the build, collected once per build at the edge. + + :param epoch: the build time as a Unix timestamp + :param epoch_from_environment: whether ``SOURCE_DATE_EPOCH`` set the build time + :param revision: the source revision, or ``None`` when it is unknown + :param interpreter: the Python that runs the build + :param distributions: every distribution of the build environment + + """ + + epoch: int + epoch_from_environment: bool + revision: Revision | None + interpreter: Interpreter + distributions: list[InstalledDistribution] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Settings: + """ + The options a project sets, in a hatchling hook's table or in ``[tool.python-sbom]``. + + :param supplier: the organization that supplies the project, or ``None`` to name none + :param repository: the repository URL, or ``None`` to take the first ``Source``, ``Repository`` or ``GitHub`` + project URL + :param default_branch: the branch in links to files in the repository + :param bundled_wheels: glob patterns, relative to the project root, of wheels the project ships as data files + :param content_provider: ``module:attribute`` of a content provider importable from the project root, or ``None`` + + """ + + supplier: OrganizationalEntity | None + repository: str | None + default_branch: str + bundled_wheels: tuple[str, ...] + content_provider: str | None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SbomRequest: + """ + What a backend knows about the wheel it builds or rewrites. + + :param root: the project directory + :param metadata: the wheel's core metadata + :param wheel_name: the wheel's file name, for its attestation link + :param settings: the project's options + :param build_scripts: scripts, relative to ``root``, that shape the document, listed among the build tools + + """ + + root: Path + metadata: Message + wheel_name: str + settings: Settings + build_scripts: tuple[PurePosixPath, ...] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class BundledWheel: + """ + A wheel the project ships as a data file. + + :param path: its path relative to the project root + :param sha256: the SHA-256 of its bytes, as lowercase hex + :param metadata_name: the path of its ``METADATA`` inside it + :param metadata: its core metadata + :param files: its files, from its ``RECORD`` + :param vendored: the packages it vendors, each with the file that names it + + """ + + path: str + sha256: str + metadata_name: str + metadata: Message + files: list[RecordRow] + vendored: list[tuple[str, Message]] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class BuildScript: + """ + A build script that shapes the document, so a rebuild has to match it. + + :param path: its path relative to the project root + :param sha256: the SHA-256 of its bytes, as lowercase hex + + """ + + path: PurePosixPath + sha256: str + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SourceTree: + """ + What the document takes from the project tree besides the wheel's metadata. + + :param copyright: the first ``Copyright`` line of the license files, or ``None`` + :param bundled: the wheels the project ships as data files + :param build_scripts: the scripts that shape the document + + """ + + copyright: str | None + bundled: list[BundledWheel] + build_scripts: list[BuildScript] + + +@dataclass(frozen=True, slots=True, kw_only=True) +class BundledWheelInfo: + """ + What a project adds to the component of one bundled wheel. + + :param properties: properties after ``:bundled-wheel``, before those from the wheel's metadata + :param supplier: the wheel's supplier, or ``None`` to name none; only the project decides it, never the wheel + + """ + + properties: list[Property] + supplier: OrganizationalEntity | None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Contributions: + """ + What a project adds to the document beyond its bundled wheels, merged before references and the serial are derived. + + :param components: extra top-level components, such as vendored C libraries; the root depends on each + :param dependencies: extra dependency entries, such as the dependencies of those components + :param root_properties: extra properties of the root component + :param tools: extra build tools + + """ + + components: list[Component] + dependencies: list[Dependency] + root_properties: list[Property] + tools: list[Component] + + +__all__ = [ + "BuildFacts", + "BuildScript", + "BundledWheel", + "BundledWheelInfo", + "Contributions", + "InstalledDistribution", + "Interpreter", + "RecordRow", + "Revision", + "SbomRequest", + "Settings", + "SourceTree", +] diff --git a/src/python_sbom/_model/_markers.py b/src/python_sbom/_model/_markers.py new file mode 100644 index 0000000..fd09dde --- /dev/null +++ b/src/python_sbom/_model/_markers.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from packaging._parser import Op, Value, Variable # ruff: ignore[import-private-name] # packaging exposes the parsed marker tree only privately +from packaging.markers import Marker + +if TYPE_CHECKING: + from packaging._parser import MarkerAtom, MarkerList # see above + from packaging.requirements import Requirement + + +def applies_without_extras(requirement: Requirement) -> bool: + # the conditions on extra are decided with no extra requested and every other condition stays unknown, so the + # requirement applies unless its extra conditions alone rule it out + if requirement.marker is None: + return True + return _extra_verdict(requirement.marker._markers) is not False # ruff: ignore[private-member-access] # the tree, see the import above + + +def _extra_verdict(markers: MarkerList | MarkerAtom) -> bool | None: + # three-valued, with "and" binding tighter than "or", as packaging's own evaluation groups them + groups: list[list[bool | None]] = [[]] + for item in markers: + if item == "or": + groups.append([]) + elif isinstance(item, list): + groups[-1].append(_extra_verdict(item)) + elif isinstance(item, tuple): + match item: + case Variable(value="extra"), Op(value=operator), Value(value=value): + groups[-1].append(Marker(f"extra {operator} {value!r}").evaluate({"extra": ""})) + case Value(value=value), Op(value=operator), Variable(value="extra"): + groups[-1].append(Marker(f"{value!r} {operator} extra").evaluate({"extra": ""})) + case _: + groups[-1].append(None) + verdicts = [False if False in group else (None if None in group else True) for group in groups] + return True if True in verdicts else (None if None in verdicts else False) + + +__all__ = [ + "applies_without_extras", +] diff --git a/src/python_sbom/_model/_tools.py b/src/python_sbom/_model/_tools.py new file mode 100644 index 0000000..beb75ea --- /dev/null +++ b/src/python_sbom/_model/_tools.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from packaging.utils import canonicalize_name + +from ._components import component_from_metadata, file_component, purl, tool_ref + +if TYPE_CHECKING: + from ._inputs import BuildFacts + from ._types import Component, Dependency + + +def tool_components(facts: BuildFacts) -> tuple[list[Component], list[Dependency]]: + """ + Describe the build environment as build tools: the interpreter first, then every distribution by name. + + :param facts: the collected build facts + :returns: the tool components and the dependency entries between the distributions + + """ + interpreter = facts.interpreter + package = f"pkg:generic/{interpreter.name}@{interpreter.version}" + tools: Final[list[Component]] = [ + { + "type": "platform", + "bom-ref": tool_ref(package), + "name": interpreter.name, + "version": interpreter.version, + "purl": package, + "properties": [{"name": "python:implementation", "value": interpreter.implementation}], + } + ] + refs: Final[dict[str, str]] = { + canonicalize_name(distribution.metadata["Name"]): tool_ref( + purl(distribution.metadata["Name"], distribution.metadata["Version"]) + ) + for distribution in facts.distributions + } + dependencies: Final[list[Dependency]] = [] + for distribution in sorted(facts.distributions, key=lambda entry: canonicalize_name(entry.metadata["Name"])): + component = component_from_metadata(distribution.metadata) + component["bom-ref"] = refs[canonicalize_name(distribution.metadata["Name"])] + if distribution.files: + component["components"] = [file_component(component["bom-ref"], row) for row in distribution.files] + tools.append(component) + dependencies.append({ + "ref": component["bom-ref"], + "dependsOn": sorted(refs[name] for name in distribution.depends_on), + }) + return tools, dependencies + + +__all__ = [ + "tool_components", +] diff --git a/src/python_sbom/_model/_types.py b/src/python_sbom/_model/_types.py index 7593199..8d68600 100644 --- a/src/python_sbom/_model/_types.py +++ b/src/python_sbom/_model/_types.py @@ -4,7 +4,13 @@ class OrganizationalEntity(TypedDict): - """An organization that supplies or makes a component.""" + """ + An organization that supplies or makes a component. + + :ivar name: the organization's name + :ivar url: the organization's web sites + + """ name: str url: list[str] @@ -21,7 +27,13 @@ class _Hash(TypedDict): class Property(TypedDict): - """A name and value pair; names this package writes start with the project name or ``python:``.""" + """ + A name and value pair attached to a component or the document. + + :ivar name: the property's name; names this package writes start with the project name or ``python:`` + :ivar value: the property's value + + """ name: str value: str @@ -40,6 +52,15 @@ class LicenseInfo(TypedDict): class License(TypedDict): + """ + One license entry: either a license by SPDX id or name, or an SPDX expression. + + :ivar license: the license, by ``id`` or ``name`` + :ivar expression: an SPDX license expression + :ivar acknowledgement: ``declared`` or ``concluded``, for an expression + + """ + license: NotRequired[LicenseInfo] expression: NotRequired[str] acknowledgement: NotRequired[str] @@ -51,14 +72,14 @@ class _Method(TypedDict): value: str -class _Identity(TypedDict): +class Identity(TypedDict): field: str confidence: int methods: list[_Method] class _Evidence(TypedDict): - identity: list[_Identity] + identity: list[Identity] #: A CycloneDX component: a distribution, an application, the interpreter or a file. @@ -85,7 +106,13 @@ class _Evidence(TypedDict): class Dependency(TypedDict): - """The components one component depends on, by ``bom-ref``.""" + """ + The components one component depends on. + + :ivar ref: the ``bom-ref`` of the dependent component + :ivar dependsOn: the ``bom-ref`` values of its dependencies + + """ ref: str dependsOn: NotRequired[list[str]] @@ -113,12 +140,13 @@ class _Input(TypedDict): "taskTypes": list[str], "resourceReferences": list[_Reference], "outputs": list[_Output], - "inputs": list[_Input], + "inputs": NotRequired[list[_Input]], }, ) -_Formula = TypedDict("_Formula", {"bom-ref": str, "workflows": list[_Workflow]}) +#: How an artifact was built: one or more workflows, each with its tools, inputs and outputs. +Formula = TypedDict("Formula", {"bom-ref": str, "workflows": list[_Workflow]}) class Lifecycle(TypedDict): @@ -147,13 +175,22 @@ class Composition(TypedDict): class Body(TypedDict): - """The content of a document, everything but the header :func:`document` adds.""" + """ + The content of a document, everything but the header :func:`~python_sbom.document` adds. + + :ivar metadata: the document's metadata, with the described component at ``component`` + :ivar components: the components the described component holds or depends on + :ivar dependencies: the dependency graph, by ``bom-ref`` + :ivar compositions: how complete the inventory is + :ivar formulation: how the described component was built + + """ metadata: Metadata components: list[Component] dependencies: list[Dependency] compositions: NotRequired[list[Composition]] - formulation: NotRequired[list[_Formula]] + formulation: NotRequired[list[Formula]] #: A complete CycloneDX 1.6 JSON document. @@ -169,7 +206,7 @@ class Body(TypedDict): "components": list[Component], "dependencies": list[Dependency], "compositions": NotRequired[list[Composition]], - "formulation": NotRequired[list[_Formula]], + "formulation": NotRequired[list[Formula]], }, ) @@ -182,6 +219,8 @@ class Body(TypedDict): "Dependency", "Document", "ExternalReference", + "Formula", + "Identity", "License", "LicenseInfo", "Lifecycle", diff --git a/src/python_sbom/_model/_wheel.py b/src/python_sbom/_model/_wheel.py deleted file mode 100644 index 97bffc4..0000000 --- a/src/python_sbom/_model/_wheel.py +++ /dev/null @@ -1,329 +0,0 @@ -from __future__ import annotations - -import csv -import uuid -from dataclasses import dataclass -from io import StringIO -from pathlib import PurePosixPath -from typing import TYPE_CHECKING, Final -from urllib.parse import quote, urlsplit - -from packaging.markers import Marker -from packaging.requirements import Requirement -from packaging.utils import canonicalize_name - -from ._components import component_from_metadata, contacts, document, external_reference, file_component, purl - -if TYPE_CHECKING: - from collections.abc import Sequence - from email.message import Message - - from python_sbom._collect._facts import BuildFacts - from python_sbom._collect._settings import Settings - from python_sbom._collect._source import BuildScript, BundledWheel, SourceTree - - from ._types import Component, Contact, Dependency, Document, ExternalReference, License, Property - - -@dataclass(frozen=True, slots=True, kw_only=True) -class WheelInputs: - # Everything the document of one wheel is built from, gathered beforehand. - metadata: Message - wheel_name: str - settings: Settings - facts: BuildFacts - source: SourceTree - - -def describe(inputs: WheelInputs) -> Document: - metadata, settings = inputs.metadata, inputs.settings - namespace = canonicalize_name(metadata["Name"]) - repository = settings.repository or next( - ( - url.strip().rstrip("/") - for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) - if external_reference(label.strip(), url.strip())["type"] == "vcs" - ), - None, - ) - directories = " and ".join(sorted({str(PurePosixPath(pattern).parent) for pattern in settings.bundled_wheels})) - scope = ( - f"Components are the wheels bundled as data files under {directories} and the direct runtime dependencies" - " declared in the wheel's core metadata. The latter carry no version because they are resolved at install" - " time, not at build time." - if directories - else "Components are the direct runtime dependencies declared in the wheel's core metadata. They carry no" - " version because they are resolved at install time, not at build time." - ) - root = _root_component(inputs, repository) - bundled = [_bundled_component(wheel, namespace, settings, repository) for wheel in inputs.source.bundled] - declared = [_declared_dependency(requirement, namespace) for requirement in _runtime_requirements(metadata)] - bundled_dependencies: Final[list[Dependency]] = [ - {"ref": component["bom-ref"], "dependsOn": references} - for component in bundled - if (references := [child["bom-ref"] for child in component["components"] if child["type"] == "library"]) - ] - tools: Final[list[Component]] = list(inputs.facts.tools) - if (script := inputs.source.build_script) is not None: - tools.insert(0, _build_script_tool(script, metadata["Version"], settings, repository)) - authors = _authors(metadata) - licenses = _licenses(metadata) - return document( - { - "metadata": { - "timestamp": inputs.facts.timestamp, - "lifecycles": [{"phase": "build"}], - "tools": {"components": tools}, - **({"manufacturer": settings.supplier} if settings.supplier else {}), - **({"authors": authors} if authors else {}), - **({"supplier": settings.supplier} if settings.supplier else {}), - "component": root, - **({"licenses": licenses} if licenses else {}), - "properties": [{"name": f"{namespace}:sbom:scope", "value": scope}], - }, - "components": [*bundled, *declared], - "dependencies": [ - {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in [*bundled, *declared]]}, - *bundled_dependencies, - *inputs.facts.tool_dependencies, - ], - # transitive runtime dependencies are unknown until install time - "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], - "formulation": [ - { - "bom-ref": "formula:wheel-build", - "workflows": [ - { - "bom-ref": "workflow:wheel-build", - "uid": "wheel-build", - "name": "build the wheel and this SBOM", - "taskTypes": ["build"], - "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], - "outputs": [{"type": "artifact", "resource": {"ref": root["bom-ref"]}}], - "inputs": [ - {"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": str(inputs.facts.epoch)}]} - ], - } - ], - } - ], - }, - uuid.uuid5(uuid.NAMESPACE_URL, f"{repository or f'https://pypi.org/project/{namespace}'}/sboms"), - ) - - -def _root_component(inputs: WheelInputs, repository: str | None) -> Component: - metadata, settings = inputs.metadata, inputs.settings - name, version = metadata["Name"], metadata["Version"] - namespace = canonicalize_name(name) - references: Final[list[ExternalReference]] = [ - external_reference(label.strip(), url.strip()) - for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) - ] - references.extend([ - {"type": "distribution", "url": f"https://pypi.org/project/{name}/{version}/"}, - { - "type": "attestation", - "url": f"https://pypi.org/integrity/{name}/{version}/{inputs.wheel_name}/provenance", - }, - ]) - if repository and urlsplit(repository).netloc == "github.com": - references.extend([ - {"type": "security-contact", "url": f"{repository}/security/policy"}, - {"type": "advisories", "url": f"{repository}/security/advisories"}, - ]) - license_files: Final[list[str]] = metadata.get_all("License-File", []) - if repository and license_files: - references.append({"type": "license", "url": f"{repository}/blob/{settings.default_branch}/{license_files[0]}"}) - properties: Final[list[Property]] = [] - if requires_python := metadata.get("Requires-Python"): - properties.append({"name": f"{namespace}:requires-python", "value": requires_python}) - properties.extend({"name": "python:classifier", "value": value} for value in metadata.get_all("Classifier", [])) - properties.extend( - {"name": "python:keyword", "value": keyword.strip()} - for keyword in (metadata.get("Keywords") or "").split(",") - if keyword.strip() - ) - if commit := inputs.facts.commit: - if repository: - references.append({"type": "vcs", "url": f"{repository}/tree/{commit}", "comment": "exact source revision"}) - properties.append({"name": f"{namespace}:vcs-commit", "value": commit}) - authors = _authors(metadata) - licenses = _licenses(metadata) - package = purl(name, version) - return { - "type": "application", - "bom-ref": package, - **({"supplier": settings.supplier} if settings.supplier else {}), - **({"authors": authors} if authors else {}), - "name": name, - "version": version, - **({"description": summary} if (summary := metadata.get("Summary")) else {}), - **({"licenses": licenses} if licenses else {}), - **({"copyright": inputs.source.copyright} if inputs.source.copyright else {}), - "purl": package, - "externalReferences": references, - "properties": properties, - } - - -def _bundled_component(wheel: BundledWheel, namespace: str, settings: Settings, repository: str | None) -> Component: - component = component_from_metadata(wheel.metadata, "library") - # the purl names the package; two variants of one release, such as per-platform wheels, are told apart by file - component["bom-ref"] = f"{component['bom-ref']}?file_name={quote(PurePosixPath(wheel.path).name)}" - # a bundled wheel published from the same forge account as this project shares its supplier - if ( - settings.supplier - and repository - and any(reference["url"].startswith(_owner(repository)) for reference in component["externalReferences"]) - ): - component["supplier"] = settings.supplier - component["hashes"] = [{"alg": "SHA-256", "content": wheel.sha256}] - component["externalReferences"].append({ - "type": "distribution", - "url": f"https://pypi.org/project/{wheel.metadata['Name']}/{wheel.metadata['Version']}/", - }) - component["properties"] = [ - {"name": f"{namespace}:bundled-wheel", "value": wheel.path}, - *wheel.extra_properties, - *component["properties"], - ] - component["evidence"] = { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": wheel.metadata_name}], - }, - { - "field": "hash", - "confidence": 1, - "methods": [{"technique": "hash-comparison", "confidence": 1, "value": wheel.sha256}], - }, - ], - } - component["components"] = [ - file_component(component["bom-ref"], path, digest, size) - for path, digest, size in (row for row in csv.reader(StringIO(wheel.record, newline="")) if row) - if digest - ] - children: Final[dict[str, Component]] = {} - for source, child_metadata in wheel.vendored: - child = component_from_metadata(child_metadata, "library") - child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['bom-ref']}" - child["properties"].append({"name": f"{namespace}:vendored-manifest", "value": source}) - child["evidence"] = { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], - } - ], - } - # a vendored .dist-info comes after the pip manifest line naming the same purl, so its metadata wins - children[purl(child["name"], child["version"])] = child - component["components"].extend(children[key] for key in sorted(children)) - return component - - -def _owner(repository: str) -> str: - parts = urlsplit(repository) - return f"{parts.scheme}://{parts.netloc}/{parts.path.strip('/').split('/')[0]}/" - - -def _declared_dependency(requirement: str, namespace: str) -> Component: - parsed = Requirement(requirement) - component: Final[Component] = { - "type": "library", - # the same distribution can be declared more than once with different markers, so the purl is not unique - "bom-ref": f"requires-dist:{requirement}", - "name": parsed.name, - "purl": purl(parsed.name), - "externalReferences": [{"type": "distribution", "url": f"https://pypi.org/project/{parsed.name}/"}], - "properties": [{"name": f"{namespace}:requires-dist", "value": requirement}], - } - if parsed.marker is not None: - component["properties"].append({"name": f"{namespace}:environment-marker", "value": str(parsed.marker)}) - return component - - -def _runtime_requirements(metadata: Message) -> list[str]: - # a requirement belongs to the wheel's runtime dependencies when it can apply with no extra requested - return [ - requirement - for requirement in metadata.get_all("Requires-Dist", []) - if (marker := Requirement(requirement).marker) is None or _without_extras(marker) is not False - ] - - -def _without_extras(marker: Marker) -> bool | None: - # three-valued: an extra condition is decided with no extra requested, any other stays unknown, so a requirement - # is kept unless its extra conditions alone rule it out; packaging exposes the parsed tree only privately - return _any_of(marker._markers) # ruff: ignore[private-member-access] # no public marker tree - - -def _any_of( - markers: Sequence[object], # simplint: ignore[SMP602] # packaging's private tree is untyped -) -> bool | None: - # "and" binds tighter than "or", as in packaging's own evaluation - groups: Final[list[list[bool | None]]] = [[]] - for item in markers: - if item == "or": - groups.append([]) - elif isinstance(item, list): - groups[-1].append(_any_of(item)) - elif isinstance(item, tuple): # skips "and", which grouping already applies - groups[-1].append(_atom(item)) - verdicts = [False if False in group else (None if None in group else True) for group in groups] - return True if True in verdicts else (None if None in verdicts else False) - - -def _atom(item: tuple[object, ...]) -> bool | None: # simplint: ignore[SMP602] # packaging's private tree - kinds = [type(part).__name__ for part in item] - left, operator, right = (str(part) for part in item) - if kinds[0] == "Variable" and left == "extra": - return Marker(f"extra {operator} {right!r}").evaluate({"extra": ""}) - if kinds[2] == "Variable" and right == "extra": - return Marker(f"{left!r} {operator} extra").evaluate({"extra": ""}) - return None - - -def _build_script_tool(script: BuildScript, version: str, settings: Settings, repository: str | None) -> Component: - tool: Final[Component] = { - "type": "application", - "bom-ref": f"tool:{script.path}", - "name": script.path.name, - "version": version, - "hashes": [{"alg": "SHA-256", "content": script.sha256}], - } - if repository: - tool["externalReferences"] = [ - {"type": "vcs", "url": f"{repository}/blob/{settings.default_branch}/{script.path}"} - ] - return tool - - -def _authors(metadata: Message) -> list[Contact]: - # core metadata joins several names into one header - if people := contacts( - [name for entry in metadata.get_all("Maintainer", []) for name in entry.split(", ")], - metadata.get_all("Maintainer-email", []), - ): - return people - return contacts( - [name for entry in metadata.get_all("Author", []) for name in entry.split(", ")], - metadata.get_all("Author-email", []), - ) - - -def _licenses(metadata: Message) -> list[License]: - if expression := metadata.get("License-Expression"): - return [{"expression": expression, "acknowledgement": "declared"}] - return [] - - -__all__ = [ - "WheelInputs", - "describe", -] diff --git a/src/python_sbom/_validate/_checks.py b/src/python_sbom/_validate/_checks.py deleted file mode 100644 index 45e60fc..0000000 --- a/src/python_sbom/_validate/_checks.py +++ /dev/null @@ -1,218 +0,0 @@ -from __future__ import annotations - -import json -import re -import zipfile -from typing import TYPE_CHECKING, Final, TypeAlias - -from python_sbom._formats._spdx import to_spdx - -if TYPE_CHECKING: - from collections.abc import Iterator - from pathlib import Path - - from python_sbom._model._types import Component, Document, License - -#: a JSON value, as :func:`json.loads` hands it over -Json: TypeAlias = "str | int | float | bool | list[Json] | dict[str, Json] | None" -# SPDX matches operators only in upper case -_OPERATORS: Final[frozenset[str]] = frozenset({"AND", "OR", "WITH", "(", ")"}) -_TOKEN: Final[re.Pattern[str]] = re.compile(r"[()]|[^\s()]+") - - -def validate_wheel(wheel: Path, allowed_licenses: frozenset[str] | None) -> list[str]: - """ - Check every PEP 770 CycloneDX SBOM in a wheel; needs the ``validate`` extra. - - :param wheel: the wheel to read - :param allowed_licenses: SPDX license ids the shipped components may use, or ``None`` to skip the license policy - :returns: one message per problem, empty when the wheel carries at least one SBOM and every SBOM is valid - - """ - with zipfile.ZipFile(wheel) as archive: - sboms = { - name: archive.read(name).decode("utf-8") - for name in sorted(archive.namelist()) - if re.fullmatch(r"[^/]+\.dist-info/sboms/[^/]+\.cdx\.json", name) - } - if not sboms: - return ["no .dist-info/sboms/*.cdx.json found in the wheel"] - return [f"{name}: {problem}" for name, raw in sboms.items() for problem in validate(raw, allowed_licenses)] - - -def validate(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: - """ - Check one CycloneDX SBOM; needs the ``validate`` extra. - - Beyond the schema it requires the ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, - unique and resolvable ``bom-ref`` values, a root that depends on exactly the top-level components, and an SPDX 2.3 - rendering that validates. - - :param raw: the JSON text of the document - :param allowed_licenses: SPDX license ids the shipped components may use, or ``None`` to skip the license policy - :returns: one message per problem, empty when the document is valid - - """ - from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # optional extra - - if error := schema_error(raw): - return [f"does not conform to the CycloneDX 1.6 schema: {error}"] - loaded: Json = json.loads(raw) - # the schema leaves these optional, yet the checks below and the SPDX rendering read them - required: tuple[tuple[str, ...], ...] = ( - ("metadata", "timestamp"), - ("metadata", "tools", "components"), - ("metadata", "component", "bom-ref"), - ("metadata", "component", "name"), - ("components",), - ("dependencies",), - ) - if missing := [".".join(path) for path in required if not _has(loaded, path)]: - return [f"lacks {', '.join(missing)}, which every document python-sbom writes carries"] - document: Final[Document] = json.loads(raw) - # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize the document - # as CycloneDX at all, and silently rejects anything missing it as an unknown format - if not re.fullmatch( - r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", - serial := document.get("serialNumber", ""), - ): - return [f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {serial!r}"] - problems = _reference_problems(document) - problems.extend(spdx_problems(to_spdx(document))) - if allowed_licenses is not None: - # only what the artifact carries bytes for is judged: a declared requirement resolves at install time, and a - # library a bundled wheel vendors ships under that wheel's terms - problems.extend( - f"{component['bom-ref']}: {problem}" - for component in _shipped(document["components"]) - if (problem := _judge(component, allowed_licenses)) - ) - return problems - - -def _has(value: Json, path: tuple[str, ...]) -> bool: - for key in path: - if not isinstance(value, dict) or key not in value: - return False - value = value[key] - return True - - -def _reference_problems(document: Document) -> list[str]: - root_ref = document["metadata"]["component"]["bom-ref"] - tools = document["metadata"]["tools"]["components"] - every_ref = list(_bom_refs([document["metadata"]["component"], *document["components"], *tools])) - if duplicates := {ref for ref in every_ref if every_ref.count(ref) > 1}: - return [f"bom-ref values are not unique: {sorted(duplicates)}"] - dependencies = {entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"]} - problems = [] - if unknown := (set(dependencies) | set().union(*dependencies.values())) - set(every_ref): - problems.append(f"dependencies reference bom-refs that do not exist: {sorted(unknown)}") - if dependencies.get(root_ref) != (component_refs := {component["bom-ref"] for component in document["components"]}): - problems.append( - f"root dependsOn {sorted(dependencies.get(root_ref, []))} != components {sorted(component_refs)}" - ) - tool_refs = {tool["bom-ref"] for tool in tools} - problems.extend( - f"workflow {workflow['uid']} references unknown tools: {sorted(unknown)}" - for workflow in (workflow for formula in document.get("formulation", []) for workflow in formula["workflows"]) - if (unknown := {reference["ref"] for reference in workflow["resourceReferences"]} - tool_refs) - ) - return problems - - -def _bom_refs(components: list[Component]) -> Iterator[str]: - for component in components: - yield component["bom-ref"] - yield from _bom_refs(component.get("components", [])) - - -def _shipped(components: list[Component]) -> Iterator[Component]: - for component in components: - if component["type"] == "library" and ( - "hashes" in component or any(child["type"] == "file" for child in component.get("components", [])) - ): - yield component - yield from _shipped(component.get("components", [])) - - -def _judge(component: Component, allowed: frozenset[str]) -> str | None: - licenses: Final[list[License]] = component.get("licenses", []) - terms: Final[list[str]] = [ - entry["expression"] if "expression" in entry else entry["license"]["id"] - for entry in licenses - if "expression" in entry or "id" in entry["license"] - ] - if not terms: - if names := [entry["license"]["name"] for entry in licenses if "license" in entry]: - return f"declares license names {names} with no SPDX id" - return "declares no license" - try: - denied: Final[list[str]] = [term for term in terms if not _allowed(term, allowed)] - except ValueError as error: - return str(error) - if denied: - return f"license {' AND '.join(denied)} is outside the allowed licenses {sorted(allowed)}" - return None - - -def _allowed(expression: str, allowed: frozenset[str]) -> bool: - tokens: Final[list[str]] = _TOKEN.findall(expression)[::-1] - try: - verdict: Final[bool] = _any_of(tokens, allowed) - except ValueError as error: - msg = f"license expression {expression!r} is malformed: {error}" - raise ValueError(msg) from error - if tokens: - msg = f"license expression {expression!r} is malformed: unexpected {tokens[-1]!r}" - raise ValueError(msg) - return verdict - - -def _any_of(tokens: list[str], allowed: frozenset[str]) -> bool: - verdict = _all_of(tokens, allowed) - while tokens and tokens[-1] == "OR": - tokens.pop() - verdict |= _all_of(tokens, allowed) - return verdict - - -def _all_of(tokens: list[str], allowed: frozenset[str]) -> bool: - verdict = _term(tokens, allowed) - while tokens and tokens[-1] == "AND": - tokens.pop() - verdict &= _term(tokens, allowed) - return verdict - - -def _term(tokens: list[str], allowed: frozenset[str]) -> bool: - if (token := _pop(tokens)) == "(": - verdict: Final[bool] = _any_of(tokens, allowed) - if (closing := _pop(tokens)) != ")": - msg = f"expected ')', got {closing!r}" - raise ValueError(msg) - return verdict - if token in _OPERATORS: - msg = f"unexpected {token!r}" - raise ValueError(msg) - if tokens and tokens[-1] == "WITH": - tokens.pop() - # an exception grants extra permissions on top of the license it modifies, so that license alone decides - if (exception := _pop(tokens)) in _OPERATORS: - msg = f"unexpected {exception!r}" - raise ValueError(msg) - # SPDX matches license ids without regard to case - return token.casefold() in {identifier.casefold() for identifier in allowed} - - -def _pop(tokens: list[str]) -> str: - if not tokens: - msg: Final[str] = "it ends early" - raise ValueError(msg) - return tokens.pop() - - -__all__ = [ - "validate", - "validate_wheel", -] diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py new file mode 100644 index 0000000..da78693 --- /dev/null +++ b/src/python_sbom/_validate/_document.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +import re +from collections import Counter +from typing import TYPE_CHECKING, Final, TypeAlias + +from python_sbom._formats._spdx import to_spdx +from python_sbom.version import __version__ + +from ._licenses import license_problems + +if TYPE_CHECKING: + from python_sbom._model._types import Component, Document + +# a JSON value, as json.loads hands it over +_Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" +# the schema leaves these optional, yet the checks below and the SPDX rendering read them +_REQUIRED: Final[tuple[tuple[str, ...], ...]] = ( + ("metadata", "timestamp"), + ("metadata", "tools", "components"), + ("metadata", "component", "bom-ref"), + ("metadata", "component", "name"), + ("components",), + ("dependencies",), +) + + +def validate(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: + """ + Check one CycloneDX SBOM; needs the ``validate`` extra. + + Every document must conform to the CycloneDX 1.6 schema. A document python-sbom wrote must also carry the + ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, unique and resolvable ``bom-ref`` + values, a root that depends on exactly the top-level components, an SPDX 2.3 rendering that validates, and, with + ``allowed_licenses``, licenses the policy allows; documents other tools wrote get the schema check alone. + + :param raw: the JSON text of the document + :param allowed_licenses: SPDX license ids the shipped components may use, or ``None`` to skip the license policy + :returns: one message per problem, empty when the document is valid + :raises MissingExtraError: when the ``validate`` extra is not installed + + """ + from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # the optional extra loads on first use + + if error := schema_error(raw): + return [f"does not conform to the CycloneDX 1.6 schema: {error}"] + loaded = json.loads(raw) + # python-sbom names its file in a metadata property, so documents other tools wrote get the schema check alone; the + # schema already guarantees the shape of what this reads + if not any( + prop.get("name", "").endswith(":sbom:file") for prop in loaded.get("metadata", {}).get("properties", []) + ): + return [] + if missing := [".".join(path) for path in _REQUIRED if not _has(loaded, path)]: + return [f"lacks {', '.join(missing)}, which every document python-sbom writes carries"] + # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize the document + # as CycloneDX at all, and silently rejects anything missing it as an unknown format + if not re.fullmatch( + r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", + serial := loaded.get("serialNumber", ""), + ): + return [f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {serial!r}"] + # the SPDX rendering of a document whose references are broken only repeats those problems + problems = _reference_problems(loaded) or spdx_problems(to_spdx(loaded, __version__)) + if allowed_licenses is not None: + problems.extend(license_problems(loaded["components"], allowed_licenses)) + return problems + + +def _has(value: _Json, path: tuple[str, ...]) -> bool: + for key in path: + if not isinstance(value, dict) or key not in value: + return False + value = value[key] + return True + + +def _reference_problems(document: Document) -> list[str]: + root_ref = document["metadata"]["component"]["bom-ref"] + tools = document["metadata"]["tools"]["components"] + every_ref = _bom_refs([document["metadata"]["component"], *document["components"], *tools]) + if duplicates := sorted(ref for ref, count in Counter(every_ref).items() if count > 1): + return [f"bom-ref values are not unique: {duplicates}"] + problems: Final[list[str]] = [] + dependencies = {entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"]} + if unknown := (set(dependencies) | set().union(*dependencies.values())) - set(every_ref): + problems.append(f"dependencies reference bom-refs that do not exist: {sorted(unknown)}") + if dependencies.get(root_ref) != (component_refs := {component["bom-ref"] for component in document["components"]}): + problems.append( + f"root dependsOn {sorted(dependencies.get(root_ref, []))} != components {sorted(component_refs)}" + ) + tool_refs = {tool["bom-ref"] for tool in tools} + for formula in document.get("formulation", []): + for workflow in formula.get("workflows", []): + if unknown := {reference["ref"] for reference in workflow["resourceReferences"]} - tool_refs: + problems.append(f"workflow {workflow['uid']} references unknown tools: {sorted(unknown)}") + return problems + + +def _bom_refs(components: list[Component]) -> list[str]: + # iterative, so a deeply nested hostile document cannot exhaust the stack + found: Final[list[str]] = [] + pending = list(components) + while pending: + component = pending.pop() + found.append(component["bom-ref"]) + pending.extend(component.get("components", [])) + return found + + +__all__ = [ + "validate", +] diff --git a/src/python_sbom/_validate/_licenses.py b/src/python_sbom/_validate/_licenses.py new file mode 100644 index 0000000..9ce2970 --- /dev/null +++ b/src/python_sbom/_validate/_licenses.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from python_sbom._model._types import Component, License + +# SPDX matches operators only in upper case +_OPERATORS: Final[frozenset[str]] = frozenset({"AND", "OR", "WITH", "(", ")"}) +_TOKEN: Final[re.Pattern[str]] = re.compile(r"[()]|[^\s()]+") + + +def license_problems(components: list[Component], allowed: frozenset[str]) -> list[str]: + return [ + f"{component['bom-ref']}: {problem}" + for component in _shipped(components) + if (problem := _judge(component, allowed)) + ] + + +def _shipped(components: list[Component]) -> list[Component]: + # only what the artifact carries bytes for is judged: a declared requirement resolves at install time, and a + # library a bundled wheel vendors ships under that wheel's terms; iterative, so deep nesting cannot exhaust the + # stack + found: Final[list[Component]] = [] + pending = list(reversed(components)) + while pending: + component = pending.pop() + if component["type"] == "library" and ( + "hashes" in component or any(child["type"] == "file" for child in component.get("components", [])) + ): + found.append(component) + pending.extend(reversed(component.get("components", []))) + return found + + +def _judge(component: Component, allowed: frozenset[str]) -> str | None: + licenses: Final[list[License]] = component.get("licenses", []) + terms: Final[list[str]] = [ + entry["expression"] if "expression" in entry else entry["license"]["id"] + for entry in licenses + if "expression" in entry or "id" in entry["license"] + ] + if not terms: + if names := [entry["license"]["name"] for entry in licenses if "license" in entry]: + return f"declares license names {names} with no SPDX id" + return "declares no license" + denied: Final[list[str]] = [] + for term in terms: + try: + if not _allowed(term, allowed): + denied.append(term) + except ValueError as error: + return f"license expression {term!r} is malformed: {error}" + if denied: + return f"license {' AND '.join(denied)} is outside the allowed licenses {sorted(allowed)}" + return None + + +def _allowed(expression: str, allowed: frozenset[str]) -> bool: + # reversed, so the parser takes the next token with a cheap pop from the end + tokens: Final[list[str]] = _TOKEN.findall(expression)[::-1] + verdict: Final[bool] = _license_any_of(tokens, allowed) + if tokens: + msg = f"unexpected {tokens[-1]!r}" + raise ValueError(msg) + return verdict + + +def _license_any_of(tokens: list[str], allowed: frozenset[str]) -> bool: + verdict = _all_of(tokens, allowed) + while tokens and tokens[-1] == "OR": + tokens.pop() + verdict |= _all_of(tokens, allowed) + return verdict + + +def _all_of(tokens: list[str], allowed: frozenset[str]) -> bool: + verdict = _term(tokens, allowed) + while tokens and tokens[-1] == "AND": + tokens.pop() + verdict &= _term(tokens, allowed) + return verdict + + +def _term(tokens: list[str], allowed: frozenset[str]) -> bool: + if (token := _pop(tokens)) == "(": + verdict: Final[bool] = _license_any_of(tokens, allowed) + if (closing := _pop(tokens)) != ")": + msg = f"expected ')', got {closing!r}" + raise ValueError(msg) + return verdict + if token in _OPERATORS: + msg = f"unexpected {token!r}" + raise ValueError(msg) + if tokens and tokens[-1] == "WITH": + tokens.pop() + # an exception grants extra permissions on top of the license it modifies, so that license alone decides + if (exception := _pop(tokens)) in _OPERATORS: + msg = f"unexpected {exception!r}" + raise ValueError(msg) + # SPDX matches license ids without regard to case + return token.casefold() in {identifier.casefold() for identifier in allowed} + + +def _pop(tokens: list[str]) -> str: + if not tokens: + msg: Final[str] = "it ends early" + raise ValueError(msg) + return tokens.pop() + + +__all__ = [ + "license_problems", +] diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index b6eebdd..8b280fa 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -1,28 +1,35 @@ from __future__ import annotations -from functools import cache -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final -from cyclonedx.schema import SchemaVersion -from cyclonedx.validation.json import JsonStrictValidator -from spdx_tools.spdx.parser.error import SPDXParsingError -from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser -from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document +from python_sbom._errors import MissingExtraError + +try: + from cyclonedx.schema import SchemaVersion + from cyclonedx.validation.json import JsonStrictValidator + from spdx_tools.spdx.parser.error import SPDXParsingError + from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser + from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document +except ImportError as error: + _MESSAGE: Final[str] = "validation needs the validate extra: pip install 'python-sbom[validate]'" + raise MissingExtraError(_MESSAGE) from error if TYPE_CHECKING: from python_sbom._formats._spdx import Spdx +# loading the schema takes about a second, so the validator is built once +_VALIDATOR: Final[JsonStrictValidator] = JsonStrictValidator(SchemaVersion.V1_6) + def schema_error(raw: str) -> str | None: - error = _validator().validate_str(raw, all_errors=False) + try: + error = _VALIDATOR.validate_str(raw, all_errors=False) + except RecursionError: + # the schema validator recurses once per nesting level, so a hostile document can exhaust the stack + return "it nests too deeply to check" return None if error is None else str(error) -@cache # building the validator loads the whole schema, which takes a second -def _validator() -> JsonStrictValidator: - return JsonStrictValidator(SchemaVersion.V1_6) - - def spdx_problems(spdx: Spdx) -> list[str]: try: parsed = JsonLikeDictParser().parse(dict(spdx)) diff --git a/src/python_sbom/_validate/_wheel.py b/src/python_sbom/_validate/_wheel.py new file mode 100644 index 0000000..1042571 --- /dev/null +++ b/src/python_sbom/_validate/_wheel.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import zipfile +from typing import TYPE_CHECKING + +from python_sbom._wheel._archive import find_dist_info + +from ._document import validate + +if TYPE_CHECKING: + from pathlib import Path + + +def validate_wheel(wheel: Path, allowed_licenses: frozenset[str] | None) -> list[str]: + """ + Check every CycloneDX SBOM a wheel carries under ``.dist-info/sboms/``; needs the ``validate`` extra. + + :param wheel: the wheel to read + :param allowed_licenses: SPDX license ids the shipped components may use, or ``None`` to skip the license policy + :returns: one message per problem, empty when the wheel carries at least one SBOM and every SBOM is valid + :raises MissingExtraError: when the ``validate`` extra is not installed + + """ + with zipfile.ZipFile(wheel) as archive: + names = archive.namelist() + prefix = f"{find_dist_info(names)}/sboms/" + sboms = { + name: archive.read(name).decode("utf-8") + for name in sorted(names) + if name.startswith(prefix) and name.endswith(".cdx.json") + } + if not sboms: + return ["no .dist-info/sboms/*.cdx.json found in the wheel"] + return [f"{name}: {problem}" for name, raw in sboms.items() for problem in validate(raw, allowed_licenses)] + + +__all__ = [ + "validate_wheel", +] diff --git a/src/python_sbom/_wheel/__init__.py b/src/python_sbom/_wheel/__init__.py index 4c83645..094cffe 100644 --- a/src/python_sbom/_wheel/__init__.py +++ b/src/python_sbom/_wheel/__init__.py @@ -1 +1 @@ -"""Writing an SBOM into a wheel some build backend already built.""" +"""Wheel archives: their RECORD, their metadata files, and writing an SBOM into a built one.""" diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py new file mode 100644 index 0000000..ad1d1dc --- /dev/null +++ b/src/python_sbom/_wheel/_archive.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import hashlib +import os +import stat +import tempfile +import time +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from python_sbom._errors import WheelError + +from ._record import record_line + +if TYPE_CHECKING: + from collections.abc import Iterable + +# metadata files beyond this size are refused, so a hostile wheel cannot exhaust memory +_METADATA_LIMIT: Final[int] = 16 * 1024 * 1024 + + +def embed_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: + """ + Write an SBOM into a built wheel under ``.dist-info/sboms/``, with its ``RECORD`` row. + + An SBOM of the same name is replaced. Every other entry keeps its name, order, timestamp and attributes; the new + entry takes ``epoch`` as its timestamp and fixed attributes, so the result is the same on every operating system. + The wheel keeps its permissions, and a failure leaves it untouched with no temporary file behind. + + :param wheel: the wheel to rewrite in place + :param name: the SBOM's file name, such as ``project.cdx.json`` + :param content: the SBOM's bytes + :param epoch: the Unix timestamp of the new entry + :returns: the SBOM's path inside the wheel + :raises WheelError: when the archive is not a wheel + + """ + with zipfile.ZipFile(wheel) as archive: + entries: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [ + (info, archive.read(info)) for info in archive.infolist() + ] + if (dist_info := find_dist_info(info.filename for info, _ in entries)) is None: + msg = f"{wheel} has no .dist-info/METADATA, so it is not a wheel" + raise WheelError(msg) + record: Final[str] = f"{dist_info}/RECORD" + if record not in {info.filename for info, _ in entries}: + msg = f"{wheel} has no {record}" + raise WheelError(msg) + sbom: Final[str] = f"{dist_info}/sboms/{name}" + lines: Final[list[str]] = [ + line + for info, data in entries + if info.filename == record + for line in data.decode("utf-8").splitlines(keepends=True) + if not line.startswith(f"{sbom},") + ] + newline = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" + # the row of RECORD itself stays last + at = next((index for index, line in enumerate(lines) if line.startswith(f"{record},")), len(lines)) + lines.insert(at, f"{record_line(sbom, content)}{newline}") + # zip timestamps start at 1980 + entry = zipfile.ZipInfo(sbom, time.gmtime(max(epoch, 315532800))[:6]) + entry.compress_type = zipfile.ZIP_DEFLATED + entry.create_system = 3 # Unix, whichever system writes the entry, so the bytes match across operating systems + entry.external_attr = (stat.S_IFREG | 0o644) << 16 + # the SBOM goes right before RECORD, which stays the last entry + members: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [] + for info, data in entries: + if info.filename == record: + members.extend([(entry, content), (info, "".join(lines).encode())]) + elif info.filename != sbom: + members.append((info, data)) + # a name no "dist/*.whl" glob matches, in the wheel's directory so the final rename stays on one file system + descriptor, temporary = tempfile.mkstemp(dir=wheel.parent, prefix=".python-sbom-", suffix=".tmp") + os.close(descriptor) + try: + with zipfile.ZipFile(temporary, "w") as archive: + for info, data in members: + archive.writestr(info, data) + Path(temporary).chmod(stat.S_IMODE(wheel.stat().st_mode)) + Path(temporary).replace(wheel) + except BaseException: + Path(temporary).unlink(missing_ok=True) + raise + return sbom + + +def find_dist_info(names: Iterable[str]) -> str | None: + return next( + (name.split("/")[0] for name in names if name.count("/") == 1 and name.endswith(".dist-info/METADATA")), + None, + ) + + +def read_metadata_member(archive: zipfile.ZipFile, name: str, where: str) -> bytes: + try: + info = archive.getinfo(name) + except KeyError as error: + msg = f"{where} has no {name}" + raise WheelError(msg) from error + if info.file_size > _METADATA_LIMIT: + msg = f"{where}: {name} is {info.file_size} bytes, over the {_METADATA_LIMIT} byte limit" + raise WheelError(msg) + return archive.read(info) + + +def member_sha256(archive: zipfile.ZipFile, name: str) -> str: + digest = hashlib.sha256() + with archive.open(name) as member: + for chunk in iter(lambda: member.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +__all__ = [ + "embed_sbom", + "find_dist_info", + "member_sha256", + "read_metadata_member", +] diff --git a/src/python_sbom/_wheel/_embed.py b/src/python_sbom/_wheel/_embed.py deleted file mode 100644 index 22c61ff..0000000 --- a/src/python_sbom/_wheel/_embed.py +++ /dev/null @@ -1,128 +0,0 @@ -from __future__ import annotations - -import os -import tempfile -import time -import zipfile -from email.parser import Parser -from pathlib import Path -from typing import TYPE_CHECKING, Final - -from python_sbom._backend import Backend -from python_sbom._errors import WheelError -from python_sbom._model._components import record_digest - -if TYPE_CHECKING: - from email.message import Message - - from python_sbom._collect._facts import BuildFacts - from python_sbom._collect._settings import Settings - - -class BuiltWheel(Backend): - """ - A wheel some other build backend already built, such as meson-python, and the project it came from. - - :param wheel: the wheel, rewritten in place by :meth:`embed` - :param root: the project directory the wheel was built from - :param settings: the project's options, usually from ``[tool.python-sbom]`` - - """ - - def __init__(self, wheel: Path, root: Path, settings: Settings) -> None: - self._wheel = wheel - self._root = root - self._settings = settings - with zipfile.ZipFile(wheel) as archive: - self._entries: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [ - (info, archive.read(info)) for info in archive.infolist() - ] - if ( - dist_info := next( - ( - info.filename.split("/")[0] - for info, _ in self._entries - if info.filename.count("/") == 1 and info.filename.endswith(".dist-info/METADATA") - ), - None, - ) - ) is None: - msg = f"{wheel} has no .dist-info/METADATA, so it is not a wheel" - raise WheelError(msg) - self._dist_info: Final[str] = dist_info - - @property - def root(self) -> Path: - """The project directory the wheel was built from.""" - return self._root - - @property - def settings(self) -> Settings: - """The project's options.""" - return self._settings - - def metadata(self) -> Message: - """ - Read the core metadata from the wheel. - - :returns: the parsed ``METADATA`` - - """ - return Parser().parsestr(self._read(f"{self._dist_info}/METADATA").decode("utf-8")) - - def _read(self, name: str) -> bytes: - return next(data for info, data in self._entries if info.filename == name) - - def wheel_name(self) -> str: - """ - Return the wheel's file name. - - :returns: the file name - - """ - return self._wheel.name - - def embed(self, facts: BuildFacts) -> str: - """ - Write the SBOM into the wheel under ``.dist-info/sboms/``, with a ``RECORD`` row. - - An SBOM of the same name is replaced. Every other entry keeps its name, order, timestamp and permissions, and - the new one takes its timestamp from ``SOURCE_DATE_EPOCH``, so embedding into the same wheel twice gives the - same bytes. - - :param facts: what the environment says about the build - :returns: the SBOM's path inside the wheel - - """ - name, content = self.sbom(facts) - sbom: Final[str] = f"{self._dist_info}/sboms/{name}" - record: Final[str] = f"{self._dist_info}/RECORD" - lines: Final[list[str]] = [ - line - for line in self._read(record).decode("utf-8").splitlines(keepends=True) - if not line.startswith(f"{sbom},") - ] - newline = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" - # the RECORD row of RECORD itself stays last - at = next((index for index, line in enumerate(lines) if line.startswith(f"{record},")), len(lines)) - lines.insert(at, f"{sbom},{record_digest(content)},{len(content)}{newline}") - entry = zipfile.ZipInfo(sbom, time.gmtime(max(facts.epoch, 315532800))[:6]) # zip timestamps start at 1980 - entry.compress_type = zipfile.ZIP_DEFLATED - entry.external_attr = 0o644 << 16 - # written next to the wheel and moved over it, so a failure leaves the wheel as it was - descriptor, temporary = tempfile.mkstemp(dir=self._wheel.parent, suffix=".whl") - os.close(descriptor) - with zipfile.ZipFile(temporary, "w") as archive: - for info, data in self._entries: - if info.filename == record: - archive.writestr(entry, content) - archive.writestr(info, "".join(lines).encode()) - elif info.filename != sbom: - archive.writestr(info, data) - Path(temporary).replace(self._wheel) - return sbom - - -__all__ = [ - "BuiltWheel", -] diff --git a/src/python_sbom/_wheel/_record.py b/src/python_sbom/_wheel/_record.py new file mode 100644 index 0000000..b2701fd --- /dev/null +++ b/src/python_sbom/_wheel/_record.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import base64 +import binascii +import csv +import hashlib +from io import StringIO +from typing import Final + +from python_sbom._errors import WheelError +from python_sbom._model._inputs import RecordRow + +# the algorithms a wheel RECORD may name, as CycloneDX spells them +_ALGORITHMS: Final[dict[str, str]] = { + "md5": "MD5", + "sha1": "SHA-1", + "sha256": "SHA-256", + "sha384": "SHA-384", + "sha512": "SHA-512", +} + + +def parse_record(text: str, where: str) -> list[RecordRow]: + # the row of RECORD itself has no digest and is left out + rows: Final[list[RecordRow]] = [] + for number, columns in enumerate(csv.reader(StringIO(text, newline="")), start=1): + if not columns: + continue + if len(columns) != 3: # ruff: ignore[magic-value-comparison] # path, digest and size, as the wheel specification lays out a row + msg = f"{where}: RECORD row {number} has {len(columns)} columns, not 3" + raise WheelError(msg) + path, digest, size = columns + if digest: + algorithm, hexdigest = decode_digest(digest, f"{where}: RECORD row {number}") + rows.append(RecordRow(path=path, algorithm=algorithm, digest=hexdigest, size=size)) + return rows + + +def decode_digest(digest: str, where: str) -> tuple[str, str]: + name, _, encoded = digest.partition("=") + if (algorithm := _ALGORITHMS.get(name)) is None: + msg = f"{where}: unknown hash algorithm {name!r}" + raise WheelError(msg) + try: + raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + except binascii.Error as error: + msg = f"{where}: digest {encoded!r} is not urlsafe base64" + raise WheelError(msg) from error + return algorithm, raw.hex() + + +def record_line(name: str, content: bytes) -> str: + encoded = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() + return f"{name},sha256={encoded},{len(content)}" + + +__all__ = [ + "decode_digest", + "parse_record", + "record_line", +] diff --git a/src/python_sbom/hatch/__init__.py b/src/python_sbom/hatch/__init__.py index 8c62fa0..4e6ce53 100644 --- a/src/python_sbom/hatch/__init__.py +++ b/src/python_sbom/hatch/__init__.py @@ -1,4 +1,4 @@ -"""The hatchling build hook: the backend that writes the SBOM while hatchling builds the wheel.""" +"""The hatchling build hook, which writes the SBOM into every wheel hatchling builds.""" from __future__ import annotations diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index 6f41961..fbff7be 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -1,132 +1,110 @@ from __future__ import annotations -import shutil -import tempfile -from email.parser import Parser from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final from hatchling.builders.hooks.plugin.interface import BuildHookInterface -from hatchling.builders.wheel import WheelBuilder +from hatchling.builders.utils import get_reproducible_timestamp from hatchling.utils.constants import DEFAULT_BUILD_SCRIPT -from python_sbom._backend import Backend -from python_sbom._collect._facts import BuildFacts, git_revision -from python_sbom._collect._settings import Settings -from python_sbom._extension import ContentProvider +from python_sbom._collect._facts import collect_facts, default_revision +from python_sbom._collect._settings import settings_from_table +from python_sbom._collect._source import read_source_tree +from python_sbom._generate._backend import BuiltWheel +from python_sbom._generate._content import ContentProvider, load_content_provider +from python_sbom._generate._pipeline import build_sbom +from python_sbom._wheel._archive import embed_sbom if TYPE_CHECKING: - from email.message import Message + from python_sbom._generate._backend import Backend + from python_sbom._model._inputs import Settings - from python_sbom._model._types import Property +# options of every hatchling hook table, which hatchling itself reads +_HATCHLING_OPTIONS: Final[frozenset[str]] = frozenset({ + "dependencies", + "enable-by-default", + "path", + "require-runtime-dependencies", + "require-runtime-features", +}) -class SbomBuildHook(BuildHookInterface, ContentProvider): +class SbomBuildHook(BuildHookInterface): """ Hatchling build hook that writes a CycloneDX SBOM into the wheel's ``.dist-info/sboms/``, as PEP 770 describes. - Enable it with a ``[tool.hatch.build.hooks.python-sbom]`` table, or subclass it in a hatchling custom build hook to - add project-specific detail through :meth:`bundled_wheel_properties`. + It runs once hatchling has written the wheel, so the SBOM describes the wheel's final metadata and file name + whatever other hooks changed. Options come from ``[tool.python-sbom]``, overridden by the hook's own table. + + :cvar PLUGIN_NAME: the hook's name in ``[tool.hatch.build.hooks]`` """ PLUGIN_NAME = "python-sbom" - _sbom_directory: Path | None = None - def initialize( + def finalize( self, - version: str, # ruff: ignore[unused-method-argument] # hatchling's signature - build_data: dict[str, Any], # simplint: ignore[SMP602] # hatchling API + version: str, + build_data: dict[str, Any], # ruff: ignore[unused-method-argument] # simplint: ignore[SMP602] # hatchling's signature + artifact_path: str, ) -> None: """ - Write the SBOM and register it with the wheel build. + Write the SBOM into the wheel hatchling built. - :param version: hatchling's build variant name, such as ``standard`` - :param build_data: the build data, whose ``sbom_files`` receives the SBOM + :param version: hatchling's build variant; an editable wheel gets no SBOM, since it carries no code + :param build_data: unused + :param artifact_path: the built wheel """ - if not isinstance(builder := self.build_config.builder, WheelBuilder): + if self.target_name != "wheel" or version == "editable": return - tag = build_data.get("tag") or ( - builder.get_best_matching_tag() if build_data["infer_tag"] else builder.get_default_tag() + root = Path(self.root) + # hatchling has parsed pyproject.toml already + settings = settings_from_table( + { + **self.metadata.config.get("tool", {}).get("python-sbom", {}), + **{key: value for key, value in self.config.items() if key not in _HATCHLING_OPTIONS}, + }, + f"[tool.python-sbom] or the {self.PLUGIN_NAME} hook's table", ) - backend = _HatchBackend( - hook=self, - metadata=builder.config.core_metadata_constructor( - self.metadata, extra_dependencies=build_data["dependencies"] - ), - wheel_name=f"{builder.artifact_project_id}-{tag}.whl", + content, provider_scripts = self.content_provider(settings) + # a custom hook script shapes the document, so the SBOM lists it among the build tools + own_script = ( + (PurePosixPath(self.config.get("path", DEFAULT_BUILD_SCRIPT)),) if self.PLUGIN_NAME == "custom" else () ) - name, content = backend.sbom(BuildFacts.collect(backend.root, git_revision)) - # written outside self.directory, which is also the wheel's output directory, so no stray copy lands next to - # the built artifacts - self._sbom_directory = Path(tempfile.mkdtemp(prefix="python-sbom-")) - (sbom := self._sbom_directory / name).write_bytes(content) - build_data["sbom_files"].append(str(sbom)) - - def bundled_wheel_properties( # ruff: ignore[no-self-use] # subclasses override it - self, - wheel: Path, # ruff: ignore[unused-method-argument] # subclasses use it - ) -> list[Property]: - """ - Return extra properties for a wheel the project bundles as a data file. + request = self.backend(Path(artifact_path), settings, (*own_script, *provider_scripts)).request() + # hatchling's own fixed time for a reproducible build when SOURCE_DATE_EPOCH is unset, so the SBOM agrees with + # the zip entries hatchling wrote + facts = collect_facts( + root, default_revision, None, get_reproducible_timestamp() if self.build_config.reproducible else None + ) + sbom = build_sbom(request, read_source_tree(request), facts, content) + embed_sbom(Path(artifact_path), sbom.file_name, sbom.content, facts.epoch) - Override it to say what the project does with the wheel, for example which Python versions receive it. The - properties follow ``:bundled-wheel`` and precede the ones taken from the wheel's metadata. + def content_provider(self, settings: Settings) -> tuple[ContentProvider, tuple[PurePosixPath, ...]]: + """ + Return what the project adds to its SBOM; override it in a custom build hook to supply a provider directly. - :param wheel: the bundled wheel, an absolute path under the project root - :returns: the properties to add; the default adds none + :param settings: the project's options + :returns: the provider and the scripts that define it; by default the ``content-provider`` option's, or none """ - return [] + if settings.content_provider is None: + return ContentProvider(), () + provider, script = load_content_provider(settings.content_provider, Path(self.root)) + return provider, () if script is None else (script,) - def finalize( - self, - version: str, # ruff: ignore[unused-method-argument] # hatchling's signature - build_data: dict[str, Any], # ruff: ignore[unused-method-argument] # simplint: ignore[SMP602] # hatchling API - artifact_path: str, # ruff: ignore[unused-method-argument] # hatchling's signature - ) -> None: + def backend(self, wheel: Path, settings: Settings, build_scripts: tuple[PurePosixPath, ...]) -> Backend: """ - Remove the SBOM's temporary directory once hatchling copied the file into the wheel. + Return the backend that describes the built wheel. - :param version: hatchling's build variant name - :param build_data: the build data - :param artifact_path: the built wheel + :param wheel: the wheel hatchling built + :param settings: the project's options + :param build_scripts: the scripts that shape the document + :returns: a :class:`~python_sbom.BuiltWheel` by default """ - if self._sbom_directory is not None: - shutil.rmtree(self._sbom_directory) - self._sbom_directory = None - - -class _HatchBackend(Backend): - def __init__(self, *, hook: SbomBuildHook, metadata: str, wheel_name: str) -> None: - self._hook = hook - self._metadata = metadata - self._wheel_name = wheel_name - - @property - def root(self) -> Path: - return Path(self._hook.root) - - @property - def settings(self) -> Settings: - return Settings.from_table(self._hook.config, f"build hook `{self._hook.PLUGIN_NAME}`") - - def metadata(self) -> Message: - return Parser().parsestr(self._metadata) - - def wheel_name(self) -> str: - return self._wheel_name - - def build_script(self) -> PurePosixPath | None: - # a custom build hook script shapes the document, so the SBOM lists it among the build tools - if self._hook.PLUGIN_NAME != "custom": - return None - return PurePosixPath(self._hook.config.get("path", DEFAULT_BUILD_SCRIPT)) - - def bundled_wheel_properties(self, wheel: Path) -> list[Property]: - return self._hook.bundled_wheel_properties(wheel) + return BuiltWheel(wheel=wheel, root=Path(self.root), settings=settings, build_scripts=build_scripts) __all__ = [ diff --git a/tests/_cli/test_cli.py b/tests/_cli/test_cli.py new file mode 100644 index 0000000..3970b16 --- /dev/null +++ b/tests/_cli/test_cli.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +import runpy +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom import dumps, main + +if TYPE_CHECKING: + from conftest import Describe, Install, WriteWheel + + from python_sbom import Document + +_SBOM: Final[str] = "native-1.0.dist-info/sboms/native.cdx.json" + + +@pytest.fixture +def project(tmp_path: Path) -> Path: + (root := tmp_path / "project").mkdir() + (root / "pyproject.toml").write_text("[tool.python-sbom]\ndefault-branch = 'trunk'\n", encoding="utf-8") + return root + + +@pytest.fixture +def wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: + return write_wheel( + tmp_path / "dist" / "native-1.0-py3-none-any.whl", + {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"}, + ) + + +def test_embed( + project: Path, wheel: Path, tmp_path: Path, install: Install, capsys: pytest.CaptureFixture[str] +) -> None: + install(site := tmp_path / "site", "Name: meson-python\nVersion: 0.18\n", {}) + main(["embed", str(wheel), "--root", str(project), "--commit", "c0ffee", "--build-env", str(site)]) + document = embedded(wheel) + assert capsys.readouterr().out == f"wrote {_SBOM} into {wheel.name}\n" + assert {"name": "native:vcs-commit", "value": "c0ffee"} in document["metadata"]["component"]["properties"] + assert [tool["name"] for tool in document["metadata"]["tools"]["components"][1:]] == ["meson-python"] + + +def embedded(wheel: Path) -> Document: + return json.loads(zipfile.ZipFile(wheel).read(_SBOM)) + + +def test_embed_defaults_to_current_directory(project: Path, wheel: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(project) + monkeypatch.setenv("PYTHON_SBOM_COMMIT", "cafe") + main(["embed", str(wheel), "--build-env", str(project)]) + assert {"name": "native:vcs-commit", "value": "cafe"} in embedded(wheel)["metadata"]["component"]["properties"] + + +@pytest.mark.parametrize( + ("option", "table"), + [ + pytest.param(["--content-provider", "MODULE:Native"], "", id="option"), + pytest.param([], "content-provider = 'MODULE:Native'\n", id="setting"), + ], +) +def test_embed_with_content_provider(tmp_path: Path, project: Path, wheel: Path, option: list[str], table: str) -> None: + # a module name unique to the test, so the import cache never hands over another test's module + provider = """\ +from python_sbom import ContentProvider, Contributions + + +class Native(ContentProvider): + def contributions(self, request): + return Contributions( + components=[], dependencies=[], root_properties=[{"name": "native:by", "value": "cli"}], tools=[] + ) +""" + (project / f"{tmp_path.name}.py").write_text(provider, encoding="utf-8") + (project / "pyproject.toml").write_text(f"[tool.python-sbom]\n{table.replace('MODULE', tmp_path.name)}") + main([ + "embed", + str(wheel), + "--root", + str(project), + "--build-env", + str(project), + *(argument.replace("MODULE", tmp_path.name) for argument in option), + ]) + document = embedded(wheel) + assert {"name": "native:by", "value": "cli"} in document["metadata"]["component"]["properties"] + assert document["metadata"]["tools"]["components"][0]["name"] == f"{tmp_path.name}.py" + + +def test_error_without_traceback(tmp_path: Path, wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as exit_info: + main(["embed", str(wheel), "--root", str(tmp_path)]) + assert exit_info.value.code == 1 + assert capsys.readouterr().err == f"python-sbom: {tmp_path} has no pyproject.toml to read [tool.python-sbom] from\n" + + +def test_spdx(tmp_path: Path, describe: Describe) -> None: + (source := tmp_path / "example.cdx.json").write_bytes(dumps(describe())) + main(["spdx", str(source), str(target := tmp_path / "example.spdx.json")]) + assert json.loads(target.read_text(encoding="utf-8"))["spdxVersion"] == "SPDX-2.3" + + +def test_validate_directory(project: Path, wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + main(["embed", str(wheel), "--root", str(project), "--build-env", str(project)]) + capsys.readouterr() + main(["validate", str(wheel.parent), "--allow-license", "MIT"]) + assert capsys.readouterr().out == f"SBOM in {wheel.name} is valid\n" + + +def test_validate_reports_problems(wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["validate", str(wheel)]) + assert capsys.readouterr().err == f"SBOM invalid: {wheel.name}: no .dist-info/sboms/*.cdx.json found in the wheel\n" + + +def test_validate_without_wheels(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["validate", str(tmp_path)]) + assert capsys.readouterr().err == f"no wheel found in {tmp_path}\n" + + +def test_validate_without_extra( + wheel: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setitem(sys.modules, "cyclonedx.schema", None) + monkeypatch.delitem(sys.modules, "python_sbom._validate._schemas", raising=False) + zipfile.ZipFile(wheel, "a").writestr(_SBOM, b"{}") + with pytest.raises(SystemExit): + main(["validate", str(wheel)]) + assert capsys.readouterr().err == ( + "python-sbom: validation needs the validate extra: pip install 'python-sbom[validate]'\n" + ) + + +def test_module_entry_point(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sys, "argv", ["python-sbom", "--help"]) + with pytest.raises(SystemExit): + runpy.run_module("python_sbom", run_name="__main__") + assert capsys.readouterr().out.startswith("usage: python-sbom ") + + +def test_console_script() -> None: + script = Path(sys.executable).parent / "python-sbom" + assert subprocess.check_output([script, "--help"], text=True).startswith("usage: python-sbom ") diff --git a/tests/_collect/test_collect_facts.py b/tests/_collect/test_collect_facts.py new file mode 100644 index 0000000..27bcce5 --- /dev/null +++ b/tests/_collect/test_collect_facts.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import hashlib +import platform +import sys +import time +from typing import TYPE_CHECKING + +import pytest + +from python_sbom import ( + Interpreter, + RecordRow, + Revision, + SettingsError, + SourceRevisionError, + collect_facts, + default_revision, + git_revision, +) + +if TYPE_CHECKING: + from pathlib import Path + + from conftest import Git, Install + + from python_sbom import BuildFacts + + +def test_epoch_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + facts = collect_facts(tmp_path, no_revision, [], 0) + assert facts.epoch == 1_700_000_000 + assert facts.epoch_from_environment + + +def no_revision(_root: Path, /) -> None: + return None + + +def test_epoch_default(tmp_path: Path) -> None: + facts = collect_facts(tmp_path, no_revision, [], 42) + assert (facts.epoch, facts.epoch_from_environment) == (42, False) + + +def test_empty_epoch_is_unset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "") + facts = collect_facts(tmp_path, no_revision, [], 42) + assert (facts.epoch, facts.epoch_from_environment) == (42, False) + + +def test_epoch_without_default_is_now(tmp_path: Path) -> None: + before = int(time.time()) + assert before <= collect_facts(tmp_path, no_revision, [], None).epoch <= time.time() + + +def test_invalid_epoch(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "yesterday") + with pytest.raises(SettingsError, match="SOURCE_DATE_EPOCH must be a Unix timestamp, got 'yesterday'"): + collect_facts(tmp_path, no_revision, [], None) + + +def test_interpreter(tmp_path: Path) -> None: + assert collect_facts(tmp_path, no_revision, [], 0).interpreter == Interpreter( + name=sys.implementation.name, + version=platform.python_version(), + implementation=platform.python_implementation(), + ) + + +def test_revision_reads_root(tmp_path: Path) -> None: + def reader(root: Path, /) -> Revision: + return Revision(commit=root.name, dirty=False) + + assert collect_facts(tmp_path, reader, [], 0).revision == Revision(commit=tmp_path.name, dirty=False) + + +def test_default_path_is_this_interpreter(tmp_path: Path) -> None: + names = { + distribution.metadata["Name"] for distribution in collect_facts(tmp_path, no_revision, None, 0).distributions + } + assert "pytest" in names + + +def test_pure_distribution_files(tmp_path: Path, install: Install) -> None: + # nested, so the console script, two levels up as installers write it, stays inside the test's directory + install( + site := tmp_path / "lib" / "site-packages", + "Name: alpha\nVersion: 1.0\n", + { + "alpha/__init__.py": b"x = 1\n", + "../../bin/alpha": b"#!/usr/bin/python\n", + "alpha-1.0.dist-info/INSTALLER": b"uv\n", + "alpha-1.0.dist-info/REQUESTED": b"", + "alpha-1.0.dist-info/direct_url.json": b'{"url": "file:///tmp/alpha.whl", "archive_info": {}}', + "alpha-1.0.dist-info/uv_cache.json": b'{"timestamp": 1}', + }, + ) + (distribution,) = facts_from(site).distributions + assert distribution.files == [ + RecordRow( + path="alpha/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"x = 1\n").hexdigest(), size="6" + ), + RecordRow( + path="alpha-1.0.dist-info/METADATA", + algorithm="SHA-256", + digest=hashlib.sha256(b"Name: alpha\nVersion: 1.0\n").hexdigest(), + size="25", + ), + RecordRow( + path="alpha-1.0.dist-info/WHEEL", + algorithm="SHA-256", + digest=hashlib.sha256(b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\n").hexdigest(), + size="41", + ), + ] + + +def facts_from(site: Path) -> BuildFacts: + return collect_facts(site, no_revision, [str(site)], 0) + + +def test_platform_distribution_lists_no_files(tmp_path: Path, install: Install) -> None: + install(tmp_path, "Name: native\nVersion: 1.0\n", {"native.so": b"\x7fELF"}, purelib=False) + assert facts_from(tmp_path).distributions[0].files == [] + + +def test_editable_distribution_lists_no_files(tmp_path: Path, install: Install) -> None: + editable = '{"url": "file:///checkout", "dir_info": {"editable": true}}' + install(tmp_path, "Name: local\nVersion: 1.0\n", {"local.pth": b"/checkout/src\n"}, direct_url=editable) + assert facts_from(tmp_path).distributions[0].files == [] + + +def test_first_distribution_on_path_wins(tmp_path: Path, install: Install) -> None: + install(first := tmp_path / "first", "Name: alpha\nVersion: 1.0\n", {}) + install(second := tmp_path / "second", "Name: Alpha\nVersion: 2.0\n", {}) + (distribution,) = collect_facts(tmp_path, no_revision, [str(first), str(second)], 0).distributions + assert distribution.metadata["Version"] == "1.0" + + +def test_depends_on_installed_requirements_that_apply(tmp_path: Path, install: Install) -> None: + install( + tmp_path, + "Name: alpha\nVersion: 1.0\nRequires-Dist: Beta>=1\nRequires-Dist: gamma; python_version < '3'\n" + "Requires-Dist: delta; extra == 'cli'\nRequires-Dist: missing\n", + {}, + ) + for name in ("beta", "gamma", "delta"): + install(tmp_path, f"Name: {name}\nVersion: 1.0\n", {}) + by_name = {distribution.metadata["Name"]: distribution for distribution in facts_from(tmp_path).distributions} + assert by_name["alpha"].depends_on == ["beta"] + + +def test_commit_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PYTHON_SBOM_COMMIT", "c0ffee") + assert default_revision(tmp_path) == Revision(commit="c0ffee", dirty=False) + + +@pytest.fixture +def checkout(tmp_path: Path, git: Git) -> Path: + (tmp_path / "checkout" / "tracked.txt").write_text("one\n", encoding="utf-8") + git("add", "tracked.txt") + git("commit", "--quiet", "--message", "first") + return tmp_path / "checkout" + + +def test_git_revision(checkout: Path, git: Git) -> None: + assert default_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=False) + + +def test_git_revision_dirty(checkout: Path, git: Git) -> None: + (checkout / "tracked.txt").write_text("two\n", encoding="utf-8") + assert git_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=True) + + +def test_untracked_files_leave_revision_clean(checkout: Path) -> None: + (checkout / "untracked.txt").write_text("new\n", encoding="utf-8") + revision = git_revision(checkout) + assert revision is not None + assert not revision.dirty + + +def test_git_revision_from_subdirectory(checkout: Path, git: Git) -> None: + (subdirectory := checkout / "packages" / "one").mkdir(parents=True) + assert git_revision(subdirectory) == Revision(commit=git("rev-parse", "HEAD"), dirty=False) + + +def test_git_revision_before_first_commit(tmp_path: Path, git: Git) -> None: + git("status") + assert git_revision(tmp_path / "checkout") is None + + +def test_git_revision_outside_checkout(tmp_path: Path) -> None: + assert git_revision(tmp_path) is None + + +def test_git_revision_without_git(checkout: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PATH", "") + assert git_revision(checkout) is None + + +def test_git_revision_unreadable_checkout(tmp_path: Path) -> None: + (tmp_path / ".git").write_text(f"gitdir: {tmp_path / 'missing'}\n", encoding="utf-8") + with pytest.raises(SourceRevisionError, match="git rev-parse HEAD failed in"): + git_revision(tmp_path) + + +@pytest.mark.parametrize( + ("archival", "expected"), + [ + pytest.param(f"node: {'a' * 40}\n", Revision(commit="a" * 40, dirty=False), id="expanded"), + pytest.param("node: $Format:%H$\n", None, id="unexpanded"), + ], +) +def test_git_archival(tmp_path: Path, archival: str, expected: Revision | None) -> None: + (tmp_path / ".git_archival.txt").write_text(archival, encoding="utf-8") + assert default_revision(tmp_path) == expected + + +def test_no_revision_source(tmp_path: Path) -> None: + assert default_revision(tmp_path) is None diff --git a/tests/_collect/test_collect_settings.py b/tests/_collect/test_collect_settings.py new file mode 100644 index 0000000..17020c0 --- /dev/null +++ b/tests/_collect/test_collect_settings.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from python_sbom import Settings, SettingsError, settings_from_pyproject, settings_from_table + +if TYPE_CHECKING: + from pathlib import Path + + from python_sbom import Toml + + +def test_missing_pyproject(tmp_path: Path) -> None: + with pytest.raises(SettingsError, match=r"has no pyproject\.toml to read"): + settings_from_pyproject(tmp_path, {}) + + +def test_invalid_pyproject(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text("[tool\n", encoding="utf-8") + with pytest.raises(SettingsError, match=r"pyproject\.toml is not valid TOML"): + settings_from_pyproject(tmp_path, {}) + + +def test_defaults(tmp_path: Path, settings: Settings) -> None: + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'a'\n", encoding="utf-8") + assert settings_from_pyproject(tmp_path, {}) == settings + + +def test_table_with_overrides(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text( + "[tool.python-sbom]\n" + "repository = 'https://github.com/example/example/'\n" + "default-branch = 'dev'\n" + "bundled-wheels = ['src/*.whl']\n" + "supplier = { name = 'Example Org' }\n", + encoding="utf-8", + ) + assert settings_from_pyproject( + tmp_path, {"default-branch": "trunk", "content-provider": "sbom:Content"} + ) == Settings( + supplier={"name": "Example Org", "url": []}, + repository="https://github.com/example/example", + default_branch="trunk", + bundled_wheels=("src/*.whl",), + content_provider="sbom:Content", + ) + + +def test_unknown_option() -> None: + with pytest.raises(SettingsError, match=r"here has unknown option\(s\) bundled, suplier; known: bundled-wheels"): + settings_from_table({"suplier": {}, "bundled": []}, "here") + + +@pytest.mark.parametrize( + ("table", "message"), + [ + pytest.param({"repository": 1}, "`repository` in here must be a string", id="repository"), + pytest.param({"default-branch": 1}, "`default-branch` in here must be a string", id="default-branch"), + pytest.param({"bundled-wheels": "*.whl"}, "`bundled-wheels` in here must be an array", id="bundled-not-list"), + pytest.param({"bundled-wheels": [1]}, "`bundled-wheels` in here must be an array", id="bundled-not-strings"), + pytest.param({"content-provider": 1}, "`content-provider` in here must be a string", id="content-provider"), + pytest.param({"supplier": "Example"}, "`supplier` in here must be a table", id="supplier-not-table"), + pytest.param({"supplier": {"name": 1}}, "`supplier` in here must be a table", id="supplier-name"), + pytest.param({"supplier": {"name": "a", "url": "x"}}, "`supplier` in here must be a table", id="supplier-url"), + pytest.param( + {"supplier": {"name": "a", "email": "x"}}, "`supplier` in here must be a table", id="supplier-key" + ), + ], +) +def test_wrong_type(table: dict[str, Toml], message: str) -> None: + with pytest.raises(SettingsError, match=message): + settings_from_table(table, "here") + + +def test_supplier_with_urls() -> None: + settings = settings_from_table({"supplier": {"name": "Example Org", "url": ["https://example.org"]}}, "here") + assert settings.supplier == {"name": "Example Org", "url": ["https://example.org"]} diff --git a/tests/_collect/test_collect_source.py b/tests/_collect/test_collect_source.py new file mode 100644 index 0000000..aec3235 --- /dev/null +++ b/tests/_collect/test_collect_source.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import dataclasses +import hashlib +from email import message_from_string +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final + +import pytest +from conftest import record_digest + +from python_sbom import BuildScript, RecordRow, SbomRequest, WheelError, read_bundled_wheel, read_source_tree + +if TYPE_CHECKING: + from collections.abc import Mapping + + from conftest import WriteWheel + + from python_sbom import Settings + +_PIP: Final[dict[str, bytes]] = { + "pip/__init__.py": b"", + "pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n", +} + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + pytest.param({}, None, id="no-license-file"), + pytest.param({"LICENSE": "MIT terms\n"}, None, id="no-copyright-line"), + pytest.param( + {"LICENSE": "MIT\n\nCopyright (c) 2026 Example\nCopyright (c) 2025 Other\n"}, + "Copyright (c) 2026 Example", + id="first-copyright-line", + ), + ], +) +def test_copyright(tmp_path: Path, settings: Settings, files: Mapping[str, str], expected: str | None) -> None: + for name, text in files.items(): + (tmp_path / name).write_text(text, encoding="utf-8") + source = read_source_tree(request(tmp_path, settings, "License-File: LICENSE\nLicense-File: NOTICE\n")) + assert source.copyright == expected + + +def request(root: Path, settings: Settings, metadata: str = "", scripts: tuple[PurePosixPath, ...] = ()) -> SbomRequest: + return SbomRequest( + root=root, + metadata=message_from_string(f"Name: example\nVersion: 1.0\n{metadata}"), + wheel_name="example-1.0-py3-none-any.whl", + settings=settings, + build_scripts=scripts, + ) + + +def test_bundled_wheels_sorted_once(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: + for directory in ("b", "a"): + write_wheel(tmp_path / "src" / directory / "pip-1.0-py3-none-any.whl", _PIP) + settings = dataclasses.replace(settings, bundled_wheels=("src/*/*.whl", "src/a/*.whl")) + paths = [wheel.path for wheel in read_source_tree(request(tmp_path, settings)).bundled] + assert paths == ["src/a/pip-1.0-py3-none-any.whl", "src/b/pip-1.0-py3-none-any.whl"] + + +def test_build_scripts_hashed(tmp_path: Path, settings: Settings) -> None: + (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") + script = PurePosixPath("hatch_build.py") + source = read_source_tree(request(tmp_path, settings, scripts=(script,))) + assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] + + +def test_build_script_outside_project(tmp_path: Path, settings: Settings) -> None: + # a custom hook path may be absolute, which on Windows carries a drive the join with the root must keep + (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") + script = PurePosixPath((tmp_path / "hatch_build.py").as_posix()) + source = read_source_tree(request(tmp_path / "project", settings, scripts=(script,))) + assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] + + +def test_bundled_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: + wheel = write_wheel( + tmp_path / "src" / "pip-1.0-py3-none-any.whl", + { + **_PIP, + "pip/_vendor/vendor.txt": b"# pinned\n\nurllib3==1.26.4 # http\n", + "pip/_vendor/rich-13.0.dist-info/METADATA": b"Name: rich\nVersion: 13.0\n", + }, + ) + bundled = read_bundled_wheel(tmp_path, wheel) + assert (bundled.path, bundled.sha256, bundled.metadata_name, bundled.metadata["Name"], bundled.files[0]) == ( + "src/pip-1.0-py3-none-any.whl", + hashlib.sha256(wheel.read_bytes()).hexdigest(), + "pip-1.0.dist-info/METADATA", + "pip", + RecordRow(path="pip/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"").hexdigest(), size="0"), + ) + assert [(source, str(metadata)) for source, metadata in bundled.vendored] == [ + ("pip/_vendor/vendor.txt", "Name: urllib3\nVersion: 1.26.4\n\n"), + ("pip/_vendor/rich-13.0.dist-info/METADATA", "Name: rich\nVersion: 13.0\n\n"), + ] + + +_EMPTY: Final[str] = "47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU" +_METADATA: Final[str] = f"pip-1.0.dist-info/METADATA,sha256={_EMPTY},0\n" + + +@pytest.mark.parametrize( + ("files", "message"), + [ + pytest.param({"pip/__init__.py": b""}, "has no .dist-info/METADATA", id="no-metadata"), + pytest.param({"pip-1.0.dist-info/METADATA": b"Name: pip\n"}, "METADATA lacks Name or Version", id="no-version"), + pytest.param( + {"pip-1.0.dist-info/METADATA": _PIP["pip-1.0.dist-info/METADATA"]}, + "has no pip-1.0.dist-info/RECORD", + id="no-record", + ), + pytest.param( + {"pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n" + b" " * 16 * 1024 * 1024}, + "METADATA is 16777239 bytes, over the 16777216 byte limit", + id="oversized-metadata", + ), + ], +) +def test_malformed_bundled_wheel( + tmp_path: Path, write_wheel: WriteWheel, files: Mapping[str, bytes], message: str +) -> None: + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + with pytest.raises(WheelError, match=message): + read_bundled_wheel(tmp_path, wheel) + + +@pytest.mark.parametrize( + ("record", "message"), + [ + pytest.param("pip/__init__.py,sha256=x\n", "RECORD row 1 has 2 columns, not 3", id="columns"), + pytest.param("pip/__init__.py,blake2b=x,0\n", "RECORD row 1: unknown hash algorithm 'blake2b'", id="algorithm"), + pytest.param("pip/__init__.py,sha256=abcde,0\n", "digest 'abcde' is not urlsafe base64", id="base64"), + pytest.param(_METADATA, "RECORD lacks a SHA-256 for pip/__init__.py", id="member-not-recorded"), + pytest.param( + f"{_METADATA}pip/__init__.py,sha256={_EMPTY},0\npip/gone.py,sha256={_EMPTY},0\n", + "RECORD lists pip/gone.py, which the wheel does not hold", + id="recorded-not-member", + ), + pytest.param( + f"pip/__init__.py,sha256={_EMPTY},0\npip-1.0.dist-info/METADATA,sha256={_EMPTY},0\n", + "the SHA-256 in RECORD differs from the content of pip-1.0.dist-info/METADATA", + id="digest-mismatch", + ), + ], +) +def test_record_disagrees_with_wheel(tmp_path: Path, write_wheel: WriteWheel, record: str, message: str) -> None: + files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + with pytest.raises(WheelError, match=message): + read_bundled_wheel(tmp_path, wheel) + + +@pytest.mark.parametrize( + ("manifest", "message"), + [ + pytest.param(b"urllib3 ==\n", "does not parse: urllib3 ==", id="unparsable"), + pytest.param(b"urllib3>=1\n", "pins no one exact version: urllib3>=1", id="range"), + pytest.param(b"urllib3==1.*\n", "pins no one exact version: urllib3==1.*", id="wildcard"), + pytest.param(b"urllib3==1,!=1.1\n", "pins no one exact version: urllib3==1,!=1.1", id="two-specifiers"), + ], +) +def test_vendor_manifest_needs_exact_pins( + tmp_path: Path, write_wheel: WriteWheel, manifest: bytes, message: str +) -> None: + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", {**_PIP, "pip/_vendor/vendor.txt": manifest}) + with pytest.raises(WheelError, match=message): + read_bundled_wheel(tmp_path, wheel) + + +def test_record_blank_lines_are_skipped(tmp_path: Path, write_wheel: WriteWheel) -> None: + metadata = _PIP["pip-1.0.dist-info/METADATA"] + record = ( + f"pip/__init__.py,sha256={_EMPTY},0\n\n" + f"pip-1.0.dist-info/METADATA,{record_digest(metadata)},{len(metadata)}\n" + "pip-1.0.dist-info/RECORD,,\n" + ) + wheel = write_wheel(tmp_path / "pip.whl", {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()}, record=False) + assert [row.path for row in read_bundled_wheel(tmp_path, wheel).files] == [ + "pip/__init__.py", + "pip-1.0.dist-info/METADATA", + ] diff --git a/tests/_formats/test_formats_spdx.py b/tests/_formats/test_formats_spdx.py new file mode 100644 index 0000000..c20574f --- /dev/null +++ b/tests/_formats/test_formats_spdx.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import dataclasses +import re +from typing import TYPE_CHECKING + +import pytest + +from python_sbom import to_spdx + +if TYPE_CHECKING: + from conftest import Describe + + from python_sbom import Component, Settings + + +def test_minimal(describe: Describe) -> None: + document = describe() + spdx = to_spdx(document, "1.2.3") + ids = {package["name"]: package["SPDXID"] for package in spdx["packages"]} + serial = document["serialNumber"] + assert spdx == { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "Example-1.0", + "documentNamespace": f"https://spdx.org/spdxdocs/sboms/Example-1.0-{serial.removeprefix('urn:uuid:')}", + "comment": f"Rendered from the CycloneDX SBOM {serial} (example.cdx.json), which also carries per-file hashes" + " and the build formulation.", + "creationInfo": {"created": "2023-11-14T22:13:20Z", "creators": ["Tool: python-sbom-1.2.3"]}, + "packages": [ + { + "SPDXID": ids["Example"], + "name": "Example", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "primaryPackagePurpose": "APPLICATION", + "versionInfo": "1.0", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:pypi/example@1.0", + } + ], + }, + { + "SPDXID": ids["cpython"], + "name": "cpython", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "primaryPackagePurpose": "OTHER", + "versionInfo": "3.15.0", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython@3.15.0", + } + ], + }, + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": ids["Example"], + }, + { + "spdxElementId": ids["cpython"], + "relationshipType": "BUILD_TOOL_OF", + "relatedSpdxElement": ids["Example"], + }, + ], + } + + +def test_spdx_ids_are_valid_and_tell_similar_refs_apart(describe: Describe) -> None: + document = describe("Requires-Dist: click; os_name == 'nt'\nRequires-Dist: click; os_name != 'nt'\n") + ids = [package["SPDXID"] for package in to_spdx(document, "1")["packages"]] + assert all(re.fullmatch(r"SPDXRef-[A-Za-z0-9.-]+", spdx_id) for spdx_id in ids) + assert len(set(ids)) == len(ids) + + +def test_repository_names_the_namespace(describe: Describe, settings: Settings) -> None: + document = describe(settings=dataclasses.replace(settings, repository="https://github.com/example/example")) + namespace = to_spdx(document, "1")["documentNamespace"] + assert namespace.startswith("https://github.com/example/example/sboms/Example-1.0-") + + +def test_supplier_and_application_tools_are_creators(describe: Describe, settings: Settings) -> None: + document = describe(settings=dataclasses.replace(settings, supplier={"name": "Example Org", "url": []})) + document["metadata"]["tools"]["components"].extend([ + {"type": "application", "bom-ref": "tool:hatchling", "name": "hatchling", "version": "1.28"}, + {"type": "application", "bom-ref": "tool:script", "name": "script"}, + ]) + assert to_spdx(document, "1")["creationInfo"]["creators"] == [ + "Organization: Example Org", + "Tool: hatchling-1.28", + "Tool: script", + "Tool: python-sbom-1", + ] + + +def test_versionless_root(describe: Describe) -> None: + document = describe() + del document["metadata"]["component"]["version"] + spdx = to_spdx(document, "1") + assert spdx["name"] == "Example" + assert "versionInfo" not in spdx["packages"][0] + + +def test_package_fields(describe: Describe) -> None: + document = describe() + component: Component = { + "type": "library", + "bom-ref": "bundled:a.whl", + "name": "a", + "supplier": {"name": "PyPA", "url": []}, + "hashes": [{"alg": "SHA-256", "content": "ab" * 32}], + "externalReferences": [{"type": "website", "url": "https://example.org"}], + "copyright": "Copyright (c) PyPA", + "description": "a library", + "licenses": [ + {"license": {"name": "BSD License", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ], + "properties": [{"name": "example:bundled-wheel", "value": "a.whl"}, {"name": "python:keywords", "value": "a"}], + "components": [{"type": "file", "bom-ref": "bundled:a.whl#a.py", "name": "a.py"}], + } + document["components"].append(component) + (package,) = (package for package in to_spdx(document, "1")["packages"] if package["name"] == "a") + assert {key: value for key, value in package.items() if key != "SPDXID"} == { + "name": "a", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "licenseConcluded": "MIT", + "licenseDeclared": "NOASSERTION", + "licenseComments": "Declared in package metadata as: BSD License", + "primaryPackagePurpose": "LIBRARY", + "supplier": "Organization: PyPA", + "checksums": [{"algorithm": "SHA256", "checksumValue": "ab" * 32}], + "homepage": "https://example.org", + "copyrightText": "Copyright (c) PyPA", + "summary": "a library", + "comment": "example:bundled-wheel: a.whl", + } + + +def test_license_refs_carry_extracted_text(describe: Describe) -> None: + document = describe("License-Expression: MIT AND LicenseRef-Vendor\n") + assert to_spdx(document, "1").get("hasExtractedLicensingInfos") == [ + { + "licenseId": "LicenseRef-Vendor", + "extractedText": "LicenseRef-Vendor, as named in package metadata", + "name": "LicenseRef-Vendor", + } + ] + + +def test_nested_libraries_are_contained(describe: Describe) -> None: + document = describe() + document["components"].append({ + "type": "library", + "bom-ref": "outer", + "name": "outer", + "components": [{"type": "library", "bom-ref": "inner", "name": "inner"}], + }) + spdx = to_spdx(document, "1") + ids = {package["name"]: package["SPDXID"] for package in spdx["packages"]} + assert {"spdxElementId": ids["outer"], "relationshipType": "CONTAINS", "relatedSpdxElement": ids["inner"]} in spdx[ + "relationships" + ] + + +@pytest.mark.parametrize( + ("kind", "purpose"), + [ + pytest.param("application", "APPLICATION", id="application"), + pytest.param("library", "LIBRARY", id="library"), + pytest.param("framework", "OTHER", id="other"), + ], +) +def test_package_purpose(describe: Describe, kind: str, purpose: str) -> None: + document = describe() + document["components"].append({"type": kind, "bom-ref": "extra", "name": "extra"}) + (package,) = (package for package in to_spdx(document, "1")["packages"] if package["name"] == "extra") + assert package["primaryPackagePurpose"] == purpose diff --git a/tests/_generate/test_generate_backend.py b/tests/_generate/test_generate_backend.py new file mode 100644 index 0000000..2bd64e0 --- /dev/null +++ b/tests/_generate/test_generate_backend.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +import pytest + +from python_sbom import BuiltWheel, MetadataBackend, WheelError + +if TYPE_CHECKING: + from conftest import WriteWheel + + from python_sbom import Settings + + +def test_metadata_backend(tmp_path: Path, settings: Settings) -> None: + backend = MetadataBackend( + root=tmp_path, + metadata="Name: example\nVersion: 1.0\n", + wheel_name="example-1.0-py3-none-any.whl", + settings=settings, + build_scripts=(PurePosixPath("hatch_build.py"),), + ) + request = backend.request() + assert (request.root, request.metadata["Name"], request.wheel_name, request.settings, request.build_scripts) == ( + tmp_path, + "example", + "example-1.0-py3-none-any.whl", + settings, + (PurePosixPath("hatch_build.py"),), + ) + + +def test_built_wheel(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: + wheel = write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"Name: native\n"}) + request = BuiltWheel(wheel=wheel, root=tmp_path, settings=settings, build_scripts=()).request() + assert (request.metadata["Name"], request.wheel_name) == ("native", "native-1.0-py3-none-any.whl") + + +def test_built_wheel_needs_a_wheel(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: + archive = write_wheel(tmp_path / "native.whl", {"native/__init__.py": b""}, record=False) + with pytest.raises(WheelError, match=r"has no \.dist-info/METADATA, so it is not a wheel"): + BuiltWheel(wheel=archive, root=Path(), settings=settings, build_scripts=()).request() diff --git a/tests/_generate/test_generate_content.py b/tests/_generate/test_generate_content.py new file mode 100644 index 0000000..f8a4b7e --- /dev/null +++ b/tests/_generate/test_generate_content.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from email import message_from_string +from pathlib import Path, PurePosixPath + +import pytest + +from python_sbom import ( + BundledWheel, + BundledWheelInfo, + ContentProvider, + Contributions, + SbomRequest, + Settings, + SettingsError, + load_content_provider, +) + + +def test_default_provider_adds_nothing_to_bundled_wheels() -> None: + wheel = BundledWheel( + path="a.whl", sha256="", metadata_name="", metadata=message_from_string(""), files=[], vendored=[] + ) + assert ContentProvider().bundled_wheel(wheel) == BundledWheelInfo(properties=[], supplier=None) + + +def test_default_provider_contributes_nothing(settings: Settings) -> None: + request = SbomRequest( + root=Path(), metadata=message_from_string(""), wheel_name="", settings=settings, build_scripts=() + ) + assert ContentProvider().contributions(request) == Contributions( + components=[], dependencies=[], root_properties=[], tools=[] + ) + + +@pytest.fixture +def module(tmp_path: Path) -> str: + # a name unique to the test, so the import cache never hands over another test's module + (tmp_path / "tools").mkdir() + provider = """\ +from python_sbom import ContentProvider + + +class Content(ContentProvider): + pass + + +instance = Content() +other = object() +""" + (tmp_path / "tools" / f"{tmp_path.name}.py").write_text(provider, encoding="utf-8") + return f"tools.{tmp_path.name}" + + +@pytest.mark.parametrize("attribute", [pytest.param("Content", id="class"), pytest.param("instance", id="instance")]) +def test_load_from_project(tmp_path: Path, module: str, attribute: str) -> None: + provider, script = load_content_provider(f"{module}:{attribute}", tmp_path) + assert type(provider).__name__ == "Content" + assert script == PurePosixPath(f"tools/{tmp_path.name}.py") + + +def test_load_from_outside_project(tmp_path: Path) -> None: + provider, script = load_content_provider("python_sbom:ContentProvider", tmp_path) + assert type(provider) is ContentProvider + assert script is None + + +@pytest.mark.parametrize( + ("spec", "message"), + [ + pytest.param("tools", "must be `module:attribute`", id="no-attribute"), + pytest.param(":Content", "must be `module:attribute`", id="no-module"), + pytest.param("not_a_module_anywhere:Content", "does not import from", id="import-fails"), + pytest.param("MODULE:other", "is not a ContentProvider subclass or instance", id="not-a-provider"), + pytest.param("MODULE:missing", "is not a ContentProvider subclass or instance", id="missing-attribute"), + ], +) +def test_load_refuses(tmp_path: Path, module: str, spec: str, message: str) -> None: + with pytest.raises(SettingsError, match=message): + load_content_provider(spec.replace("MODULE", module), tmp_path) diff --git a/tests/_model/test_model_bom.py b/tests/_model/test_model_bom.py new file mode 100644 index 0000000..b2eab19 --- /dev/null +++ b/tests/_model/test_model_bom.py @@ -0,0 +1,379 @@ +from __future__ import annotations + +import dataclasses +import json +import uuid +from email import message_from_string +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom import ( + BuildScript, + BundledWheel, + BundledWheelInfo, + ContentProvider, + Contributions, + Revision, + SbomRequest, + SourceTree, + build_sbom, + dumps, +) + +if TYPE_CHECKING: + from conftest import Describe + + from python_sbom import BuildFacts, Component, OrganizationalEntity, Settings + +_INTERPRETER: Final[str] = "tool:pkg:generic/cpython@3.15.0" + + +def test_minimal_document(describe: Describe) -> None: + document = describe() + body = {key: value for key, value in document.items() if key not in {"$schema", "bomFormat", "specVersion"}} + serial = body.pop("serialNumber") + del body["version"] + expected_body = { + "metadata": { + "timestamp": "2023-11-14T22:13:20+00:00", + "lifecycles": [{"phase": "build"}], + "tools": { + "components": [ + { + "type": "platform", + "bom-ref": _INTERPRETER, + "name": "cpython", + "version": "3.15.0", + "purl": "pkg:generic/cpython@3.15.0", + "properties": [{"name": "python:implementation", "value": "CPython"}], + } + ] + }, + "component": { + "type": "application", + "bom-ref": "pkg:pypi/example@1.0", + "name": "Example", + "version": "1.0", + "purl": "pkg:pypi/example@1.0", + "externalReferences": [ + {"type": "distribution", "url": "https://pypi.org/project/Example/1.0/"}, + { + "type": "attestation", + "url": "https://pypi.org/integrity/Example/1.0/example-1.0-py3-none-any.whl/provenance", + }, + ], + "properties": [], + }, + "properties": [ + { + "name": "example:sbom:scope", + "value": "Components are the direct runtime dependencies declared in the wheel's core metadata," + " which carry no version because the installer resolves them.", + }, + {"name": "example:sbom:file", "value": "example.cdx.json"}, + ], + }, + "components": [], + "dependencies": [{"ref": "pkg:pypi/example@1.0", "dependsOn": []}], + "compositions": [{"aggregate": "incomplete", "dependencies": ["pkg:pypi/example@1.0"]}], + "formulation": [ + { + "bom-ref": "formula:wheel-build", + "workflows": [ + { + "bom-ref": "workflow:wheel-build", + "uid": "wheel-build", + "name": "build the wheel and this SBOM", + "taskTypes": ["build"], + "resourceReferences": [{"ref": _INTERPRETER}], + "outputs": [{"type": "artifact", "resource": {"ref": "pkg:pypi/example@1.0"}}], + } + ], + } + ], + } + assert body == expected_body + namespace = uuid.uuid5(uuid.NAMESPACE_URL, "https://pypi.org/project/example/sboms") + assert serial == f"urn:uuid:{uuid.uuid5(namespace, json.dumps(expected_body, sort_keys=True))}" + + +def test_header(describe: Describe) -> None: + document = describe() + header = {key: document[key] for key in ("$schema", "bomFormat", "specVersion", "version")} + assert header == { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + } + + +def test_serial_is_stable(describe: Describe) -> None: + assert describe()["serialNumber"] == describe()["serialNumber"] + + +def test_serial_follows_content(describe: Describe) -> None: + assert describe()["serialNumber"] != describe("Summary: other\n")["serialNumber"] + + +def test_sbom_file_name_and_content(settings: Settings, facts: BuildFacts) -> None: + request = SbomRequest( + root=Path(), + metadata=message_from_string("Name: My.Project\nVersion: 1.0\n"), + wheel_name="my_project-1.0-py3-none-any.whl", + settings=settings, + build_scripts=(), + ) + sbom = build_sbom(request, SourceTree(copyright=None, bundled=[], build_scripts=[]), facts, ContentProvider()) + assert (sbom.file_name, sbom.content) == ("my-project.cdx.json", dumps(sbom.document)) + + +@pytest.mark.parametrize( + ("dirty", "trail"), + [ + pytest.param(False, ("exact source revision", []), id="clean"), + pytest.param( + True, + ("source revision, with uncommitted changes", [{"name": "example:vcs-dirty", "value": "true"}]), + id="dirty", + ), + ], +) +def test_root_component( + describe: Describe, settings: Settings, facts: BuildFacts, dirty: bool, trail: tuple[str, list[dict[str, str]]] +) -> None: + comment, dirty_properties = trail + supplier: OrganizationalEntity = {"name": "Example Org", "url": ["https://example.org"]} + document = describe( + "License-File: LICENSE\nProject-URL: Source, https://github.com/example/example/\n", + settings=dataclasses.replace(settings, supplier=supplier), + facts=dataclasses.replace(facts, revision=Revision(commit="c0ffee", dirty=dirty)), + source=SourceTree(copyright="Copyright (c) 2026 Example", bundled=[], build_scripts=[]), + ) + repository = "https://github.com/example/example" + assert document["metadata"]["component"] == { + "type": "application", + "bom-ref": "pkg:pypi/example@1.0", + "name": "Example", + "version": "1.0", + "purl": "pkg:pypi/example@1.0", + "supplier": supplier, + "copyright": "Copyright (c) 2026 Example", + "externalReferences": [ + {"type": "vcs", "url": f"{repository}/", "comment": "Project-URL: Source"}, + {"type": "distribution", "url": "https://pypi.org/project/Example/1.0/"}, + { + "type": "attestation", + "url": "https://pypi.org/integrity/Example/1.0/example-1.0-py3-none-any.whl/provenance", + }, + {"type": "security-contact", "url": f"{repository}/security/policy"}, + {"type": "advisories", "url": f"{repository}/security/advisories"}, + {"type": "license", "url": f"{repository}/blob/main/LICENSE"}, + {"type": "vcs", "url": f"{repository}/tree/c0ffee", "comment": comment}, + ], + "properties": [{"name": "example:vcs-commit", "value": "c0ffee"}, *dirty_properties], + } + + +def test_supplier_makes_and_supplies_the_document(describe: Describe, settings: Settings) -> None: + supplier: OrganizationalEntity = {"name": "Example Org", "url": []} + metadata = describe(settings=dataclasses.replace(settings, supplier=supplier))["metadata"] + assert metadata.get("manufacturer") == supplier + assert metadata.get("supplier") == supplier + + +def test_metadata_repeats_root_authors_and_licenses(describe: Describe) -> None: + metadata = describe("Author: Author\nLicense-Expression: MIT\n")["metadata"] + assert metadata.get("authors") == [{"name": "Author"}] + assert metadata.get("licenses") == [{"expression": "MIT", "acknowledgement": "declared"}] + + +def test_root_licenses_from_classifiers(describe: Describe) -> None: + document = describe("Classifier: License :: OSI Approved :: MIT License\n") + assert document["metadata"]["component"].get("licenses") == [ + {"license": {"name": "MIT License", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ] + + +def test_repository_setting_wins_over_project_urls(describe: Describe, settings: Settings) -> None: + document = describe( + "Project-URL: Source, https://github.com/example/example\n", + settings=dataclasses.replace(settings, repository="https://git.example.org/example"), + ) + assert {"name": "example:sbom:repository", "value": "https://git.example.org/example"} in document["metadata"][ + "properties" + ] + + +def test_no_security_links_outside_github(describe: Describe, settings: Settings) -> None: + document = describe(settings=dataclasses.replace(settings, repository="https://git.example.org/example")) + types = {reference["type"] for reference in document["metadata"]["component"]["externalReferences"]} + assert types.isdisjoint({"security-contact", "advisories"}) + + +def test_no_license_link_without_repository(describe: Describe) -> None: + document = describe("License-File: LICENSE\n") + types = {reference["type"] for reference in document["metadata"]["component"]["externalReferences"]} + assert "license" not in types + + +def test_revision_without_repository_records_commit_only(describe: Describe, facts: BuildFacts) -> None: + document = describe(facts=dataclasses.replace(facts, revision=Revision(commit="c0ffee", dirty=False))) + root = document["metadata"]["component"] + assert root["properties"] == [{"name": "example:vcs-commit", "value": "c0ffee"}] + assert [reference["type"] for reference in root["externalReferences"]] == ["distribution", "attestation"] + + +def test_declared_dependency(describe: Describe) -> None: + requirement = 'tomli>=2; python_version < "3.11"' + document = describe(f"Requires-Dist: {requirement}\n") + assert document["components"] == [ + { + "type": "library", + "bom-ref": f"requires-dist:{requirement}", + "name": "tomli", + "purl": "pkg:pypi/tomli", + "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/tomli/"}], + "properties": [ + {"name": "example:requires-dist", "value": requirement}, + {"name": "example:environment-marker", "value": 'python_version < "3.11"'}, + ], + } + ] + + +def test_repeated_requirement_is_one_component(describe: Describe) -> None: + document = describe("Requires-Dist: packaging>=24\nRequires-Dist: packaging>=24\n") + assert [component["bom-ref"] for component in document["components"]] == ["requires-dist:packaging>=24"] + + +@pytest.mark.parametrize( + ("marker", "applies"), + [ + pytest.param('extra == "cli"', False, id="extra"), + pytest.param('"cli" == extra', False, id="extra-on-the-right"), + pytest.param('extra != "cli"', True, id="not-extra"), + pytest.param('python_version < "3.11"', True, id="environment"), + pytest.param('extra == "cli" or python_version < "3.11"', True, id="extra-or-environment"), + pytest.param('extra == "cli" and python_version < "3.11"', False, id="extra-and-environment"), + pytest.param('(extra == "a" or extra == "b") and os_name == "nt"', False, id="grouped-extras"), + pytest.param('python_version < "3.11" and (extra == "a" or os_name == "nt")', True, id="grouped-unknown"), + pytest.param('python_version >= "3" or extra == "a"', True, id="environment-or-extra"), + ], +) +def test_requirement_marker(describe: Describe, marker: str, applies: bool) -> None: + document = describe(f"Requires-Dist: click; {marker}\n") + assert bool(document["components"]) is applies + + +def test_scope_names_bundled_directories(describe: Describe, settings: Settings) -> None: + document = describe(settings=dataclasses.replace(settings, bundled_wheels=("src/b/*.whl", "src/a/x/*.whl"))) + assert document["metadata"]["properties"][0]["value"] == ( + "Components are the wheels bundled as data files under src/a/x and src/b and the direct runtime dependencies" + " declared in the wheel's core metadata, which carry no version because the installer resolves them." + ) + + +def test_bundled_wheel_and_its_vendored_packages(describe: Describe) -> None: + wheel = BundledWheel( + path="src/pip-1.0-py3-none-any.whl", + sha256="ab" * 32, + metadata_name="pip-1.0.dist-info/METADATA", + metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + files=[], + vendored=[("pip/_vendor/vendor.txt", message_from_string("Name: urllib3\nVersion: 1.26.4\n"))], + ) + + class Seeded(ContentProvider): + def __init__(self, name: str) -> None: + self._name = name + + def bundled_wheel(self, wheel: BundledWheel) -> BundledWheelInfo: + return BundledWheelInfo(properties=[{"name": self._name, "value": wheel.path}], supplier=None) + + document = describe( + source=SourceTree(copyright=None, bundled=[wheel], build_scripts=[]), content=Seeded("example:seeded") + ) + assert document["components"][0]["properties"][1] == { + "name": "example:seeded", + "value": "src/pip-1.0-py3-none-any.whl", + } + bundled_ref = "bundled:src/pip-1.0-py3-none-any.whl" + assert document["dependencies"][:2] == [ + {"ref": "pkg:pypi/example@1.0", "dependsOn": [bundled_ref]}, + {"ref": bundled_ref, "dependsOn": [f"{bundled_ref}#vendored/pkg:pypi/urllib3@1.26.4"]}, + ] + + +def test_build_script_is_a_linked_tool(describe: Describe, settings: Settings) -> None: + source = SourceTree( + copyright=None, + bundled=[], + build_scripts=[BuildScript(path=PurePosixPath("hatch_build.py"), sha256="ef" * 32)], + ) + document = describe( + settings=dataclasses.replace(settings, repository="https://github.com/example/example"), source=source + ) + assert document["metadata"]["tools"]["components"][0] == { + "type": "application", + "bom-ref": "tool:hatch_build.py", + "name": "hatch_build.py", + "version": "1.0", + "hashes": [{"alg": "SHA-256", "content": "ef" * 32}], + "externalReferences": [{"type": "vcs", "url": "https://github.com/example/example/blob/main/hatch_build.py"}], + } + + +@pytest.mark.parametrize( + ("from_environment", "inputs"), + [ + pytest.param(False, None, id="unset"), + pytest.param( + True, [{"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": "1700000000"}]}], id="from-environment" + ), + ], +) +def test_formulation_names_source_date_epoch_only_when_set( + describe: Describe, facts: BuildFacts, from_environment: bool, inputs: list[dict[str, list[dict[str, str]]]] | None +) -> None: + document = describe(facts=dataclasses.replace(facts, epoch_from_environment=from_environment)) + assert document.get("formulation", [])[0]["workflows"][0].get("inputs") == inputs + + +def test_contributions_join_before_references_and_serial(describe: Describe) -> None: + library: Component = {"type": "library", "bom-ref": "pkg:generic/zlib@1.3", "name": "zlib", "version": "1.3"} + tool: Component = {"type": "application", "bom-ref": "tool:pkg:generic/cmake@4", "name": "cmake", "version": "4"} + + class Native(ContentProvider): + def __init__(self, components: list[Component]) -> None: + self._components = components + + def contributions(self, request: SbomRequest) -> Contributions: + return Contributions( + components=self._components, + dependencies=[{"ref": component["bom-ref"], "dependsOn": []} for component in self._components], + root_properties=[{"name": "example:native", "value": request.wheel_name}], + tools=[tool], + ) + + native = describe(content=Native([library])) + assert ( + native["components"], + native["dependencies"], + native["metadata"]["component"]["properties"], + native["metadata"]["tools"]["components"][-1], + native.get("formulation", [])[0]["workflows"][0]["resourceReferences"], + ) == ( + [library], + [ + {"ref": "pkg:pypi/example@1.0", "dependsOn": ["pkg:generic/zlib@1.3"]}, + {"ref": "pkg:generic/zlib@1.3", "dependsOn": []}, + ], + [{"name": "example:native", "value": "example-1.0-py3-none-any.whl"}], + tool, + [{"ref": _INTERPRETER}, {"ref": "tool:pkg:generic/cmake@4"}], + ) + assert native["serialNumber"] != describe()["serialNumber"] diff --git a/tests/_model/test_model_components.py b/tests/_model/test_model_components.py new file mode 100644 index 0000000..8455a69 --- /dev/null +++ b/tests/_model/test_model_components.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +import hashlib +from email import message_from_string +from pathlib import PurePosixPath +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom import ( + BundledWheel, + BundledWheelInfo, + RecordRow, + bundled_wheel_component, + component_from_metadata, + file_component_from_bytes, + script_tool, +) + +if TYPE_CHECKING: + from python_sbom import Component, License, OrganizationalEntity + +_WHEEL: Final[str] = "src/example/wheels/pip-1.0-py3-none-any.whl" + + +def test_component_from_metadata() -> None: + metadata = message_from_string( + "Name: Example\nVersion: 1.0\nSummary: an example\nMaintainer-email: Maintainer \n" + "License-Expression: MIT\nRequires-Python: >=3.11\nRequires-Dist: packaging>=24\nProvides-Extra: cli\n" + "Classifier: Programming Language :: Python :: 3\nKeywords: sample,demo\n" + "Project-URL: Source, https://github.com/example/example\n" + ) + assert component_from_metadata(metadata) == { + "type": "library", + "bom-ref": "pkg:pypi/example@1.0", + "name": "Example", + "version": "1.0", + "purl": "pkg:pypi/example@1.0", + "description": "an example", + "authors": [{"name": "Maintainer", "email": "maintainer@example.com"}], + "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], + "externalReferences": [ + {"type": "vcs", "url": "https://github.com/example/example", "comment": "Project-URL: Source"} + ], + "properties": [ + {"name": "python:requires-python", "value": ">=3.11"}, + {"name": "python:requires-dist", "value": "packaging>=24"}, + {"name": "python:provides-extra", "value": "cli"}, + {"name": "python:classifier", "value": "Programming Language :: Python :: 3"}, + {"name": "python:keywords", "value": "sample,demo"}, + ], + } + + +@pytest.mark.parametrize( + ("fields", "expected"), + [ + pytest.param("Author: Author\nMaintainer: Maintainer\n", [{"name": "Maintainer"}], id="maintainer-wins"), + pytest.param( + "Author-email: Author , other@example.com\n", + [{"name": "Author", "email": "author@example.com"}, {"email": "other@example.com"}], + id="authors-without-maintainer", + ), + pytest.param( + "Author-email: Jane Doe \n", [{"name": "Jane Doe jane at example.com"}], id="bad-email" + ), + ], +) +def test_component_authors(fields: str, expected: list[dict[str, str]]) -> None: + assert component_from_metadata(message_from_string(f"Name: a\nVersion: 1\n{fields}"))["authors"] == expected + + +def test_component_without_authors_has_no_key() -> None: + assert "authors" not in component_from_metadata(message_from_string("Name: a\nVersion: 1\n")) + + +@pytest.mark.parametrize( + ("fields", "expected"), + [ + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\n", + [ + {"license": {"name": "MIT License", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ], + id="classifier", + ), + pytest.param( + "License: apache-2.0\n", + [ + {"license": {"name": "apache-2.0", "acknowledgement": "declared"}}, + {"license": {"id": "Apache-2.0", "acknowledgement": "concluded"}}, + ], + id="license-field", + ), + pytest.param( + "Classifier: License :: OSI Approved :: MIT License\nLicense: MIT\n", + [ + {"license": {"name": "MIT License", "acknowledgement": "declared"}}, + {"license": {"name": "MIT", "acknowledgement": "declared"}}, + {"license": {"id": "MIT", "acknowledgement": "concluded"}}, + ], + id="same-id-once", + ), + pytest.param( + "Classifier: License :: OSI Approved :: BSD License\nLicense: MIT\n", + [ + {"license": {"name": "BSD License", "acknowledgement": "declared"}}, + {"license": {"name": "MIT", "acknowledgement": "declared"}}, + ], + id="one-unresolved-name-concludes-nothing", + ), + pytest.param( + "License: LicenseRef-Proprietary\n", + [{"license": {"name": "LicenseRef-Proprietary", "acknowledgement": "declared"}}], + id="license-ref", + ), + pytest.param( + "License: MIT OR Apache-2.0\n", + [{"license": {"name": "MIT OR Apache-2.0", "acknowledgement": "declared"}}], + id="expression-in-license-field", + ), + pytest.param( + "License: not a license\n", + [{"license": {"name": "not a license", "acknowledgement": "declared"}}], + id="invalid-expression", + ), + ], +) +def test_component_licenses(fields: str, expected: list[License]) -> None: + assert component_from_metadata(message_from_string(f"Name: a\nVersion: 1\n{fields}"))["licenses"] == expected + + +def test_component_ignores_license_text() -> None: + metadata = message_from_string("Name: a\nVersion: 1\nLicense: Copyright\n the full text\n") + assert "licenses" not in component_from_metadata(metadata) + + +@pytest.mark.parametrize( + ("fields", "expected"), + [ + pytest.param("Home-page: UNKNOWN\n", [], id="unknown-home-page"), + pytest.param( + "Home-page: https://example.com\n", [{"type": "website", "url": "https://example.com"}], id="home-page" + ), + pytest.param("Project-URL: no comma\n", [], id="url-without-label"), + pytest.param( + "Project-URL: Bug Tracker, https://example.com/issues\n", + [{"type": "issue-tracker", "url": "https://example.com/issues", "comment": "Project-URL: Bug Tracker"}], + id="normalized-label", + ), + pytest.param( + "Project-URL: Funding, https://example.com/fund\n", + [{"type": "other", "url": "https://example.com/fund", "comment": "Project-URL: Funding"}], + id="unknown-label", + ), + ], +) +def test_component_references(fields: str, expected: list[dict[str, str]]) -> None: + component = component_from_metadata(message_from_string(f"Name: a\nVersion: 1\n{fields}")) + assert component["externalReferences"] == expected + + +def test_bundled_wheel_component() -> None: + wheel = BundledWheel( + path=_WHEEL, + sha256="ab" * 32, + metadata_name="pip-1.0.dist-info/METADATA", + metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + files=[RecordRow(path="pip/__init__.py", algorithm="SHA-256", digest="cd" * 32, size="0")], + vendored=[], + ) + supplier: OrganizationalEntity = {"name": "PyPA", "url": ["https://www.pypa.io"]} + info = BundledWheelInfo(properties=[{"name": "example:seeded-for-python", "value": "3.15"}], supplier=supplier) + assert bundled_wheel_component(wheel, "example", info) == { + "type": "library", + "bom-ref": f"bundled:{_WHEEL}", + "name": "pip", + "version": "1.0", + "purl": "pkg:pypi/pip@1.0", + "supplier": supplier, + "hashes": [{"alg": "SHA-256", "content": "ab" * 32}], + "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/pip/1.0/"}], + "properties": [ + {"name": "example:bundled-wheel", "value": _WHEEL}, + {"name": "example:seeded-for-python", "value": "3.15"}, + ], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [ + {"technique": "manifest-analysis", "confidence": 1, "value": "pip-1.0.dist-info/METADATA"} + ], + }, + { + "field": "hash", + "confidence": 1, + "methods": [{"technique": "hash-comparison", "confidence": 1, "value": "ab" * 32}], + }, + ] + }, + "components": [ + { + "type": "file", + "bom-ref": f"bundled:{_WHEEL}#pip/__init__.py", + "name": "pip/__init__.py", + "hashes": [{"alg": "SHA-256", "content": "cd" * 32}], + "properties": [{"name": "size", "value": "0"}], + } + ], + } + + +def test_bundled_wheel_vendored_metadata_wins_over_manifest() -> None: + manifest = "pip/_vendor/vendor.txt" + dist_info = "pip/_vendor/urllib3-1.26.4.dist-info/METADATA" + wheel = BundledWheel( + path=_WHEEL, + sha256="ab" * 32, + metadata_name="pip-1.0.dist-info/METADATA", + metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + files=[], + vendored=[ + (manifest, message_from_string("Name: urllib3\nVersion: 1.26.4\n")), + (dist_info, message_from_string("Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n")), + ], + ) + component = bundled_wheel_component(wheel, "example", BundledWheelInfo(properties=[], supplier=None)) + assert component["components"] == [ + { + "type": "library", + "bom-ref": f"bundled:{_WHEEL}#vendored/pkg:pypi/urllib3@1.26.4", + "name": "urllib3", + "version": "1.26.4", + "purl": "pkg:pypi/urllib3@1.26.4", + "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], + "externalReferences": [], + "properties": [{"name": "example:vendored-manifest", "value": dist_info}], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": dist_info}], + } + ] + }, + } + ] + + +def test_bundled_wheel_component_ref_tells_copies_apart() -> None: + def component(path: str) -> Component: + wheel = BundledWheel( + path=path, + sha256="ab" * 32, + metadata_name="pip-1.0.dist-info/METADATA", + metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + files=[], + vendored=[], + ) + return bundled_wheel_component(wheel, "example", BundledWheelInfo(properties=[], supplier=None)) + + refs = {component(f"src/{directory}/pip-1.0-py3-none-any.whl")["bom-ref"] for directory in ("a", "b")} + assert refs == {"bundled:src/a/pip-1.0-py3-none-any.whl", "bundled:src/b/pip-1.0-py3-none-any.whl"} + + +def test_bundled_wheel_supplier_never_comes_from_the_wheel() -> None: + wheel = BundledWheel( + path=_WHEEL, + sha256="ab" * 32, + metadata_name="pip-1.0.dist-info/METADATA", + metadata=message_from_string("Name: pip\nVersion: 1.0\nProject-URL: Source, https://github.com/example/pip\n"), + files=[], + vendored=[], + ) + component = bundled_wheel_component(wheel, "example", BundledWheelInfo(properties=[], supplier=None)) + assert "supplier" not in component + + +def test_file_component_from_bytes() -> None: + assert file_component_from_bytes("app", "a.py", b"abc") == { + "type": "file", + "bom-ref": "app#a.py", + "name": "a.py", + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(b"abc").hexdigest()}], + "properties": [{"name": "size", "value": "3"}], + } + + +def test_script_tool_with_link() -> None: + tool = script_tool(PurePosixPath("build/hook.py"), "ef" * 32, "1.0", "https://example.com/hook.py") + assert tool == { + "type": "application", + "bom-ref": "tool:build/hook.py", + "name": "hook.py", + "version": "1.0", + "hashes": [{"alg": "SHA-256", "content": "ef" * 32}], + "externalReferences": [{"type": "vcs", "url": "https://example.com/hook.py"}], + } + + +def test_script_tool_without_link() -> None: + assert "externalReferences" not in script_tool(PurePosixPath("hook.py"), "ef" * 32, "1.0", None) diff --git a/tests/_model/test_model_document.py b/tests/_model/test_model_document.py new file mode 100644 index 0000000..cc38683 --- /dev/null +++ b/tests/_model/test_model_document.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import dataclasses +import json +import uuid +from email import message_from_string +from typing import TYPE_CHECKING + +import pytest + +from python_sbom import ProjectIdentity, document, project_identity + +if TYPE_CHECKING: + from python_sbom import Body, Settings + + +@pytest.mark.parametrize( + ("repository", "urls", "expected"), + [ + pytest.param(None, "", None, id="none"), + pytest.param( + None, + "Project-URL: Docs, https://example.org\nProject-URL: Source, https://github.com/example/example/\n", + "https://github.com/example/example", + id="first-vcs-project-url", + ), + pytest.param( + "https://git.example.org/example", + "Project-URL: Source, https://github.com/example/example\n", + "https://git.example.org/example", + id="setting-wins", + ), + ], +) +def test_project_identity_repository( + settings: Settings, repository: str | None, urls: str, expected: str | None +) -> None: + metadata = message_from_string(f"Name: My_Project\nVersion: 1.0\n{urls}") + identity = project_identity(metadata, dataclasses.replace(settings, repository=repository)) + assert identity == ProjectIdentity( + namespace="my-project", + repository=expected, + serial_namespace=uuid.uuid5(uuid.NAMESPACE_URL, f"{expected or 'https://pypi.org/project/my-project'}/sboms"), + ) + + +def test_document_serial_derives_from_body() -> None: + body: Body = { + "metadata": { + "timestamp": "2023-11-14T22:13:20+00:00", + "lifecycles": [{"phase": "build"}], + "tools": {"components": []}, + "component": {"type": "application", "bom-ref": "app", "name": "app"}, + }, + "components": [], + "dependencies": [], + } + namespace = uuid.uuid5(uuid.NAMESPACE_URL, "https://example.org/sboms") + assert document(body, namespace) == { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": f"urn:uuid:{uuid.uuid5(namespace, json.dumps(body, sort_keys=True))}", + "version": 1, + **body, + } diff --git a/tests/_model/test_model_tools.py b/tests/_model/test_model_tools.py new file mode 100644 index 0000000..4019b03 --- /dev/null +++ b/tests/_model/test_model_tools.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import dataclasses +from email import message_from_string +from typing import TYPE_CHECKING + +from python_sbom import InstalledDistribution, RecordRow, tool_components + +if TYPE_CHECKING: + from python_sbom import BuildFacts + + +def test_tools_list_interpreter_then_distributions_by_name(facts: BuildFacts) -> None: + row = RecordRow(path="zeta/__init__.py", algorithm="SHA-256", digest="ab" * 32, size="0") + distributions = [ + InstalledDistribution(metadata=message_from_string("Name: Zeta\nVersion: 2\n"), files=[row], depends_on=[]), + InstalledDistribution(metadata=message_from_string("Name: alpha\nVersion: 1\n"), files=[], depends_on=["zeta"]), + ] + tools, dependencies = tool_components(dataclasses.replace(facts, distributions=distributions)) + assert tools[1:] == [ + { + "type": "library", + "bom-ref": "tool:pkg:pypi/alpha@1", + "name": "alpha", + "version": "1", + "purl": "pkg:pypi/alpha@1", + "externalReferences": [], + "properties": [], + }, + { + "type": "library", + "bom-ref": "tool:pkg:pypi/zeta@2", + "name": "Zeta", + "version": "2", + "purl": "pkg:pypi/zeta@2", + "externalReferences": [], + "properties": [], + "components": [ + { + "type": "file", + "bom-ref": "tool:pkg:pypi/zeta@2#zeta/__init__.py", + "name": "zeta/__init__.py", + "hashes": [{"alg": "SHA-256", "content": "ab" * 32}], + "properties": [{"name": "size", "value": "0"}], + } + ], + }, + ] + assert dependencies == [ + {"ref": "tool:pkg:pypi/alpha@1", "dependsOn": ["tool:pkg:pypi/zeta@2"]}, + {"ref": "tool:pkg:pypi/zeta@2", "dependsOn": []}, + ] diff --git a/tests/seeder/LICENSE b/tests/_seeder/LICENSE similarity index 100% rename from tests/seeder/LICENSE rename to tests/_seeder/LICENSE diff --git a/tests/_seeder/hatch_build.py b/tests/_seeder/hatch_build.py new file mode 100644 index 0000000..c2877e1 --- /dev/null +++ b/tests/_seeder/hatch_build.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import ast +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +from python_sbom import BundledWheelInfo, ContentProvider +from python_sbom.hatch import SbomBuildHook + +if TYPE_CHECKING: + from python_sbom import BundledWheel, Settings + + +def get_build_hook() -> type[SeedSbomHook]: + # the module also sees the base class, so hatchling needs to be told which subclass to load + return SeedSbomHook + + +class SeedSbomHook(SbomBuildHook): + def content_provider( + self, + settings: Settings, # ruff: ignore[unused-method-argument] # the base class's signature + ) -> tuple[ContentProvider, tuple[PurePosixPath, ...]]: + # no script to add: the SBOM already lists this one, the hook, among the build tools + return BundleSupport(Path(self.root), self.metadata.name), () + + +class BundleSupport(ContentProvider): + def __init__(self, root: Path, project: str) -> None: + self._root = root + self._project = project + + def bundled_wheel(self, wheel: BundledWheel) -> BundledWheelInfo: + # the BUNDLE_SUPPORT table next to the wheels decides which Python versions receive each one; it is read by + # name, without importing the package + path = PurePosixPath(wheel.path) + module = ast.parse((self._root / str(path.parent) / "__init__.py").read_text(encoding="utf-8")) + support: dict[str, dict[str, str]] = ast.literal_eval( + next( + node.value + for node in module.body + if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "BUNDLE_SUPPORT" for target in node.targets) + ) + ) + return BundledWheelInfo( + properties=[ + {"name": f"{self._project}:seeded-for-python", "value": python} + for python, wheels in support.items() + if path.name in wheels.values() + ], + supplier=None, + ) diff --git a/tests/seeder/pyproject.toml b/tests/_seeder/pyproject.toml similarity index 100% rename from tests/seeder/pyproject.toml rename to tests/_seeder/pyproject.toml diff --git a/tests/seeder/src/seeder/__init__.py b/tests/_seeder/src/seeder/__init__.py similarity index 100% rename from tests/seeder/src/seeder/__init__.py rename to tests/_seeder/src/seeder/__init__.py diff --git a/tests/_seeder/src/seeder/embed/__init__.py b/tests/_seeder/src/seeder/embed/__init__.py new file mode 100644 index 0000000..1f40729 --- /dev/null +++ b/tests/_seeder/src/seeder/embed/__init__.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from pathlib import Path + +# the layout of virtualenv's seed/wheels/embed/__init__.py: another assignment comes before the table +BUNDLE_FOLDER = Path(__file__).absolute().parent +BUNDLE_SUPPORT = { + "3.14": {"pip": "pip-1.0-py3-none-any.whl"}, + "3.15": {"pip": "pip-1.0-py3-none-any.whl"}, +} diff --git a/tests/_validate/test_validate_document.py b/tests/_validate/test_validate_document.py new file mode 100644 index 0000000..8be907e --- /dev/null +++ b/tests/_validate/test_validate_document.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom import dumps, validate + +if TYPE_CHECKING: + from conftest import Describe + + from python_sbom import Component, Document, License + +# what auditwheel writes: CycloneDX 1.4 conventions, a legacy tools array, no timestamp +_FOREIGN: Final[str] = json.dumps({ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "version": 1, + "metadata": { + "component": {"type": "library", "bom-ref": "pkg:pypi/plain@1.0", "name": "plain", "version": "1.0"}, + "tools": [{"name": "auditwheel", "version": "6.4.0"}], + }, + "components": [{"type": "library", "bom-ref": "pkg:rpm/almalinux/libfoo@1", "name": "libfoo", "version": "1"}], + "dependencies": [{"ref": "pkg:pypi/plain@1.0", "dependsOn": ["pkg:rpm/almalinux/libfoo@1"]}], +}) +_ALLOWED: Final[frozenset[str]] = frozenset({"MIT", "BSD-3-Clause", "GPL-2.0-only"}) + + +def test_valid(describe: Describe) -> None: + assert validate(dumps(describe("Requires-Dist: click\nLicense-Expression: MIT\n")).decode(), None) == [] + + +def test_schema_violation() -> None: + (problem,) = validate(json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"}), None) + assert problem.startswith("does not conform to the CycloneDX 1.6 schema: ") + + +def test_other_tools_documents_get_the_schema_check_alone() -> None: + assert validate(_FOREIGN, None) == [] + + +def test_missing_fields(describe: Describe) -> None: + # the schema makes the timestamp optional, so the loaded JSON, not the typed document, loses it + loaded = json.loads(dumps(describe())) + del loaded["metadata"]["timestamp"] + assert validate(json.dumps(loaded), None) == [ + "lacks metadata.timestamp, which every document python-sbom writes carries" + ] + + +def test_serial_is_required(describe: Describe) -> None: + # the schema makes it optional, so the loaded JSON, not the typed document, loses it + loaded = json.loads(dumps(describe())) + del loaded["serialNumber"] + assert validate(json.dumps(loaded), None) == ["serialNumber must be a urn:uuid: RFC 4122 UUID, got ''"] + + +def test_duplicate_refs(describe: Describe) -> None: + library: Component = {"type": "library", "bom-ref": "pkg:pypi/example@1.0", "name": "copy"} + assert validate(with_components(describe(), library), None) == [ + "bom-ref values are not unique: ['pkg:pypi/example@1.0']" + ] + + +def with_components(document: Document, *components: Component) -> str: + document["components"].extend(components) + document["dependencies"][0].setdefault("dependsOn", []).extend(component["bom-ref"] for component in components) + return dumps(document).decode() + + +def test_unknown_dependency_refs(describe: Describe) -> None: + document = describe() + document["dependencies"].append({"ref": "ghost", "dependsOn": ["phantom"]}) + assert validate(dumps(document).decode(), None) == [ + "dependencies reference bom-refs that do not exist: ['ghost', 'phantom']" + ] + + +def test_root_must_depend_on_components(describe: Describe) -> None: + document = describe("Requires-Dist: click\n") + document["dependencies"][0]["dependsOn"] = [] + assert validate(dumps(document).decode(), None) == ["root dependsOn [] != components ['requires-dist:click']"] + + +def test_workflow_must_reference_tools(describe: Describe) -> None: + document = describe() + document.get("formulation", [])[0]["workflows"][0]["resourceReferences"].append({"ref": "pkg:pypi/example@1.0"}) + assert validate(dumps(document).decode(), None) == [ + "workflow wheel-build references unknown tools: ['pkg:pypi/example@1.0']" + ] + + +def test_formula_without_workflows(describe: Describe) -> None: + loaded = json.loads(dumps(describe())) + loaded["formulation"] = [{"bom-ref": "formula:other"}] + assert validate(json.dumps(loaded), None) == [] + + +def test_spdx_rendering_must_validate(describe: Describe) -> None: + document = describe() + properties = document["metadata"].get("properties", []) + properties.append({"name": "example:sbom:repository", "value": "https://example.org/a#fragment"}) + (problem,) = validate(dumps(document).decode(), None) + assert problem.startswith("SPDX rendering invalid: ") + + +def test_spdx_rendering_must_parse(describe: Describe) -> None: + raw = with_components(describe(), shipped("broken", [{"expression": "MIT AND", "acknowledgement": "declared"}])) + assert any(problem.startswith("SPDX rendering does not parse: ") for problem in validate(raw, None)) + + +def shipped(name: str, licenses: list[License]) -> Component: + return { + "type": "library", + "bom-ref": f"bundled:{name}.whl", + "name": name, + "hashes": [{"alg": "SHA-256", "content": "ab" * 32}], + "licenses": licenses, + } + + +def test_deep_nesting_is_reported(describe: Describe) -> None: + nested = '{"type": "library", "bom-ref": "deep", "name": "deep", "components": [' * 500 + "]}" * 500 + raw = json.dumps(describe()).replace('"components": []', f'"components": [{nested}]', 1) + assert validate(raw, None) == ["does not conform to the CycloneDX 1.6 schema: it nests too deeply to check"] + + +@pytest.mark.parametrize( + "licenses", + [ + pytest.param([{"expression": "MIT"}], id="expression"), + pytest.param([{"license": {"id": "mit", "acknowledgement": "concluded"}}], id="id-case-insensitive"), + pytest.param([{"expression": "MIT OR GPL-3.0-only"}], id="any-of"), + pytest.param([{"expression": "(GPL-3.0-only OR MIT) AND BSD-3-Clause"}], id="grouped"), + pytest.param([{"expression": "GPL-2.0-only WITH Classpath-exception-2.0"}], id="exception"), + ], +) +def test_license_policy_allows(describe: Describe, licenses: list[License]) -> None: + problems = validate(with_components(describe(), shipped("lib", licenses)), _ALLOWED) + assert [problem for problem in problems if problem.startswith("bundled:lib.whl")] == [] + + +@pytest.mark.parametrize( + ("licenses", "problem"), + [ + pytest.param( + [{"expression": "MIT AND GPL-3.0-only"}], + "license MIT AND GPL-3.0-only is outside the allowed licenses ['BSD-3-Clause', 'GPL-2.0-only', 'MIT']", + id="denied", + ), + pytest.param( + [{"license": {"name": "BSD License", "acknowledgement": "declared"}}], + "declares license names ['BSD License'] with no SPDX id", + id="names", + ), + pytest.param([], "declares no license", id="none"), + pytest.param([{"expression": "MIT AND"}], "license expression 'MIT AND' is malformed: it ends early", id="end"), + pytest.param([{"expression": "(MIT"}], "license expression '(MIT' is malformed: it ends early", id="open"), + pytest.param( + [{"expression": "(MIT ISC)"}], + "license expression '(MIT ISC)' is malformed: expected ')', got 'ISC'", + id="pair", + ), + pytest.param([{"expression": "MIT)"}], "license expression 'MIT)' is malformed: unexpected ')'", id="close"), + pytest.param( + [{"expression": "AND MIT"}], "license expression 'AND MIT' is malformed: unexpected 'AND'", id="op" + ), + pytest.param( + [{"expression": "MIT WITH OR"}], "license expression 'MIT WITH OR' is malformed: unexpected 'OR'", id="with" + ), + ], +) +def test_license_policy_reports(describe: Describe, licenses: list[License], problem: str) -> None: + problems = validate(with_components(describe(), shipped("lib", licenses)), _ALLOWED) + assert f"bundled:lib.whl: {problem}" in problems + + +def test_license_policy_judges_only_shipped_components(describe: Describe) -> None: + declared: Component = {"type": "library", "bom-ref": "requires-dist:click", "name": "click"} + vendored: Component = {"type": "library", "bom-ref": "bundled:outer.whl#vendored", "name": "vendored"} + outer = shipped("outer", [{"expression": "MIT"}]) + outer["components"] = [vendored, {"type": "file", "bom-ref": "outer#a.py", "name": "a.py"}] + assert validate(with_components(describe(), declared, outer), frozenset({"MIT"})) == [] diff --git a/tests/_validate/test_validate_extra.py b/tests/_validate/test_validate_extra.py new file mode 100644 index 0000000..e5029b8 --- /dev/null +++ b/tests/_validate/test_validate_extra.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +import sys + +import pytest + +from python_sbom import MissingExtraError, validate + + +def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch) -> None: + # None in sys.modules makes the import fail, as when the extra is not installed + monkeypatch.setitem(sys.modules, "cyclonedx.schema", None) + monkeypatch.delitem(sys.modules, "python_sbom._validate._schemas", raising=False) + with pytest.raises(MissingExtraError, match=r"pip install 'python-sbom\[validate\]'"): + validate("{}", None) diff --git a/tests/_validate/test_validate_wheel.py b/tests/_validate/test_validate_wheel.py new file mode 100644 index 0000000..4dbb381 --- /dev/null +++ b/tests/_validate/test_validate_wheel.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Final + +from python_sbom import dumps, validate_wheel + +if TYPE_CHECKING: + from pathlib import Path + + from conftest import Describe, WriteWheel + +_METADATA: Final[dict[str, bytes]] = {"example-1.0.dist-info/METADATA": b"Name: Example\nVersion: 1.0\n"} + + +def test_wheel_without_sbom(tmp_path: Path, write_wheel: WriteWheel) -> None: + wheel = write_wheel(tmp_path / "example-1.0-py3-none-any.whl", _METADATA) + assert validate_wheel(wheel, None) == ["no .dist-info/sboms/*.cdx.json found in the wheel"] + + +def test_every_sbom_is_checked(tmp_path: Path, write_wheel: WriteWheel, describe: Describe) -> None: + # the schema makes the serial optional, so the loaded JSON, not the typed document, loses it + broken = json.loads(dumps(describe())) + del broken["serialNumber"] + wheel = write_wheel( + tmp_path / "example-1.0-py3-none-any.whl", + { + **_METADATA, + "example-1.0.dist-info/sboms/example.cdx.json": json.dumps(broken).encode(), + "example-1.0.dist-info/sboms/other.cdx.json": json.dumps({ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + }).encode(), + "example-1.0.dist-info/sboms/notes.txt": b"not an SBOM", + }, + ) + assert validate_wheel(wheel, None) == [ + "example-1.0.dist-info/sboms/example.cdx.json: serialNumber must be a urn:uuid: RFC 4122 UUID, got ''" + ] diff --git a/tests/_wheel/test_wheel_embed.py b/tests/_wheel/test_wheel_embed.py new file mode 100644 index 0000000..7bb084f --- /dev/null +++ b/tests/_wheel/test_wheel_embed.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import stat +import sys +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom import WheelError, embed_sbom + +if TYPE_CHECKING: + from conftest import WriteWheel + from pytest_mock import MockerFixture + +_SBOM: Final[str] = "native-1.0.dist-info/sboms/native.cdx.json" +# the SHA-256 of b"{}\n", urlsafe base64 without padding +_SBOM_ROW: Final[str] = f"{_SBOM},sha256=yj0WO6sFU4GCciYUBWjzvvfqrBh869doeOC2Pp5EI1Y,3" + + +@pytest.fixture +def wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: + return write_wheel( + tmp_path / "dist" / "native-1.0-py3-none-any.whl", + {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"}, + ) + + +def test_embed_returns_member_path(wheel: Path) -> None: + assert embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) == _SBOM + + +def test_embed_puts_sbom_before_record(wheel: Path) -> None: + embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert zipfile.ZipFile(wheel).namelist() == [ + "native/__init__.py", + "native-1.0.dist-info/METADATA", + _SBOM, + "native-1.0.dist-info/RECORD", + ] + + +def test_embed_records_sbom_before_record_row(wheel: Path) -> None: + embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + record = zipfile.ZipFile(wheel).read("native-1.0.dist-info/RECORD").decode().splitlines() + assert record[-2:] == [_SBOM_ROW, "native-1.0.dist-info/RECORD,,"] + + +def test_embed_replaces_existing_sbom(wheel: Path) -> None: + embed_sbom(wheel, "native.cdx.json", b"[]\n", 1_700_000_000) + embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + with zipfile.ZipFile(wheel) as archive: + assert archive.read(_SBOM) == b"{}\n" + assert [ + line for line in archive.read("native-1.0.dist-info/RECORD").decode().splitlines() if "sboms" in line + ] == [_SBOM_ROW] + + +def test_embed_keeps_other_entries(wheel: Path) -> None: + before = {info.filename: (info.date_time, info.external_attr) for info in zipfile.ZipFile(wheel).infolist()} + embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + after = {info.filename: (info.date_time, info.external_attr) for info in zipfile.ZipFile(wheel).infolist()} + assert {name: after[name] for name in before} == before + + +@pytest.mark.parametrize( + ("epoch", "date_time"), + [ + pytest.param(1_700_000_000, (2023, 11, 14, 22, 13, 20), id="epoch"), + pytest.param(0, (1980, 1, 1, 0, 0, 0), id="before-zip-epoch"), + ], +) +def test_embed_entry_is_the_same_on_every_system(wheel: Path, epoch: int, date_time: tuple[int, ...]) -> None: + embed_sbom(wheel, "native.cdx.json", b"{}\n", epoch) + info = zipfile.ZipFile(wheel).getinfo(_SBOM) + assert (info.date_time, info.create_system, info.external_attr) == ( + date_time, + 3, + (stat.S_IFREG | 0o644) << 16, + ) + + +def test_embed_is_deterministic(tmp_path: Path, write_wheel: WriteWheel) -> None: + files = {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"} + first, second = (write_wheel(tmp_path / name / "native-1.0-py3-none-any.whl", files) for name in ("a", "b")) + for wheel in (first, second): + embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert first.read_bytes() == second.read_bytes() + + +def test_embed_keeps_record_line_endings(tmp_path: Path, write_wheel: WriteWheel) -> None: + record = b"native-1.0.dist-info/METADATA,,\r\nnative-1.0.dist-info/RECORD,,\r\n" + wheel = write_wheel( + tmp_path / "native-1.0-py3-none-any.whl", + {"native-1.0.dist-info/METADATA": b"Name: native\n", "native-1.0.dist-info/RECORD": record}, + record=False, + ) + embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert zipfile.ZipFile(wheel).read("native-1.0.dist-info/RECORD") == ( + f"native-1.0.dist-info/METADATA,,\r\n{_SBOM_ROW}\r\nnative-1.0.dist-info/RECORD,,\r\n".encode() + ) + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows has no POSIX permission bits") +def test_embed_keeps_wheel_permissions(wheel: Path) -> None: + wheel.chmod(0o640) + embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert stat.S_IMODE(wheel.stat().st_mode) == 0o640 + + +@pytest.mark.parametrize( + ("files", "message"), + [ + pytest.param({"native/__init__.py": b""}, "has no .dist-info/METADATA, so it is not a wheel", id="no-metadata"), + pytest.param( + {"native-1.0.dist-info/METADATA": b"Name: native\n"}, + "has no native-1.0.dist-info/RECORD", + id="no-record", + ), + ], +) +def test_embed_refuses_non_wheel(tmp_path: Path, files: dict[str, bytes], message: str) -> None: + with zipfile.ZipFile(archive := tmp_path / "native.whl", "w") as handle: + for name, content in files.items(): + handle.writestr(name, content) + with pytest.raises(WheelError, match=message): + embed_sbom(archive, "native.cdx.json", b"{}\n", 1_700_000_000) + + +def test_failed_embed_leaves_wheel_and_no_temporary_file(wheel: Path, mocker: MockerFixture) -> None: + original = wheel.read_bytes() + mocker.patch.object(Path, "replace", autospec=True, side_effect=OSError("disk full")) + with pytest.raises(OSError, match="disk full"): + embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert [path.name for path in wheel.parent.iterdir()] == [wheel.name] + assert wheel.read_bytes() == original diff --git a/tests/conftest.py b/tests/conftest.py index 9cc4e42..c12966a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,222 +1,151 @@ from __future__ import annotations import base64 -import csv import hashlib -import json import shutil +import subprocess import zipfile -from io import StringIO +from email import message_from_string from pathlib import Path -from typing import TYPE_CHECKING, Final, Protocol +from typing import TYPE_CHECKING, Protocol import pytest -from hatchling.builders.wheel import WheelBuilder -from python_sbom import BuildFacts, BuiltWheel, Settings, git_revision +from python_sbom import BuildFacts, ContentProvider, Interpreter, SbomRequest, Settings, SourceTree, build_sbom if TYPE_CHECKING: - from python_sbom import Document + from collections.abc import Mapping -_PROJECT: Final[str] = """\ -[project] -name = "Example" -version = "1.0" -description = "an example" -requires-python = ">=3.11" -license = "MIT" -license-files = ["LICENSE"] -maintainers = [{ name = "Maintainer", email = "maintainer@example.com" }] -keywords = ["sample", "demo"] -classifiers = ["Programming Language :: Python :: 3"] -dependencies = ["packaging>=24", "tomli>=2; python_version<'3.11'"] -optional-dependencies.cli = ["click>=8"] -urls.Changelog = "https://example.com/changelog" -urls.Source = "https://github.com/example/example" -""" -HOOK: Final[str] = "[tool.hatch.build.hooks.python-sbom]\n" -BUNDLED: Final[str] = 'bundled-wheels = ["src/example/wheels/*.whl"]\n' -_SEEDER: Final[Path] = Path(__file__).parent / "seeder" + from python_sbom import Document -class WriteProject(Protocol): - def __call__(self, *, hook: str = HOOK, project: str = _PROJECT) -> Path: ... +class WriteWheel(Protocol): + def __call__(self, wheel: Path, files: Mapping[str, bytes], *, record: bool = True) -> Path: ... -class BundleWheel(Protocol): - def __call__(self, files: dict[str, str], metadata: str = "", name: str = "pip") -> Path: ... +class Install(Protocol): + def __call__( + self, site: Path, metadata: str, files: Mapping[str, bytes], *, purelib: bool = True, direct_url: str = "" + ) -> None: ... -class MakeWheel(Protocol): - def __call__(self, metadata: str) -> Path: ... +class Git(Protocol): + def __call__(self, *arguments: str) -> str: ... -class InstallTool(Protocol): - def __call__(self, site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> None: ... +class Describe(Protocol): + def __call__( + self, + metadata: str = "", + *, + settings: Settings = ..., + facts: BuildFacts = ..., + source: SourceTree | None = None, + content: ContentProvider | None = None, + ) -> Document: ... @pytest.fixture -def root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - # an exported SOURCE_DATE_EPOCH would move every timestamp the tests expect - monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) - (package := tmp_path / "project" / "src" / "example").mkdir(parents=True) - (package / "__init__.py").write_text("", encoding="utf-8") - (tmp_path / "project" / "LICENSE").write_text("Copyright (c) 2026 Example\n\nMIT terms\n", encoding="utf-8") - return tmp_path / "project" +def write_wheel() -> WriteWheel: + def write(wheel: Path, files: Mapping[str, bytes], *, record: bool = True) -> Path: + members = dict(files) + if record: + dist_info = next(name.split("/")[0] for name in files if name.endswith(".dist-info/METADATA")) + rows = "".join(f"{name},{record_digest(content)},{len(content)}\n" for name, content in files.items()) + members[f"{dist_info}/RECORD"] = f"{rows}{dist_info}/RECORD,,\n".encode() + wheel.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in members.items(): + archive.writestr(zipfile.ZipInfo(name, (2024, 1, 1, 0, 0, 0)), content, zipfile.ZIP_DEFLATED) + return wheel + return write -@pytest.fixture -def write_project(root: Path) -> WriteProject: - def write(*, hook: str = HOOK, project: str = _PROJECT) -> Path: - targets = '[tool.hatch.build.targets.wheel]\npackages = ["src/example"]\n' - (root / "pyproject.toml").write_text(f"{project}\n{targets}\n{hook}", encoding="utf-8") - return root - return write +def record_digest(content: bytes) -> str: + return f"sha256={base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b'=').decode()}" @pytest.fixture -def build_sbom(write_project: WriteProject) -> BuildSbom: - def build(*, hook: str = HOOK, project: str = _PROJECT) -> Document: - return build_hook_sbom(write_project(hook=hook, project=project)) +def install() -> Install: + def install( + site: Path, metadata: str, files: Mapping[str, bytes], *, purelib: bool = True, direct_url: str = "" + ) -> None: + name, version = (line.split(": ", 1)[1] for line in metadata.splitlines()[:2]) + dist_info = f"{name}-{version}.dist-info" + members = { + **files, + f"{dist_info}/METADATA": metadata.encode(), + f"{dist_info}/WHEEL": f"Wheel-Version: 1.0\nRoot-Is-Purelib: {str(purelib).lower()}\n".encode(), + } + if direct_url: + members[f"{dist_info}/direct_url.json"] = direct_url.encode() + rows = "".join(f"{path},{record_digest(content)},{len(content)}\n" for path, content in members.items()) + members[f"{dist_info}/RECORD"] = f"{rows}{dist_info}/RECORD,,\n".encode() + for path, content in members.items(): + (target := site / path).parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) - return build + return install -class BuildSbom(Protocol): - def __call__(self, *, hook: str = HOOK, project: str = _PROJECT) -> Document: ... +@pytest.fixture +def git(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Git: + # the user's git configuration, such as commit signing, stays out of the test repository + (config := tmp_path / "gitconfig").write_text("[user]\nname = Test\nemail = test@example.com\n", encoding="utf-8") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(config)) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + (checkout := tmp_path / "checkout").mkdir() + executable = shutil.which("git") + assert executable is not None + def run(*arguments: str) -> str: + return subprocess.run( + [executable, "-C", str(checkout), *arguments], check=True, capture_output=True, text=True + ).stdout.strip() -def build_hook_sbom(root: Path) -> Document: - builder: Final[WheelBuilder] = WheelBuilder(str(root)) - build_data = builder.get_default_build_data() - (hook,) = builder.get_build_hooks(str(root / "dist")).values() - hook.initialize("standard", build_data) - (sbom,) = build_data["sbom_files"] - document: Final[Document] = json.loads(Path(sbom).read_text(encoding="utf-8")) - hook.finalize("standard", build_data, str(root / "dist" / "example.whl")) - return document + run("init", "--quiet") + return run @pytest.fixture -def seeder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) - # a copy, so the build cannot leave files in the source tree; the build script it loads stays the checked-in one - shutil.copytree(_SEEDER, target := tmp_path / "seeder", ignore=shutil.ignore_patterns("hatch_build.py")) - pyproject = (target / "pyproject.toml").read_text(encoding="utf-8") - script = (_SEEDER / "hatch_build.py").as_posix() - (target / "pyproject.toml").write_text( - pyproject.replace( - "build.hooks.custom.supplier", f'build.hooks.custom.path = "{script}"\nbuild.hooks.custom.supplier' - ), - encoding="utf-8", +def facts() -> BuildFacts: + return BuildFacts( + epoch=1_700_000_000, + epoch_from_environment=False, + revision=None, + interpreter=Interpreter(name="cpython", version="3.15.0", implementation="CPython"), + distributions=[], ) - write_seed_wheel(target / "src" / "seeder" / "embed") - return target - - -def write_seed_wheel(directory: Path) -> Path: - # the pip wheel a seeder bundles, built here rather than committed so the repository holds no binaries - files: Final[dict[str, bytes]] = { - "pip/__init__.py": b"", - "pip/_vendor/vendor.txt": b"urllib3==1.26.4\n", - "pip-1.0.dist-info/METADATA": ( - b"Metadata-Version: 2.4\nName: pip\nVersion: 1.0\nLicense-Expression: MIT\n" - b"Project-URL: Source, https://github.com/example/pip\n" - ), - "pip-1.0.dist-info/WHEEL": b"Wheel-Version: 1.0\nGenerator: tests\nRoot-Is-Purelib: true\nTag: py3-none-any\n", - } - record = "".join(f"{name},{digest(content)},{len(content)}\n" for name, content in files.items()) - files["pip-1.0.dist-info/RECORD"] = f"{record}pip-1.0.dist-info/RECORD,,\n".encode() - with zipfile.ZipFile(wheel := directory / "pip-1.0-py3-none-any.whl", "w", zipfile.ZIP_DEFLATED) as archive: - for name, content in files.items(): - archive.writestr(zipfile.ZipInfo(name, (2020, 2, 2, 0, 0, 0)), content, zipfile.ZIP_DEFLATED) - return wheel @pytest.fixture -def bundle_wheel(root: Path) -> BundleWheel: - def bundle(files: dict[str, str], metadata: str = "", name: str = "pip") -> Path: - (wheels := root / "src" / "example" / "wheels").mkdir(parents=True, exist_ok=True) - record: Final[StringIO] = StringIO(newline="") - csv.writer(record).writerows([ - (f"{name}/example.py", "sha256=ungWv48Bz-pBQUDeXa4iI7ADYaOWF3qctBD_YfIAFa0", "3"), - (f"{name}-1.0.dist-info/RECORD", "", ""), - ]) - with zipfile.ZipFile(wheel := wheels / f"{name}-1.0-py3-none-any.whl", "w") as archive: - for path, content in files.items(): - archive.writestr(zipfile.ZipInfo(path), content) - archive.writestr( - zipfile.ZipInfo(f"{name}-1.0.dist-info/METADATA"), - f"Metadata-Version: 2.4\nName: {name}\nVersion: 1.0\n{metadata}", - ) - archive.writestr(zipfile.ZipInfo(f"{name}-1.0.dist-info/RECORD"), record.getvalue()) - archive.writestr(zipfile.ZipInfo(f"{name}/example.py"), b"abc") - return wheel - - return bundle +def settings() -> Settings: + return Settings(supplier=None, repository=None, default_branch="main", bundled_wheels=(), content_provider=None) @pytest.fixture -def make_wheel(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> MakeWheel: - # the layout another build backend, such as meson-python, writes: module first, .dist-info last, RECORD at the end - monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) - - def make(metadata: str) -> Path: - files: Final[dict[str, bytes]] = { - "native/__init__.py": b"", - "native-1.0.dist-info/METADATA": f"Metadata-Version: 2.4\nName: native\nVersion: 1.0\n{metadata}".encode(), - "native-1.0.dist-info/WHEEL": ( - b"Wheel-Version: 1.0\nGenerator: meson\nRoot-Is-Purelib: false\nTag: py3-none-any\n" - ), - } - record = "".join(f"{name},{digest(content)},{len(content)}\n" for name, content in files.items()) - (dist := tmp_path / "dist").mkdir(exist_ok=True) - with zipfile.ZipFile(wheel := dist / "native-1.0-py3-none-any.whl", "w", zipfile.ZIP_DEFLATED) as archive: - for name, content in { - **files, - "native-1.0.dist-info/RECORD": f"{record}native-1.0.dist-info/RECORD,,\n".encode(), - }.items(): - info = zipfile.ZipInfo(name, (2024, 1, 1, 0, 0, 0)) - info.compress_type = zipfile.ZIP_DEFLATED - archive.writestr(info, content) - return wheel - - return make - - -def digest(content: bytes) -> str: - return f"sha256={base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b'=').decode()}" - - -@pytest.fixture -def install_tool() -> InstallTool: - def install(site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> None: - record: Final[StringIO] = StringIO(newline="") - csv.writer(record).writerows((path, digest(content), len(content)) for path, content in files.items()) - for path, content in files.items(): - (target := site / path).parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(content) - (dist_info := site / f"{name}-1.0.dist-info").mkdir(exist_ok=True) - (dist_info / "METADATA").write_text(f"Metadata-Version: 2.4\nName: {name}\nVersion: 1.0\n", encoding="utf-8") - (dist_info / "WHEEL").write_text( - f"Wheel-Version: 1.0\nRoot-Is-Purelib: {str(purelib).lower()}\n", encoding="utf-8" +def describe(settings: Settings, facts: BuildFacts) -> Describe: + def describe( + metadata: str = "", + *, + settings: Settings = settings, + facts: BuildFacts = facts, + source: SourceTree | None = None, + content: ContentProvider | None = None, + ) -> Document: + request = SbomRequest( + root=Path(), + metadata=message_from_string(f"Name: Example\nVersion: 1.0\n{metadata}"), + wheel_name="example-1.0-py3-none-any.whl", + settings=settings, + build_scripts=(), ) - (dist_info / "RECORD").write_bytes(record.getvalue().encode()) - - return install - - -_PLAIN: Final[Settings] = Settings(supplier=None, repository=None, default_branch="main", bundled_wheels=()) - - -class EmbedInto(Protocol): - def __call__(self, wheel: Path, settings: Settings = _PLAIN) -> str: ... - - -@pytest.fixture -def embed_into(tmp_path: Path) -> EmbedInto: - def embed(wheel: Path, settings: Settings = _PLAIN) -> str: - return BuiltWheel(wheel, tmp_path, settings).embed(BuildFacts.collect(tmp_path, git_revision)) - - return embed + return build_sbom( + request, + source or SourceTree(copyright=None, bundled=[], build_scripts=[]), + facts, + content or ContentProvider(), + ).document + + return describe diff --git a/tests/hatch/conftest.py b/tests/hatch/conftest.py new file mode 100644 index 0000000..3bf15bc --- /dev/null +++ b/tests/hatch/conftest.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import shutil +from pathlib import Path +from typing import TYPE_CHECKING, Final, Protocol + +import pytest +from hatchling.builders.sdist import SdistBuilder +from hatchling.builders.wheel import WheelBuilder + +if TYPE_CHECKING: + from tests.conftest import WriteWheel + + +class BuildWheel(Protocol): + def __call__(self, root: Path, *, version: str = "standard", target: str = "wheel") -> Path: ... + + +_SEEDER: Final[Path] = Path(__file__).parent.parent / "_seeder" + + +@pytest.fixture +def project(tmp_path: Path) -> Path: + project = """\ +[project] +name = "Example" +version = "1.0" +description = "an example" +license = "MIT" +license-files = ["LICENSE"] +dependencies = ["packaging>=24"] +urls.Source = "https://github.com/example/example" + +[tool.hatch.build.targets.wheel] +packages = ["src/example"] +""" + (package := tmp_path / "project" / "src" / "example").mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (tmp_path / "project" / "LICENSE").write_text("Copyright (c) 2026 Example\n\nMIT terms\n", encoding="utf-8") + (tmp_path / "project" / "pyproject.toml").write_text( + f"{project}\n[tool.hatch.build.hooks.python-sbom]\n", encoding="utf-8" + ) + return tmp_path / "project" + + +@pytest.fixture +def build_wheel(tmp_path: Path) -> BuildWheel: + def build(root: Path, *, version: str = "standard", target: str = "wheel") -> Path: + builder = (WheelBuilder if target == "wheel" else SdistBuilder)(str(root)) + (artifact,) = builder.build(directory=str(tmp_path / "dist"), versions=[version]) + return Path(artifact) + + return build + + +@pytest.fixture +def seeder(tmp_path: Path, write_wheel: WriteWheel) -> Path: + # a copy, so the build leaves nothing in the source tree; the hook script stays the checked-in one, so coverage + # measures it + shutil.copytree(_SEEDER, target := tmp_path / "seeder", ignore=shutil.ignore_patterns("hatch_build.py")) + pyproject = target / "pyproject.toml" + pyproject.write_text( + pyproject.read_text(encoding="utf-8").replace( + "build.hooks.custom.supplier", + f'build.hooks.custom.path = "{(_SEEDER / "hatch_build.py").as_posix()}"\nbuild.hooks.custom.supplier', + ), + encoding="utf-8", + ) + # the pip wheel a seeder bundles, built here so the repository holds no binaries + write_wheel( + target / "src" / "seeder" / "embed" / "pip-1.0-py3-none-any.whl", + { + "pip/__init__.py": b"", + "pip/_vendor/vendor.txt": b"urllib3==1.26.4\n", + "pip-1.0.dist-info/METADATA": b"Metadata-Version: 2.4\nName: pip\nVersion: 1.0\nLicense-Expression: MIT\n", + "pip-1.0.dist-info/WHEEL": b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + }, + ) + return target diff --git a/tests/hatch/test_hatch_hook.py b/tests/hatch/test_hatch_hook.py new file mode 100644 index 0000000..97ba2cf --- /dev/null +++ b/tests/hatch/test_hatch_hook.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import json +import time +import zipfile +from typing import TYPE_CHECKING, Final + +from python_sbom import validate_wheel + +if TYPE_CHECKING: + from pathlib import Path + + import pytest + from conftest import BuildWheel + + from python_sbom import Document + +_SBOM: Final[str] = "example-1.0.dist-info/sboms/example.cdx.json" +# a later hook that changes what the wheel declares and how it is tagged, after the SBOM hook's turn +_RETAG: Final[str] = """\ +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + + +class Retag(BuildHookInterface): + def initialize(self, version, build_data): + build_data["dependencies"].append("late-dependency>=1") + build_data["tag"] = "py3-none-manylinux_2_17_x86_64" +""" + + +def sbom_of(wheel: Path, name: str = _SBOM) -> Document: + return json.loads(zipfile.ZipFile(wheel).read(name)) + + +def test_wheel_carries_valid_sbom(project: Path, build_wheel: BuildWheel) -> None: + assert validate_wheel(build_wheel(project), frozenset({"MIT"})) == [] + + +def test_sbom_describes_the_built_wheel(project: Path, build_wheel: BuildWheel) -> None: + (project / "hatch_build.py").write_text(_RETAG, encoding="utf-8") + configure(project, "[tool.hatch.build.hooks.custom]\n") + document = sbom_of(wheel := build_wheel(project)) + references = { + reference["type"]: reference["url"] for reference in document["metadata"]["component"]["externalReferences"] + } + assert [component["name"] for component in document["components"]] == ["packaging", "late-dependency"] + assert references["attestation"] == f"https://pypi.org/integrity/Example/1.0/{wheel.name}/provenance" + assert wheel.name == "example-1.0-py3-none-manylinux_2_17_x86_64.whl" + + +def configure(root: Path, tables: str) -> None: + pyproject = root / "pyproject.toml" + pyproject.write_text(f"{pyproject.read_text(encoding='utf-8')}{tables}", encoding="utf-8") + + +def test_hook_table_overrides_project_table(project: Path, build_wheel: BuildWheel) -> None: + configure( + project, + "enable-by-default = true\nrequire-runtime-dependencies = false\ndefault-branch = 'trunk'\n" + "[tool.python-sbom]\ndefault-branch = 'dev'\nsupplier = { name = 'Example Org' }\n", + ) + root = sbom_of(build_wheel(project))["metadata"]["component"] + assert {"type": "license", "url": "https://github.com/example/example/blob/trunk/LICENSE"} in root[ + "externalReferences" + ] + assert root.get("supplier") == {"name": "Example Org", "url": []} + + +def test_reproducible_build_uses_hatchling_time(project: Path, build_wheel: BuildWheel) -> None: + assert sbom_of(build_wheel(project))["metadata"]["timestamp"] == "2020-02-02T00:00:00+00:00" + + +def test_non_reproducible_build_uses_current_time(project: Path, build_wheel: BuildWheel) -> None: + (project / "pyproject.toml").write_text( + (project / "pyproject.toml") + .read_text(encoding="utf-8") + .replace("[tool.hatch.build.targets.wheel]\n", "[tool.hatch.build.targets.wheel]\nreproducible = false\n"), + encoding="utf-8", + ) + before = time.time() + timestamp = sbom_of(build_wheel(project))["metadata"]["timestamp"] + assert timestamp >= time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(int(before))) + + +def test_source_date_epoch_wins(project: Path, build_wheel: BuildWheel, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + assert sbom_of(build_wheel(project))["metadata"]["timestamp"] == "2023-11-14T22:13:20+00:00" + + +def test_editable_wheel_has_no_sbom(project: Path, build_wheel: BuildWheel) -> None: + # a .pth editable, which needs no editables package + pyproject = project / "pyproject.toml" + pyproject.write_text( + pyproject.read_text(encoding="utf-8").replace( + "[tool.hatch.build.targets.wheel]\n", "[tool.hatch.build.targets.wheel]\ndev-mode-dirs = ['src']\n" + ), + encoding="utf-8", + ) + names = zipfile.ZipFile(build_wheel(project, version="editable")).namelist() + assert not [name for name in names if "/sboms/" in name] + + +def test_sdist_has_no_sbom(project: Path, build_wheel: BuildWheel) -> None: + configure(project, "[tool.hatch.build.targets.sdist.hooks.python-sbom]\n") + sdist = build_wheel(project, version="standard", target="sdist") + assert sdist.name.endswith(".tar.gz") + assert "sboms" not in sdist.read_bytes().decode("latin-1") + + +def test_content_provider_option(project: Path, build_wheel: BuildWheel, tmp_path: Path) -> None: + (project / f"{tmp_path.name}.py").write_text( + "from python_sbom import ContentProvider, Contributions\n\n\n" + "class Content(ContentProvider):\n" + " def contributions(self, request):\n" + " return Contributions(components=[], dependencies=[], tools=[]," + " root_properties=[{'name': 'example:by', 'value': 'hook'}])\n", + encoding="utf-8", + ) + configure(project, f"content-provider = '{tmp_path.name}:Content'\n") + document = sbom_of(build_wheel(project)) + assert {"name": "example:by", "value": "hook"} in document["metadata"]["component"]["properties"] + assert document["metadata"]["tools"]["components"][0]["name"] == f"{tmp_path.name}.py" + + +def test_seeder_bundles_described(seeder: Path, build_wheel: BuildWheel) -> None: + (pip,) = sbom_of(build_wheel(seeder), "seeder-1.0.dist-info/sboms/seeder.cdx.json")["components"] + assert pip.get("properties", [])[:3] == [ + {"name": "seeder:bundled-wheel", "value": "src/seeder/embed/pip-1.0-py3-none-any.whl"}, + {"name": "seeder:seeded-for-python", "value": "3.14"}, + {"name": "seeder:seeded-for-python", "value": "3.15"}, + ] + + +def test_seeder_hook_script_is_a_tool(seeder: Path, build_wheel: BuildWheel) -> None: + document = sbom_of(build_wheel(seeder), "seeder-1.0.dist-info/sboms/seeder.cdx.json") + assert document["metadata"]["tools"]["components"][0]["name"] == "hatch_build.py" + + +def test_seeder_supplier(seeder: Path, build_wheel: BuildWheel) -> None: + document = sbom_of(build_wheel(seeder), "seeder-1.0.dist-info/sboms/seeder.cdx.json") + assert document["metadata"].get("supplier") == {"name": "Example Org", "url": ["https://example.org"]} diff --git a/tests/seeder/hatch_build.py b/tests/seeder/hatch_build.py deleted file mode 100644 index 3234b2e..0000000 --- a/tests/seeder/hatch_build.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -import ast -from pathlib import Path -from typing import TYPE_CHECKING - -from python_sbom.hatch import SbomBuildHook - -if TYPE_CHECKING: - from python_sbom import Property - - -def get_build_hook() -> type[SeederBuildHook]: - # the module also sees the base class, so hatchling needs to be told which subclass to load - return SeederBuildHook - - -class SeederBuildHook(SbomBuildHook): - def bundled_wheel_properties(self, wheel: Path) -> list[Property]: - # the table that decides which Python versions receive each wheel, read without importing the package - tree = ast.parse((Path(self.root) / "src" / "seeder" / "embed" / "bundle.py").read_text(encoding="utf-8")) - support: dict[str, dict[str, str]] = ast.literal_eval( - next(node.value for node in tree.body if isinstance(node, ast.AnnAssign) and node.value is not None) - ) - return [ - {"name": "seeder:seeded-for-python", "value": python} - for python, wheels in support.items() - if wheel.name in wheels.values() - ] diff --git a/tests/seeder/src/seeder/embed/__init__.py b/tests/seeder/src/seeder/embed/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/seeder/src/seeder/embed/bundle.py b/tests/seeder/src/seeder/embed/bundle.py deleted file mode 100644 index dbd6a41..0000000 --- a/tests/seeder/src/seeder/embed/bundle.py +++ /dev/null @@ -1,9 +0,0 @@ -from __future__ import annotations - -from typing import Final - -# the Python versions that receive each bundled wheel, by distribution name; the build hook reads it without importing -_BUNDLE_SUPPORT: Final[dict[str, dict[str, str]]] = { - "3.14": {"pip": "pip-1.0-py3-none-any.whl"}, - "3.15": {"pip": "pip-1.0-py3-none-any.whl"}, -} diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index f68bfb2..0000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,83 +0,0 @@ -from __future__ import annotations - -import json -import zipfile -from typing import TYPE_CHECKING, Final - -import pytest - -from python_sbom import main, to_spdx - -if TYPE_CHECKING: - from pathlib import Path - - from conftest import BuildSbom, MakeWheel - - from python_sbom import Document - - -def test_cli_embed(make_wheel: MakeWheel, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - wheel: Final[Path] = make_wheel("") - (tmp_path / "pyproject.toml").write_text('[tool.python-sbom]\nsupplier = { name = "Org" }\n', encoding="utf-8") - main(["embed", str(wheel), "--root", str(tmp_path)]) - with zipfile.ZipFile(wheel) as archive: - document: Final[Document] = json.loads(archive.read("native-1.0.dist-info/sboms/native.cdx.json")) - assert (document["metadata"].get("supplier"), capsys.readouterr().out) == ( - {"name": "Org", "url": []}, - f"wrote native-1.0.dist-info/sboms/native.cdx.json into {wheel.name}\n", - ) - - -def test_cli_embed_without_settings(make_wheel: MakeWheel, tmp_path: Path) -> None: - wheel: Final[Path] = make_wheel("") - (tmp_path / "pyproject.toml").write_text("[project]\nname = 'native'\n", encoding="utf-8") - main(["embed", str(wheel), "--root", str(tmp_path)]) - with zipfile.ZipFile(wheel) as archive: - document: Final[Document] = json.loads(archive.read("native-1.0.dist-info/sboms/native.cdx.json")) - assert "supplier" not in document["metadata"] - - -def test_cli_spdx(build_sbom: BuildSbom, tmp_path: Path) -> None: - document: Final[Document] = build_sbom() - (source := tmp_path / "example.cdx.json").write_text(json.dumps(document), encoding="utf-8") - main(["spdx", str(source), str(target := tmp_path / "example.spdx.json")]) - assert target.read_bytes() == f"{json.dumps(to_spdx(document), indent=2)}\n".encode() - - -@pytest.mark.parametrize( - "target", [pytest.param("native-1.0-py3-none-any.whl", id="wheel"), pytest.param(".", id="dir")] -) -def test_cli_validate(make_wheel: MakeWheel, tmp_path: Path, capsys: pytest.CaptureFixture[str], target: str) -> None: - wheel: Final[Path] = make_wheel("License-Expression: MIT\n") - (tmp_path / "pyproject.toml").write_text("[project]\nname = 'native'\n", encoding="utf-8") - main(["embed", str(wheel), "--root", str(tmp_path)]) - main(["validate", str(wheel.parent / target), "--allow-license", "MIT"]) - assert capsys.readouterr().out.splitlines()[-1] == "SBOM in native-1.0-py3-none-any.whl is valid" - - -def test_cli_validate_problems(make_wheel: MakeWheel, capsys: pytest.CaptureFixture[str]) -> None: - wheel: Final[Path] = make_wheel("") - with pytest.raises(SystemExit) as error: - main(["validate", str(wheel)]) - assert (error.value.code, capsys.readouterr().err) == ( - 1, - "SBOM invalid: native-1.0-py3-none-any.whl: no .dist-info/sboms/*.cdx.json found in the wheel\n", - ) - - -def test_cli_validate_no_wheel(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - with pytest.raises(SystemExit) as error: - main(["validate", str(tmp_path)]) - assert (error.value.code, capsys.readouterr().err) == (1, f"no wheel found in {tmp_path}\n") - - -def test_cli_embed_not_a_wheel(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - with zipfile.ZipFile(archive := tmp_path / "broken-1.0-py3-none-any.whl", "w") as zipped: - zipped.writestr("broken/__init__.py", "") - (tmp_path / "pyproject.toml").write_text("[project]\nname = 'broken'\n", encoding="utf-8") - with pytest.raises(SystemExit) as error: - main(["embed", str(archive), "--root", str(tmp_path)]) - assert (error.value.code, capsys.readouterr().err) == ( - 1, - f"python-sbom embed: {archive} has no .dist-info/METADATA, so it is not a wheel\n", - ) diff --git a/tests/test_embed.py b/tests/test_embed.py deleted file mode 100644 index 4cbb47b..0000000 --- a/tests/test_embed.py +++ /dev/null @@ -1,142 +0,0 @@ -from __future__ import annotations - -import importlib -import json -import sys -import zipfile -from pathlib import Path -from typing import TYPE_CHECKING, Final - -import pytest -from conftest import digest, write_seed_wheel -from installer import install -from installer.destinations import SchemeDictionaryDestination -from installer.sources import WheelFile - -from python_sbom import Settings, validate_wheel - -if TYPE_CHECKING: - from conftest import EmbedInto, MakeWheel - - from python_sbom import Document - -_METADATA: Final[str] = "License-Expression: MIT\nProject-URL: Source, https://github.com/example/native\n" - - -def test_embed_writes_sbom(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: - wheel: Final[Path] = make_wheel(_METADATA) - name: Final[str] = embed_into(wheel) - with zipfile.ZipFile(wheel) as archive: - document: Final[Document] = json.loads(archive.read(name)) - assert (name, document["metadata"]["component"]["purl"]) == ( - "native-1.0.dist-info/sboms/native.cdx.json", - "pkg:pypi/native@1.0", - ) - - -def test_embed_record(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: - wheel: Final[Path] = make_wheel(_METADATA) - name: Final[str] = embed_into(wheel) - with zipfile.ZipFile(wheel) as archive: - sbom: Final[bytes] = archive.read(name) - rows: Final[list[str]] = archive.read("native-1.0.dist-info/RECORD").decode("utf-8").splitlines() - last: Final[str] = archive.namelist()[-1] - assert (rows[-2:], last) == ( - [f"{name},{digest(sbom)},{len(sbom)}", "native-1.0.dist-info/RECORD,,"], - "native-1.0.dist-info/RECORD", - ) - - -def test_embed_twice_same_bytes(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: - wheel: Final[Path] = make_wheel(_METADATA) - embed_into(wheel) - first: Final[bytes] = wheel.read_bytes() - embed_into(wheel) - assert wheel.read_bytes() == first - - -def test_embed_keeps_entries(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: - wheel: Final[Path] = make_wheel(_METADATA) - with zipfile.ZipFile(wheel) as archive: - before: Final[list[tuple[str, tuple[int, ...]]]] = [ - (info.filename, info.date_time) for info in archive.infolist() - ] - embed_into(wheel) - with zipfile.ZipFile(wheel) as archive: - after: Final[list[tuple[str, tuple[int, ...]]]] = [ - (info.filename, info.date_time) for info in archive.infolist() - ] - assert after == [*before[:-1], ("native-1.0.dist-info/sboms/native.cdx.json", (2020, 2, 2, 0, 0, 0)), before[-1]] - - -def test_embed_crlf_record(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: - wheel: Final[Path] = make_wheel(_METADATA) - with zipfile.ZipFile(wheel) as archive: - entries: Final[dict[str, bytes]] = {name: archive.read(name) for name in archive.namelist()} - entries["native-1.0.dist-info/RECORD"] = entries["native-1.0.dist-info/RECORD"].replace(b"\n", b"\r\n") - with zipfile.ZipFile(wheel, "w") as archive: - for name, content in entries.items(): - archive.writestr(name, content) - name: Final[str] = embed_into(wheel) - with zipfile.ZipFile(wheel) as archive: - record: Final[bytes] = archive.read("native-1.0.dist-info/RECORD") - assert (record.count(b"\r\n"), record.count(b"\n"), f"{name},".encode() in record) == (5, 5, True) - - -def test_embed_valid(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: - wheel: Final[Path] = make_wheel(_METADATA) - embed_into(wheel) - assert validate_wheel(wheel, frozenset({"MIT"})) == [] - - -def test_embed_survives_install(make_wheel: MakeWheel, embed_into: EmbedInto, tmp_path: Path) -> None: - wheel: Final[Path] = make_wheel(_METADATA) - name: Final[str] = embed_into(wheel) - scheme: Final[dict[str, str]] = { - key: str(tmp_path / "site" / key) for key in ("purelib", "platlib", "headers", "scripts", "data") - } - with WheelFile.open(wheel) as source: - source.validate_record() - install(source, SchemeDictionaryDestination(scheme, sys.executable, "posix"), {}) - assert (Path(scheme["platlib"]) / name).is_file() - - -def test_embed_uses_project_settings(make_wheel: MakeWheel, embed_into: EmbedInto) -> None: - wheel: Final[Path] = make_wheel(_METADATA) - settings: Final[Settings] = Settings.from_table({"supplier": {"name": "Org"}}, "[tool.python-sbom]") - name: Final[str] = embed_into(wheel, settings) - with zipfile.ZipFile(wheel) as archive: - document: Final[Document] = json.loads(archive.read(name)) - assert document["metadata"].get("supplier") == {"name": "Org", "url": []} - - -def test_embed_bundled_wheel(make_wheel: MakeWheel, embed_into: EmbedInto, tmp_path: Path) -> None: - wheel: Final[Path] = make_wheel(_METADATA) - write_seed_wheel(tmp_path) - settings: Final[Settings] = Settings.from_table({"bundled-wheels": ["*.whl"]}, "[tool.python-sbom]") - name: Final[str] = embed_into(wheel, settings) - with zipfile.ZipFile(wheel) as archive: - document: Final[Document] = json.loads(archive.read(name)) - assert document["components"][0].get("properties") == [ - {"name": "native:bundled-wheel", "value": "pip-1.0-py3-none-any.whl"} - ] - - -@pytest.fixture -def without_hatchling(monkeypatch: pytest.MonkeyPatch) -> None: - # a fresh import of the package with hatchling unimportable, as for a project built with another backend - for name in [name for name in sys.modules if name.split(".")[0] in {"python_sbom", "hatchling"}]: - monkeypatch.delitem(sys.modules, name) - monkeypatch.setitem(sys.modules, "hatchling", None) - - -@pytest.mark.usefixtures("without_hatchling") -def test_embed_without_hatchling(make_wheel: MakeWheel, tmp_path: Path) -> None: - core = importlib.import_module("python_sbom") - wheel: Final[Path] = make_wheel(_METADATA) - facts = core.BuildFacts.collect(tmp_path, core.git_revision) - name: Final[str] = core.BuiltWheel(wheel, tmp_path, core.Settings.from_table({}, "[tool.python-sbom]")).embed(facts) - assert (name, "hatchling" in {module.split(".")[0] for module in sys.modules if sys.modules[module]}) == ( - "native-1.0.dist-info/sboms/native.cdx.json", - False, - ) diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..85b74ff --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import zipfile +from typing import TYPE_CHECKING + +from docs.examples.zipapp import describe_zipapp +from python_sbom import dumps, validate + +if TYPE_CHECKING: + from pathlib import Path + + +def test_zipapp_example(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'app'\n", encoding="utf-8") + with zipfile.ZipFile(zipapp := tmp_path / "app.pyz", "w") as archive: + archive.writestr("__main__.py", "print('hi')\n") + archive.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") + archive.writestr("app/", "") + document = describe_zipapp(tmp_path, zipapp) + assert validate(dumps(document).decode(), None) == [] + assert [file["name"] for file in document["metadata"]["component"].get("components", [])] == [ + "__main__.py", + "app-1.0.dist-info/METADATA", + ] diff --git a/tests/test_hook.py b/tests/test_hook.py deleted file mode 100644 index eba3342..0000000 --- a/tests/test_hook.py +++ /dev/null @@ -1,144 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import zipfile -from pathlib import Path -from typing import TYPE_CHECKING, Final - -import pytest -from conftest import BUNDLED, HOOK, build_hook_sbom -from hatchling.builders.sdist import SdistBuilder -from hatchling.builders.wheel import WheelBuilder - -from python_sbom import validate_wheel - -if TYPE_CHECKING: - from conftest import BundleWheel, WriteProject - - from python_sbom import Document - - -def test_wheel_embeds_sbom(write_project: WriteProject, tmp_path: Path) -> None: - with zipfile.ZipFile(_build_wheel(write_project(), tmp_path / "dist")) as archive: - sbom: Final[bytes] = archive.read("example-1.0.dist-info/sboms/example.cdx.json") - record: Final[str] = archive.read("example-1.0.dist-info/RECORD").decode("utf-8") - assert ("example-1.0.dist-info/sboms/example.cdx.json,sha256=" in record, json.loads(sbom)["bomFormat"]) == ( - True, - "CycloneDX", - ) - - -def _build_wheel(root: Path, out: Path) -> Path: - (artifact,) = WheelBuilder(str(root)).build(directory=str(out), versions=["standard"]) - return Path(artifact) - - -def test_wheel_sbom_valid(write_project: WriteProject, bundle_wheel: BundleWheel, tmp_path: Path) -> None: - bundle_wheel({"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}, "License-Expression: MIT\n") - wheel: Final[Path] = _build_wheel(write_project(hook=f"{HOOK}{BUNDLED}"), tmp_path / "dist") - assert validate_wheel(wheel, frozenset({"MIT"})) == [] - - -def test_wheel_reproducible(write_project: WriteProject, tmp_path: Path) -> None: - root: Final[Path] = write_project() - assert _build_wheel(root, tmp_path / "first").read_bytes() == _build_wheel(root, tmp_path / "second").read_bytes() - - -def test_wheel_sbom_directory_removed(write_project: WriteProject) -> None: - builder: Final[WheelBuilder] = WheelBuilder(str(write_project())) - build_data = builder.get_default_build_data() - (hook,) = builder.get_build_hooks(str(builder.root)).values() - hook.initialize("standard", build_data) - hook.finalize("standard", build_data, "") - assert Path(build_data["sbom_files"][0]).parent.exists() is False - - -def test_sdist_skipped(write_project: WriteProject, tmp_path: Path) -> None: - # the hook runs for every target, and only a wheel build has somewhere to put an SBOM - (artifact,) = SdistBuilder(str(write_project())).build(directory=str(tmp_path / "dist")) - assert Path(artifact).name == "example-1.0.tar.gz" - - -def test_attestation_tag_from_another_hook(write_project: WriteProject) -> None: - builder: Final[WheelBuilder] = WheelBuilder(str(write_project())) - assert _attestations(builder, "tag", "py3-none-linux_x86_64") == [ - "https://pypi.org/integrity/Example/1.0/example-1.0-py3-none-linux_x86_64.whl/provenance" - ] - - -def _attestations(builder: WheelBuilder, key: str, value: str | bool) -> list[str]: - build_data = builder.get_default_build_data() - build_data[key] = value - (hook,) = builder.get_build_hooks(str(builder.root)).values() - hook.initialize("standard", build_data) - document: Final[Document] = json.loads(Path(build_data["sbom_files"][0]).read_text(encoding="utf-8")) - references = document["metadata"]["component"].get("externalReferences", []) - return [reference["url"] for reference in references if reference["type"] == "attestation"] - - -def test_attestation_tag_inferred(write_project: WriteProject) -> None: - builder: Final[WheelBuilder] = WheelBuilder(str(write_project())) - assert _attestations(builder, "infer_tag", True) == [ - f"https://pypi.org/integrity/Example/1.0/example-1.0-{builder.get_best_matching_tag()}.whl/provenance" - ] - - -@pytest.mark.parametrize( - ("urls", "references"), - [ - pytest.param( - 'urls.Source = "https://github.com/example/example"', - {"externalReferences": [{"type": "vcs", "url": "https://github.com/example/example/blob/main/build.py"}]}, - id="repository", - ), - pytest.param("", {}, id="no-repository"), - ], -) -def test_custom_build_script_recorded( - root: Path, write_project: WriteProject, urls: str, references: dict[str, list[dict[str, str]]] -) -> None: - (script := root / "build.py").write_text( - "from __future__ import annotations\n\nfrom python_sbom.hatch import SbomBuildHook\n\n\n" - "def get_build_hook() -> type[SbomBuildHook]:\n return SbomBuildHook\n", - encoding="utf-8", - ) - project = f'[project]\nname = "example"\nversion = "1.0"\n{urls}\n' - document = build_hook_sbom( - write_project(hook='[tool.hatch.build.hooks.custom]\npath = "build.py"\n', project=project) - ) - assert document["metadata"]["tools"]["components"][0] == { - "type": "application", - "bom-ref": "tool:build.py", - "name": "build.py", - "version": "1.0", - "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(script.read_bytes()).hexdigest()}], - **references, - } - - -def test_seeder_subclass_adds_properties(seeder: Path, tmp_path: Path) -> None: - with zipfile.ZipFile(_build_wheel(seeder, tmp_path / "dist")) as archive: - document: Final[Document] = json.loads(archive.read("seeder-1.0.dist-info/sboms/seeder.cdx.json")) - assert document["components"][0].get("properties", [])[:3] == [ - {"name": "seeder:bundled-wheel", "value": "src/seeder/embed/pip-1.0-py3-none-any.whl"}, - {"name": "seeder:seeded-for-python", "value": "3.14"}, - {"name": "seeder:seeded-for-python", "value": "3.15"}, - ] - - -def test_seeder_supplier_and_vendored(seeder: Path, tmp_path: Path) -> None: - with zipfile.ZipFile(_build_wheel(seeder, tmp_path / "dist")) as archive: - document: Final[Document] = json.loads(archive.read("seeder-1.0.dist-info/sboms/seeder.cdx.json")) - pip = document["components"][0] - assert ( - pip.get("supplier"), - [child["name"] for child in pip.get("components", []) if child["type"] == "library"], - ) == ( - {"name": "Example Org", "url": ["https://example.org"]}, - ["urllib3"], - ) - - -def test_seeder_wheel_valid(seeder: Path, tmp_path: Path) -> None: - assert validate_wheel(_build_wheel(seeder, tmp_path / "dist"), frozenset({"MIT"})) == [] diff --git a/tests/test_main.py b/tests/test_main.py deleted file mode 100644 index ec363b1..0000000 --- a/tests/test_main.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -import subprocess -import sys -from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - import pytest - - -def test_main(capfd: pytest.CaptureFixture[str]) -> None: - subprocess.check_call([sys.executable, "-m", "python_sbom", "-h"]) - out, _err = capfd.readouterr() - assert out.startswith("usage: python-sbom ") - - -def test_script(capfd: pytest.CaptureFixture[str]) -> None: - subprocess.check_call([Path(sys.executable).parent / "python-sbom", "-h"]) - out, _err = capfd.readouterr() - assert out.startswith("usage: python-sbom ") diff --git a/tests/test_no_hatchling.py b/tests/test_no_hatchling.py new file mode 100644 index 0000000..9a84f05 --- /dev/null +++ b/tests/test_no_hatchling.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +import subprocess +import sys + + +def test_core_imports_without_hatchling() -> None: + # None in sys.modules makes the import fail, as when hatchling is not installed + code = ( + "import sys; sys.modules['hatchling'] = None\n" + "import python_sbom\n" + "try:\n import python_sbom.hatch\nexcept ImportError:\n print('core only')\n" + ) + assert subprocess.check_output([sys.executable, "-c", code], text=True) == "core only\n" diff --git a/tests/test_project.py b/tests/test_project.py deleted file mode 100644 index ed2416e..0000000 --- a/tests/test_project.py +++ /dev/null @@ -1,593 +0,0 @@ -from __future__ import annotations - -import csv -import hashlib -import shutil -import subprocess -import sys -import zipfile -from io import StringIO -from typing import TYPE_CHECKING, Final - -import pytest - -if TYPE_CHECKING: - from pathlib import Path - - from conftest import BuildSbom, BundleWheel, InstallTool, WriteProject - from pytest_mock import MockerFixture - - from python_sbom import Component, Document - -from conftest import BUNDLED, HOOK, build_hook_sbom - -from python_sbom import SourceRevisionError, WheelError - -_SHA256_ABC: Final[str] = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" - - -def test_root_component(build_sbom: BuildSbom) -> None: - assert build_sbom()["metadata"]["component"] == { - "type": "application", - "bom-ref": "pkg:pypi/example@1.0", - "authors": [{"name": "Maintainer", "email": "maintainer@example.com"}], - "name": "Example", - "version": "1.0", - "description": "an example", - "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], - "copyright": "Copyright (c) 2026 Example", - "purl": "pkg:pypi/example@1.0", - "externalReferences": [ - {"type": "release-notes", "url": "https://example.com/changelog", "comment": "Project-URL: Changelog"}, - {"type": "vcs", "url": "https://github.com/example/example", "comment": "Project-URL: Source"}, - {"type": "distribution", "url": "https://pypi.org/project/Example/1.0/"}, - { - "type": "attestation", - "url": "https://pypi.org/integrity/Example/1.0/example-1.0-py3-none-any.whl/provenance", - }, - {"type": "security-contact", "url": "https://github.com/example/example/security/policy"}, - {"type": "advisories", "url": "https://github.com/example/example/security/advisories"}, - {"type": "license", "url": "https://github.com/example/example/blob/main/LICENSE"}, - ], - "properties": [ - {"name": "example:requires-python", "value": ">=3.11"}, - {"name": "python:classifier", "value": "Programming Language :: Python :: 3"}, - {"name": "python:keyword", "value": "demo"}, - {"name": "python:keyword", "value": "sample"}, - ], - } - - -def test_root_component_minimal_metadata(root: Path, build_sbom: BuildSbom) -> None: - (root / "LICENSE").write_text("MIT terms\n", encoding="utf-8") - project = '[project]\nname = "example"\nversion = "1.0"\nlicense-files = ["LICENSE"]\nauthors = [{ name = "A" }]\n' - assert build_sbom(project=project)["metadata"]["component"] == { - "type": "application", - "bom-ref": "pkg:pypi/example@1.0", - "authors": [{"name": "A"}], - "name": "example", - "version": "1.0", - "purl": "pkg:pypi/example@1.0", - "externalReferences": [ - {"type": "distribution", "url": "https://pypi.org/project/example/1.0/"}, - { - "type": "attestation", - "url": "https://pypi.org/integrity/example/1.0/example-1.0-py2.py3-none-any.whl/provenance", - }, - ], - "properties": [], - } - - -def test_document_metadata(build_sbom: BuildSbom) -> None: - metadata = build_sbom()["metadata"] - assert (metadata["timestamp"], metadata["lifecycles"], metadata.get("authors"), metadata.get("properties")) == ( - "2020-02-02T00:00:00+00:00", - [{"phase": "build"}], - [{"name": "Maintainer", "email": "maintainer@example.com"}], - [ - { - "name": "example:sbom:scope", - "value": "Components are the direct runtime dependencies declared in the wheel's core metadata. They" - " carry no version because they are resolved at install time, not at build time.", - } - ], - ) - - -def test_supplier(build_sbom: BuildSbom) -> None: - metadata = build_sbom(hook=f'{HOOK}supplier = {{ name = "Org", url = ["https://org.example"] }}\n')["metadata"] - assert [metadata.get("manufacturer"), metadata.get("supplier"), metadata["component"].get("supplier")] == [ - {"name": "Org", "url": ["https://org.example"]} - ] * 3 - - -def test_declared_dependencies(build_sbom: BuildSbom) -> None: - document = build_sbom() - assert (document["components"], document["dependencies"][0]) == ( - [ - { - "type": "library", - "bom-ref": "requires-dist:packaging>=24", - "name": "packaging", - "purl": "pkg:pypi/packaging", - "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/packaging/"}], - "properties": [{"name": "example:requires-dist", "value": "packaging>=24"}], - }, - { - "type": "library", - "bom-ref": "requires-dist:tomli>=2; python_version < '3.11'", - "name": "tomli", - "purl": "pkg:pypi/tomli", - "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/tomli/"}], - "properties": [ - {"name": "example:requires-dist", "value": "tomli>=2; python_version < '3.11'"}, - {"name": "example:environment-marker", "value": 'python_version < "3.11"'}, - ], - }, - ], - { - "ref": "pkg:pypi/example@1.0", - "dependsOn": ["requires-dist:packaging>=24", "requires-dist:tomli>=2; python_version < '3.11'"], - }, - ) - - -def test_composition_incomplete(build_sbom: BuildSbom) -> None: - assert build_sbom()["compositions"] == [{"aggregate": "incomplete", "dependencies": ["pkg:pypi/example@1.0"]}] - - -def test_build_workflow(build_sbom: BuildSbom) -> None: - document = build_sbom() - workflow = document["formulation"][0]["workflows"][0] - assert {**workflow, "resourceReferences": len(workflow["resourceReferences"])} == { - "bom-ref": "workflow:wheel-build", - "uid": "wheel-build", - "name": "build the wheel and this SBOM", - "taskTypes": ["build"], - "resourceReferences": len(document["metadata"]["tools"]["components"]), - "outputs": [{"type": "artifact", "resource": {"ref": "pkg:pypi/example@1.0"}}], - "inputs": [{"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": "1580601600"}]}], - } - - -def test_bundled_wheel(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: - sha256: Final[str] = hashlib.sha256(bundle_wheel({}, "License-Expression: MIT\n").read_bytes()).hexdigest() - assert build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0] == { - "type": "library", - "bom-ref": "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl", - "name": "pip", - "version": "1.0", - "purl": "pkg:pypi/pip@1.0", - "licenses": [{"expression": "MIT", "acknowledgement": "declared"}], - "externalReferences": [{"type": "distribution", "url": "https://pypi.org/project/pip/1.0/"}], - "properties": [{"name": "example:bundled-wheel", "value": "src/example/wheels/pip-1.0-py3-none-any.whl"}], - "hashes": [{"alg": "SHA-256", "content": sha256}], - "evidence": { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [ - {"technique": "manifest-analysis", "confidence": 1, "value": "pip-1.0.dist-info/METADATA"} - ], - }, - { - "field": "hash", - "confidence": 1, - "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], - }, - ] - }, - "components": [ - { - "type": "file", - "bom-ref": "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl#pip/example.py", - "name": "pip/example.py", - "hashes": [{"alg": "SHA-256", "content": _SHA256_ABC}], - "properties": [{"name": "size", "value": "3"}], - } - ], - } - - -def test_bundled_wheel_scope(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: - bundle_wheel({}) - assert build_sbom(hook=f"{HOOK}{BUNDLED}")["metadata"].get("properties") == [ - { - "name": "example:sbom:scope", - "value": "Components are the wheels bundled as data files under src/example/wheels and the direct runtime" - " dependencies declared in the wheel's core metadata. The latter carry no version because they are" - " resolved at install time, not at build time.", - } - ] - - -def test_bundled_wheels_sorted_once(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: - bundle_wheel({}, name="setuptools") - bundle_wheel({}, name="pip") - hook = f'{HOOK}bundled-wheels = ["src/example/wheels/*.whl", "src/example/wheels/pip-*"]\n' - assert build_sbom(hook=hook)["dependencies"][0]["dependsOn"] == [ - "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl", - "pkg:pypi/setuptools@1.0?file_name=setuptools-1.0-py3-none-any.whl", - "requires-dist:packaging>=24", - "requires-dist:tomli>=2; python_version < '3.11'", - ] - - -@pytest.mark.parametrize( - ("source", "urls", "expected"), - [ - pytest.param("https://github.com/example/pip", "urls.Source = 'https://github.com/example/x'", True, id="same"), - pytest.param("https://github.com/other/pip", "urls.Source = 'https://github.com/example/x'", False, id="other"), - pytest.param("https://github.com/example/pip", "", False, id="no-repository"), - ], -) -def test_bundled_wheel_supplier( - build_sbom: BuildSbom, bundle_wheel: BundleWheel, source: str, urls: str, expected: bool -) -> None: - bundle_wheel({}, f"Project-URL: Source, {source}\n") - project = f'[project]\nname = "example"\nversion = "1.0"\n{urls}\n' - hook = f'{HOOK}{BUNDLED}supplier = {{ name = "Org" }}\n' - assert ("supplier" in build_sbom(hook=hook, project=project)["components"][0]) is expected - - -@pytest.mark.parametrize( - "filename", - [ - pytest.param("pip/example.py", id="plain"), - pytest.param("pip/a,b.py", id="comma"), - pytest.param('pip/a"b.py', id="quote"), - pytest.param("pip/a\nb.py", id="newline"), - ], -) -def test_bundled_wheel_record_paths(root: Path, build_sbom: BuildSbom, filename: str) -> None: - record: Final[StringIO] = StringIO(newline="") - csv.writer(record).writerows([(filename, "sha256=ungWv48Bz-pBQUDeXa4iI7ADYaOWF3qctBD_YfIAFa0", "3")]) - (wheels := root / "src" / "example" / "wheels").mkdir(parents=True) - with zipfile.ZipFile(wheels / "pip-1.0-py3-none-any.whl", "w") as archive: - archive.writestr("pip-1.0.dist-info/METADATA", "Metadata-Version: 2.4\nName: pip\nVersion: 1.0\n") - archive.writestr("pip-1.0.dist-info/RECORD", f"{record.getvalue()}pip-1.0.dist-info/RECORD,,\n") - assert build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0].get("components") == [ - { - "type": "file", - "bom-ref": f"pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl#{filename}", - "name": filename, - "hashes": [{"alg": "SHA-256", "content": _SHA256_ABC}], - "properties": [{"name": "size", "value": "3"}], - }, - ] - - -@pytest.mark.parametrize( - ("source", "content"), - [ - pytest.param("pip/_vendor/vendor.txt", "# vendored\n\n urllib3==1.26.4 # patched\n", id="pip-manifest"), - pytest.param( - "pip/_vendor/urllib3-1.26.4.dist-info/METADATA", "Name: urllib3\nVersion: 1.26.4\n", id="nested-metadata" - ), - ], -) -def test_vendored_identity(build_sbom: BuildSbom, bundle_wheel: BundleWheel, source: str, content: str) -> None: - bundle_wheel({source: content}) - children: Final[list[Component]] = build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0].get("components", []) - assert [child for child in children if child["type"] == "library"] == [ - { - "type": "library", - "bom-ref": "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl#vendored/pkg:pypi/urllib3@1.26.4", - "name": "urllib3", - "version": "1.26.4", - "purl": "pkg:pypi/urllib3@1.26.4", - "externalReferences": [], - "properties": [{"name": "example:vendored-manifest", "value": source}], - "evidence": { - "identity": [ - { - "field": "purl", - "confidence": 1, - "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": source}], - } - ] - }, - } - ] - - -def test_vendored_relationship(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: - bundle_wheel({"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}) - assert build_sbom(hook=f"{HOOK}{BUNDLED}")["dependencies"][1] == { - "ref": "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl", - "dependsOn": ["pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl#vendored/pkg:pypi/urllib3@1.26.4"], - } - - -def test_vendored_metadata_precedence(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: - bundle_wheel({ - "pip/_vendor/vendor.txt": "urllib3==1.26.4\n", - "pip/_vendor/urllib3-1.26.4.dist-info/METADATA": "Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n", - }) - children: Final[list[Component]] = build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0].get("components", []) - assert [child.get("licenses") for child in children if child["type"] == "library"] == [ - [{"expression": "MIT", "acknowledgement": "declared"}], - ] - - -@pytest.mark.parametrize( - "requirement", - [ - pytest.param("urllib3", id="missing"), - pytest.param("urllib3>=1", id="range"), - pytest.param("urllib3==1.*", id="wildcard"), - pytest.param("urllib3 @ https://example.com/package.whl", id="url"), - ], -) -def test_vendored_unpinned_rejected(build_sbom: BuildSbom, bundle_wheel: BundleWheel, requirement: str) -> None: - bundle_wheel({"pip/_vendor/vendor.txt": requirement}) - with pytest.raises(ValueError, match="Unpinned vendored dependency"): - build_sbom(hook=f"{HOOK}{BUNDLED}") - - -@pytest.mark.parametrize( - ("metadata", "expected"), - [ - pytest.param( - "Classifier: License :: OSI Approved :: MIT License\n", - [ - {"license": {"name": "MIT License", "acknowledgement": "declared"}}, - {"license": {"id": "MIT", "acknowledgement": "concluded"}}, - ], - id="classifier", - ), - pytest.param( - "Classifier: License :: OSI Approved :: Python Software Foundation License\nLicense: PSF-2.0\n", - [ - {"license": {"name": "Python Software Foundation License", "acknowledgement": "declared"}}, - {"license": {"name": "PSF-2.0", "acknowledgement": "declared"}}, - {"license": {"id": "PSF-2.0", "acknowledgement": "concluded"}}, - ], - id="classifier-and-id-agree", - ), - pytest.param( - "License: mit\n", - [ - {"license": {"name": "mit", "acknowledgement": "declared"}}, - {"license": {"id": "MIT", "acknowledgement": "concluded"}}, - ], - id="id-canonical-case", - ), - pytest.param( - "Classifier: License :: OSI Approved :: MIT License\n" - "Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)\n", - [ - {"license": {"name": "MIT License", "acknowledgement": "declared"}}, - {"license": {"name": "GNU Lesser General Public License v3 (LGPLv3)", "acknowledgement": "declared"}}, - ], - id="one-unresolved", - ), - pytest.param( - "License: MIT OR Apache-2.0\n", - [{"license": {"name": "MIT OR Apache-2.0", "acknowledgement": "declared"}}], - id="expression-in-license-field", - ), - pytest.param( - "License: LicenseRef-Proprietary\n", - [{"license": {"name": "LicenseRef-Proprietary", "acknowledgement": "declared"}}], - id="license-ref", - ), - ], -) -def test_license_resolution( - build_sbom: BuildSbom, bundle_wheel: BundleWheel, metadata: str, expected: list[dict[str, dict[str, str]]] -) -> None: - bundle_wheel({}, metadata) - assert build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0].get("licenses") == expected - - -def test_bundled_wheel_metadata_fields(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: - bundle_wheel( - {}, - "Summary: installer\nHome-page: https://pip.pypa.io\nAuthor: Someone\nAuthor-email: dev@example.com\n" - "Maintainer-email: Team <>\nProject-URL: Bug Tracker, https://example.com/issues\n", - ) - component = build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][0] - assert (component.get("description"), component.get("authors"), component.get("externalReferences")) == ( - "installer", - [{"name": "Someone"}, {"email": "dev@example.com"}, {"name": "Team"}], - [ - {"type": "website", "url": "https://pip.pypa.io"}, - {"type": "issue-tracker", "url": "https://example.com/issues", "comment": "Project-URL: Bug Tracker"}, - {"type": "distribution", "url": "https://pypi.org/project/pip/1.0/"}, - ], - ) - - -@pytest.mark.parametrize( - ("epoch", "expected"), - [ - pytest.param("1580601600", ("2020-02-02T00:00:00+00:00", "1580601600"), id="hatchling-default"), - pytest.param("1700000000", ("2023-11-14T22:13:20+00:00", "1700000000"), id="source-date-epoch"), - ], -) -def test_timestamp( - build_sbom: BuildSbom, monkeypatch: pytest.MonkeyPatch, epoch: str, expected: tuple[str, str] -) -> None: - monkeypatch.setenv("SOURCE_DATE_EPOCH", epoch) - document = build_sbom() - variables = document["formulation"][0]["workflows"][0]["inputs"][0]["environmentVars"] - assert (document["metadata"]["timestamp"], variables[0]["value"]) == expected - - -def test_build_host_independent(write_project: WriteProject, mocker: MockerFixture, install_tool: InstallTool) -> None: - root = write_project() - assert _build_on_host(root, mocker, install_tool, ("x86_64", "GCC 13.3.0", b"uv\n")) == _build_on_host( - root, mocker, install_tool, ("aarch64", "Clang 20.1.4", b"pip\n") - ) - - -def _build_on_host( - root: Path, mocker: MockerFixture, install_tool: InstallTool, host: tuple[str, str, bytes] -) -> Document: - machine, compiler, installer = host - mocker.patch("platform.machine", autospec=True, return_value=machine) - mocker.patch("platform.platform", autospec=True, return_value=f"Linux-{machine}") - mocker.patch("platform.python_compiler", autospec=True, return_value=compiler) - mocker.patch.object(sys, "version", f"3.14.7 (main) [{compiler}]") - install_tool(site := root.parent / machine, "native", False, {f"native/_speedups.{machine}.so": b"\0"}) - install_tool(site, "pure", True, {"pure-1.0.dist-info/INSTALLER": installer, "pure/__init__.py": b""}) - mocker.patch.object(sys, "path", [str(site), *sys.path]) - document = build_hook_sbom(root) - mocker.stopall() - return document - - -def test_build_tools_pure_files( - write_project: WriteProject, mocker: MockerFixture, install_tool: InstallTool, tmp_path: Path -) -> None: - install_tool( - site := tmp_path / "site", - "pure", - True, - {"pure-1.0.dist-info/INSTALLER": b"uv\n", "pure/__init__.py": b"", "../../bin/pure": b"#!python\n"}, - ) - install_tool(site, "native", False, {"native/_speedups.so": b"\0"}) - mocker.patch.object(sys, "path", [str(site), *sys.path]) - tools = {tool["name"]: tool for tool in build_hook_sbom(write_project())["metadata"]["tools"]["components"]} - assert ([file["name"] for file in tools["pure"].get("components", [])], "components" in tools["native"]) == ( - ["pure/__init__.py"], - False, - ) - - -def test_ci_environment_ignored(write_project: WriteProject, monkeypatch: pytest.MonkeyPatch) -> None: - root = write_project() - monkeypatch.setenv("GITHUB_SHA", "a" * 40) - first: Final[Document] = build_hook_sbom(root) - monkeypatch.setenv("GITHUB_SHA", "b" * 40) - assert build_hook_sbom(root) == first - - -def test_source_revision(write_project: WriteProject, monkeypatch: pytest.MonkeyPatch) -> None: - root = write_project() - monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(root / "gitconfig")) - (root / "gitconfig").write_text("[user]\n\tname = Example\n\temail = example@example.com\n", encoding="utf-8") - git = shutil.which("git") or "git" - subprocess.run([git, "init", "-q"], cwd=root, check=True) - subprocess.run([git, "commit", "-q", "--allow-empty", "--no-gpg-sign", "-m", "init"], cwd=root, check=True) - commit = subprocess.run([git, "rev-parse", "HEAD"], cwd=root, check=True, capture_output=True, text=True) - component = build_hook_sbom(root)["metadata"]["component"] - assert (component.get("externalReferences", [])[-1], component.get("properties", [])[-1]) == ( - { - "type": "vcs", - "url": f"https://github.com/example/example/tree/{commit.stdout.strip()}", - "comment": "exact source revision", - }, - {"name": "example:vcs-commit", "value": commit.stdout.strip()}, - ) - - -def test_source_revision_without_repository(root: Path, build_sbom: BuildSbom, mocker: MockerFixture) -> None: - (root / ".git").mkdir() - mocker.patch("subprocess.run", autospec=True, return_value=subprocess.CompletedProcess(["git"], 0, "c0ffee\n", "")) - component = build_sbom(project='[project]\nname = "example"\nversion = "1.0"\n')["metadata"]["component"] - assert component.get("properties") == [{"name": "example:vcs-commit", "value": "c0ffee"}] - - -def test_source_revision_unreadable(root: Path, build_sbom: BuildSbom, mocker: MockerFixture) -> None: - (root / ".git").mkdir() - error = subprocess.CalledProcessError(128, ["git"], stderr="fatal: detected dubious ownership") - mocker.patch("subprocess.run", autospec=True, side_effect=error) - with pytest.raises( - SourceRevisionError, match=r"git rev-parse HEAD failed in .*: fatal: detected dubious ownership" - ): - build_sbom() - - -@pytest.mark.parametrize( - ("hook", "expected"), - [ - pytest.param( - f'{HOOK}repository = "https://gitlab.com/example/example/"\n', - ( - ["release-notes", "vcs", "distribution", "attestation", "license"], - "https://gitlab.com/example/example/blob/main/LICENSE", - ), - id="off-github", - ), - pytest.param( - f'{HOOK}default-branch = "trunk"\n', - ( - ["release-notes", "vcs", "distribution", "attestation", "security-contact", "advisories", "license"], - "https://github.com/example/example/blob/trunk/LICENSE", - ), - id="default-branch", - ), - ], -) -def test_repository_links(build_sbom: BuildSbom, hook: str, expected: tuple[list[str], str]) -> None: - references = build_sbom(hook=hook)["metadata"]["component"].get("externalReferences", []) - assert ([reference["type"] for reference in references], references[-1]["url"]) == expected - - -def test_serial_number_per_project(build_sbom: BuildSbom) -> None: - assert ( - build_sbom()["serialNumber"] - != build_sbom(project='[project]\nname = "Example"\nversion = "1.0"\n')["serialNumber"] - ) - - -@pytest.mark.parametrize( - ("hook", "message"), - [ - pytest.param( - f'{HOOK}supplier = "Org"\n', "`supplier` in build hook `python-sbom` must be a table", id="supplier" - ), - pytest.param(f'{HOOK}supplier = {{ name = "Org", email = "x" }}\n', "`supplier`", id="supplier-unknown-key"), - pytest.param(f"{HOOK}repository = 1\n", "`repository` in build hook `python-sbom` must be a string", id="repo"), - pytest.param( - f"{HOOK}default-branch = 1\n", "`default-branch` in build hook `python-sbom` must be", id="branch" - ), - pytest.param(f'{HOOK}bundled-wheels = "*.whl"\n', "`bundled-wheels` in build hook `python-sbom`", id="bundled"), - ], -) -def test_invalid_option(build_sbom: BuildSbom, hook: str, message: str) -> None: - with pytest.raises(TypeError, match=message): - build_sbom(hook=hook) - - -def test_bundled_wheel_variants_unique_refs(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: - wheel = bundle_wheel({}) - shutil.copyfile(wheel, wheel.with_name("pip-1.0-py3-none-manylinux_2_17_x86_64.whl")) - assert [component["bom-ref"] for component in build_sbom(hook=f"{HOOK}{BUNDLED}")["components"][:2]] == [ - "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-any.whl", - "pkg:pypi/pip@1.0?file_name=pip-1.0-py3-none-manylinux_2_17_x86_64.whl", - ] - - -@pytest.mark.parametrize( - ("requirement", "expected"), - [ - pytest.param("foo; extra != 'fast'", ["foo"], id="not-an-extra"), - pytest.param("foo; extra == 'fast' or python_version < '3.12'", ["foo"], id="extra-or-platform"), - pytest.param("foo; python_version < '3.12' and extra == 'fast'", [], id="platform-and-extra"), - pytest.param("foo; (extra == 'a' or extra == 'b') and os_name == 'nt'", [], id="grouped-extras"), - pytest.param("foo; os_name == 'nt'", ["foo"], id="platform-only"), - pytest.param("foo; 'fast' == extra", [], id="extra-on-the-right"), - ], -) -def test_runtime_requirements_with_extra_markers(build_sbom: BuildSbom, requirement: str, expected: list[str]) -> None: - project = f'[project]\nname = "example"\nversion = "1.0"\ndependencies = ["{requirement}"]\n' - assert [component["name"] for component in build_sbom(project=project)["components"]] == expected - - -def test_bundled_wheel_without_metadata(root: Path, build_sbom: BuildSbom) -> None: - (wheels := root / "src" / "example" / "wheels").mkdir(parents=True) - with zipfile.ZipFile(wheels / "pip-1.0-py3-none-any.whl", "w") as archive: - archive.writestr("pip/__init__.py", "") - with pytest.raises(WheelError, match=r"pip-1\.0-py3-none-any\.whl has no \.dist-info/METADATA"): - build_sbom(hook=f"{HOOK}{BUNDLED}") - - -def test_vendored_requirement_unparsable(build_sbom: BuildSbom, bundle_wheel: BundleWheel) -> None: - bundle_wheel({"pip/_vendor/vendor.txt": "not a requirement!\n"}) - with pytest.raises(WheelError, match="does not parse: not a requirement!"): - build_sbom(hook=f"{HOOK}{BUNDLED}") diff --git a/tests/test_spdx.py b/tests/test_spdx.py deleted file mode 100644 index 526e0d8..0000000 --- a/tests/test_spdx.py +++ /dev/null @@ -1,198 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Final - -import pytest -from conftest import BUNDLED, HOOK - -from python_sbom import __version__, to_spdx - -if TYPE_CHECKING: - from pathlib import Path - - from conftest import BuildSbom, BundleWheel - - from python_sbom import Document - -_SUPPLIED: Final[str] = f'{HOOK}{BUNDLED}supplier = {{ name = "Example Org" }}\n' - - -@pytest.fixture -def bundled(bundle_wheel: BundleWheel) -> None: - bundle_wheel( - {"pip/_vendor/vendor.txt": "urllib3==1.26.4\n"}, "Project-URL: Source, https://github.com/example/pip\n" - ) - - -@pytest.mark.usefixtures("bundled") -def test_spdx_bundled_wheel(build_sbom: BuildSbom) -> None: - cyclonedx: Final[Document] = build_sbom(hook=_SUPPLIED) - assert to_spdx(cyclonedx)["packages"][1] == { - "SPDXID": "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-ccff93ba", - "name": "pip", - "downloadLocation": "NOASSERTION", - "filesAnalyzed": False, - "licenseConcluded": "NOASSERTION", - "licenseDeclared": "NOASSERTION", - "primaryPackagePurpose": "LIBRARY", - "versionInfo": "1.0", - "supplier": "Organization: Example Org", - "checksums": [ - {"algorithm": "SHA256", "checksumValue": cyclonedx["components"][0].get("hashes", [])[0]["content"]} - ], - "comment": "example:bundled-wheel: src/example/wheels/pip-1.0-py3-none-any.whl", - "externalRefs": [ - {"referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", "referenceLocator": "pkg:pypi/pip@1.0"} - ], - } - - -def test_spdx_root_package(build_sbom: BuildSbom) -> None: - assert to_spdx(build_sbom())["packages"][0] == { - "SPDXID": "SPDXRef-pkg-pypi-example-1.0-42410e0c", - "name": "Example", - "downloadLocation": "NOASSERTION", - "filesAnalyzed": False, - "licenseConcluded": "NOASSERTION", - "licenseDeclared": "MIT", - "primaryPackagePurpose": "APPLICATION", - "versionInfo": "1.0", - "copyrightText": "Copyright (c) 2026 Example", - "summary": "an example", - "comment": "example:requires-python: >=3.11", - "externalRefs": [ - { - "referenceCategory": "PACKAGE-MANAGER", - "referenceType": "purl", - "referenceLocator": "pkg:pypi/example@1.0", - } - ], - } - - -@pytest.mark.usefixtures("bundled") -def test_spdx_relationships(build_sbom: BuildSbom) -> None: - urllib3: Final[str] = ( - "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-vendored-pkg-pypi-urllib3-1.26.4-82ad4d8f" - ) - assert [ - relationship - for relationship in to_spdx(build_sbom(hook=_SUPPLIED))["relationships"] - if not relationship["spdxElementId"].startswith("SPDXRef-tool-") - ] == [ - { - "spdxElementId": "SPDXRef-DOCUMENT", - "relationshipType": "DESCRIBES", - "relatedSpdxElement": "SPDXRef-pkg-pypi-example-1.0-42410e0c", - }, - { - "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-ccff93ba", - "relationshipType": "CONTAINS", - "relatedSpdxElement": urllib3, - }, - { - "spdxElementId": "SPDXRef-pkg-pypi-example-1.0-42410e0c", - "relationshipType": "DEPENDS_ON", - "relatedSpdxElement": "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-ccff93ba", - }, - { - "spdxElementId": "SPDXRef-pkg-pypi-example-1.0-42410e0c", - "relationshipType": "DEPENDS_ON", - "relatedSpdxElement": "SPDXRef-requires-dist-packaging-24-45a0ad48", - }, - { - "spdxElementId": "SPDXRef-pkg-pypi-example-1.0-42410e0c", - "relationshipType": "DEPENDS_ON", - "relatedSpdxElement": "SPDXRef-requires-dist-tomli-2-python-version-3.11-7c43c831", - }, - { - "spdxElementId": "SPDXRef-pkg-pypi-pip-1.0-file-name-pip-1.0-py3-none-any.whl-ccff93ba", - "relationshipType": "DEPENDS_ON", - "relatedSpdxElement": urllib3, - }, - ] - - -def test_spdx_build_tools(build_sbom: BuildSbom) -> None: - assert { - relationship["relatedSpdxElement"] - for relationship in to_spdx(build_sbom())["relationships"] - if relationship["relationshipType"] == "BUILD_TOOL_OF" - } == {"SPDXRef-pkg-pypi-example-1.0-42410e0c"} - - -@pytest.mark.parametrize( - ("metadata", "expected"), - [ - pytest.param("License-Expression: MIT\n", ("MIT", "NOASSERTION", None), id="expression"), - pytest.param( - "Classifier: License :: OSI Approved :: MIT License\nLicense: MIT\n", - ("NOASSERTION", "MIT", "Declared in package metadata as: MIT License; MIT"), - id="names", - ), - pytest.param( - "License: LGPLv3\n", - ("NOASSERTION", "NOASSERTION", "Declared in package metadata as: LGPLv3"), - id="unresolved-name", - ), - pytest.param("", ("NOASSERTION", "NOASSERTION", None), id="undeclared"), - ], -) -def test_spdx_declared_license( - build_sbom: BuildSbom, bundle_wheel: BundleWheel, metadata: str, expected: tuple[str, str, str | None] -) -> None: - bundle_wheel({}, metadata) - package = to_spdx(build_sbom(hook=f"{HOOK}{BUNDLED}"))["packages"][1] - assert (package["licenseDeclared"], package["licenseConcluded"], package.get("licenseComments")) == expected - - -@pytest.mark.parametrize( - ("hook", "expected"), - [ - pytest.param( - f'{HOOK}supplier = {{ name = "Example Org" }}\n', - ("https://github.com/example/example/sboms/Example-1.0-", ["Organization: Example Org"]), - id="repository-and-supplier", - ), - pytest.param( - '[tool.hatch.build.hooks.custom]\npath = "build.py"\n', - ("https://github.com/example/example/sboms/Example-1.0-", ["Tool: build.py-1.0"]), - id="custom-build-script", - ), - ], -) -def test_spdx_creation_info( - root: Path, build_sbom: BuildSbom, monkeypatch: pytest.MonkeyPatch, hook: str, expected: tuple[str, list[str]] -) -> None: - monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") - (root / "build.py").write_text( - "from python_sbom.hatch import SbomBuildHook\n\n\ndef get_build_hook():\n return SbomBuildHook\n", - encoding="utf-8", - ) - cyclonedx: Final[Document] = build_sbom(hook=hook) - spdx = to_spdx(cyclonedx) - namespace, creators = expected - assert (spdx["documentNamespace"], spdx["creationInfo"], spdx["comment"]) == ( - f"{namespace}{cyclonedx['serialNumber'].removeprefix('urn:uuid:')}", - {"created": "2023-11-14T22:13:20Z", "creators": [*creators, f"Tool: python-sbom-{__version__}"]}, - ( - f"Rendered from the CycloneDX SBOM {cyclonedx['serialNumber']} (example.cdx.json), which also carries" - " per-file hashes and the build formulation." - ), - ) - - -def test_spdx_namespace_without_repository(build_sbom: BuildSbom) -> None: - cyclonedx: Final[Document] = build_sbom(project='[project]\nname = "Example"\nversion = "1.0"\n') - assert to_spdx(cyclonedx)["documentNamespace"] == ( - f"https://spdx.org/spdxdocs/sboms/Example-1.0-{cyclonedx['serialNumber'].removeprefix('urn:uuid:')}" - ) - - -def test_spdx_conditional_requirements_unique_ids(build_sbom: BuildSbom) -> None: - project = """[project]\nname = "example"\nversion = "1.0" -dependencies = ["foo; python_version < '3.12'", "foo; python_version >= '3.12'"]\n""" - ids = [ - package["SPDXID"] for package in to_spdx(build_sbom(project=project))["packages"] if package["name"] == "foo" - ] - assert (len(ids), len(set(ids))) == (2, 2) diff --git a/tests/test_validate.py b/tests/test_validate.py deleted file mode 100644 index 000cc4e..0000000 --- a/tests/test_validate.py +++ /dev/null @@ -1,258 +0,0 @@ -from __future__ import annotations - -import json -import zipfile -from typing import TYPE_CHECKING, Final - -import pytest - -from docs.examples.zipapp import describe_zipapp -from python_sbom import validate, validate_wheel - -if TYPE_CHECKING: - from pathlib import Path - - from conftest import BuildSbom - from pytest_mock import MockerFixture - - from python_sbom import Component, Document - from python_sbom._model._types import License - -_ALLOWED: Final[frozenset[str]] = frozenset({"Apache-2.0", "BSD-3-Clause", "MIT"}) - - -def _expression(expression: str) -> License: - return {"expression": expression, "acknowledgement": "declared"} - - -def _named(name: str) -> License: - return {"license": {"name": name, "acknowledgement": "declared"}} - - -def _concluded(identifier: str) -> License: - return {"license": {"id": identifier, "acknowledgement": "concluded"}} - - -def _denied(expression: str, bom_ref: str = "pkg:pypi/bundled@1.0") -> str: - return f"{bom_ref}: license {expression} is outside the allowed licenses ['Apache-2.0', 'BSD-3-Clause', 'MIT']" - - -@pytest.fixture -def sbom_document(build_sbom: BuildSbom, mocker: MockerFixture) -> Document: - # schema validation time grows with the document, and the test environment's packages are irrelevant here - mocker.patch("python_sbom._collect._facts.distributions", autospec=True, return_value=[]) - return build_sbom() - - -def _with_component(document: Document, component: Component) -> str: - document["components"].append(component) - document["dependencies"][0]["dependsOn"].append(component["bom-ref"]) - return json.dumps(document) - - -def test_document_valid(sbom_document: Document) -> None: - assert validate(json.dumps(sbom_document), _ALLOWED) == [] - - -def test_document_schema_violation() -> None: - problems: Final[list[str]] = validate( - json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "1"}), None - ) - assert [problem.splitlines()[0] for problem in problems] == [ - "does not conform to the CycloneDX 1.6 schema: '1' is not of type 'integer'" - ] - - -def test_document_serial_number_missing(sbom_document: Document) -> None: - unnumbered = json.loads(json.dumps(sbom_document)) - del unnumbered["serialNumber"] - assert validate(json.dumps(unnumbered), None) == ["serialNumber must be a urn:uuid: RFC 4122 UUID, got ''"] - - -def test_document_duplicate_bom_ref(sbom_document: Document) -> None: - sbom_document["components"][1]["bom-ref"] = sbom_document["components"][0]["bom-ref"] - assert validate(json.dumps(sbom_document), None)[0] == ( - "bom-ref values are not unique: ['requires-dist:packaging>=24']" - ) - - -def test_document_dangling_dependency(sbom_document: Document) -> None: - sbom_document["dependencies"].append({"ref": "pkg:pypi/example@1.0", "dependsOn": ["pkg:pypi/missing"]}) - problems: Final[list[str]] = validate(json.dumps(sbom_document), None) - assert problems[0] == "dependencies reference bom-refs that do not exist: ['pkg:pypi/missing']" - - -def test_document_root_dependencies(sbom_document: Document) -> None: - sbom_document["dependencies"][0]["dependsOn"] = ["requires-dist:packaging>=24"] - assert validate(json.dumps(sbom_document), None) == [ - ( - "root dependsOn ['requires-dist:packaging>=24'] != components" - " ['requires-dist:packaging>=24', \"requires-dist:tomli>=2; python_version < '3.11'\"]" - ) - ] - - -def test_document_workflow_unknown_tool(sbom_document: Document) -> None: - sbom_document["formulation"][0]["workflows"][0]["resourceReferences"].append({"ref": "tool:missing"}) - assert validate(json.dumps(sbom_document), None) == [ - "workflow wheel-build references unknown tools: ['tool:missing']" - ] - - -def test_document_spdx_unparsable(sbom_document: Document) -> None: - sbom_document["metadata"]["component"]["hashes"] = [{"alg": "BLAKE2b-256", "content": "0" * 64}] - assert validate(json.dumps(sbom_document), None)[0].startswith("SPDX rendering does not parse:") - - -def test_document_spdx_invalid(sbom_document: Document) -> None: - sbom_document["metadata"]["component"]["purl"] = "not a purl" - assert validate(json.dumps(sbom_document), None)[0].startswith("SPDX rendering invalid:") - - -def test_wheel_without_sbom(tmp_path: Path) -> None: - with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: - archive.writestr("example-1.0.dist-info/METADATA", "Name: example\n") - assert validate_wheel(wheel, None) == ["no .dist-info/sboms/*.cdx.json found in the wheel"] - - -def test_wheel_names_each_sbom(sbom_document: Document, tmp_path: Path) -> None: - unnumbered = json.loads(json.dumps(sbom_document)) - del unnumbered["serialNumber"] - with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: - archive.writestr("example-1.0.dist-info/sboms/example.cdx.json", json.dumps(unnumbered)) - assert validate_wheel(wheel, None) == [ - "example-1.0.dist-info/sboms/example.cdx.json: serialNumber must be a urn:uuid: RFC 4122 UUID, got ''" - ] - - -@pytest.mark.parametrize( - ("licenses", "expected"), - [ - pytest.param([_expression("MIT")], [], id="allowed"), - pytest.param([_expression("mit")], [], id="allowed-any-case"), - pytest.param([_expression("GPL-3.0-only")], [_denied("GPL-3.0-only")], id="denied"), - pytest.param([_expression("Apache-2.0 OR GPL-3.0-only")], [], id="or-one-allowed"), - pytest.param( - [_expression("GPL-3.0-only OR MPL-2.0")], [_denied("GPL-3.0-only OR MPL-2.0")], id="or-none-allowed" - ), - pytest.param([_expression("MIT AND BSD-3-Clause")], [], id="and-all-allowed"), - pytest.param([_expression("MIT AND GPL-3.0-only")], [_denied("MIT AND GPL-3.0-only")], id="and-one-denied"), - pytest.param([_expression("MIT OR GPL-3.0-only AND LGPL-3.0-only")], [], id="and-binds-tighter"), - pytest.param( - [_expression("(MIT OR GPL-3.0-only) AND LGPL-3.0-only")], - [_denied("(MIT OR GPL-3.0-only) AND LGPL-3.0-only")], - id="parentheses", - ), - pytest.param([_expression("Apache-2.0 WITH LLVM-exception")], [], id="with-allowed"), - pytest.param( - [_expression("GPL-2.0-only WITH Classpath-exception-2.0")], - [_denied("GPL-2.0-only WITH Classpath-exception-2.0")], - id="with-denied", - ), - pytest.param([_named("MIT License"), _concluded("MIT")], [], id="concluded-allowed"), - pytest.param([_named("MPL 2.0"), _concluded("MPL-2.0")], [_denied("MPL-2.0")], id="concluded-denied"), - pytest.param( - [_concluded("MIT"), _concluded("LGPL-3.0-only")], [_denied("LGPL-3.0-only")], id="concluded-all-apply" - ), - pytest.param( - [_named("MIT License")], - ["pkg:pypi/bundled@1.0: declares license names ['MIT License'] with no SPDX id"], - id="name-only", - ), - pytest.param([], ["pkg:pypi/bundled@1.0: declares no license"], id="undeclared"), - ], -) -def test_license_policy(sbom_document: Document, licenses: list[License], expected: list[str]) -> None: - component: Final[Component] = { - "type": "library", - "bom-ref": "pkg:pypi/bundled@1.0", - "name": "example", - "hashes": [{"alg": "SHA-256", "content": "0" * 64}], - "licenses": licenses, - } - assert validate(_with_component(sbom_document, component), _ALLOWED) == expected - - -@pytest.mark.parametrize( - ("expression", "reason"), - [ - pytest.param("MIT AND", "it ends early", id="trailing-operator"), - pytest.param("OR MIT", "unexpected 'OR'", id="leading-operator"), - pytest.param("(MIT", "it ends early", id="unclosed"), - pytest.param("MIT)", "unexpected ')'", id="unopened"), - pytest.param("(MIT Apache-2.0)", "expected ')', got 'Apache-2.0'", id="missing-operator"), - pytest.param("MIT and Apache-2.0", "unexpected 'and'", id="lower-case-operator"), - pytest.param("Apache-2.0 WITH OR", "unexpected 'OR'", id="with-operator"), - ], -) -def test_license_policy_malformed(sbom_document: Document, expression: str, reason: str) -> None: - component: Final[Component] = { - "type": "library", - "bom-ref": "pkg:pypi/bundled@1.0", - "name": "example", - "hashes": [{"alg": "SHA-256", "content": "0" * 64}], - "licenses": [_expression(expression)], - } - # an expression the policy cannot parse does not parse as SPDX either, which the check reports first - assert validate(_with_component(sbom_document, component), _ALLOWED)[-1] == ( - f"pkg:pypi/bundled@1.0: license expression {expression!r} is malformed: {reason}" - ) - - -@pytest.mark.parametrize( - "component", - [ - pytest.param( - {"type": "library", "bom-ref": "requires-dist:distlib", "name": "distlib"}, id="declared-requirement" - ), - pytest.param( - { - "type": "library", - "bom-ref": "pkg:pypi/pip@1.0", - "name": "pip", - "hashes": [{"alg": "SHA-256", "content": "0" * 64}], - "licenses": [_expression("MIT")], - "components": [ - {"type": "library", "bom-ref": "pkg:pypi/pip@1.0#vendored/pkg:pypi/autocommand@2.2.2", "name": "a"}, - ], - }, - id="vendored-by-bundled-wheel", - ), - ], -) -def test_license_policy_skips_unshipped(sbom_document: Document, component: Component) -> None: - assert validate(_with_component(sbom_document, component), _ALLOWED) == [] - - -def test_license_policy_judges_files_owner(sbom_document: Document) -> None: - component: Final[Component] = { - "type": "library", - "bom-ref": "pkg:pypi/distlib@0.4", - "name": "distlib", - "licenses": [_concluded("LGPL-3.0-only")], - "components": [{"type": "file", "bom-ref": "pkg:pypi/distlib@0.4#distlib/__init__.py", "name": "x"}], - } - assert validate(_with_component(sbom_document, component), _ALLOWED) == [ - _denied("LGPL-3.0-only", "pkg:pypi/distlib@0.4") - ] - - -def test_document_missing_structure() -> None: - serial = f"urn:uuid:{'0' * 8}-0000-0000-0000-{'0' * 12}" - raw = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "serialNumber": serial}) - assert validate(raw, None) == [ - ( - "lacks metadata.timestamp, metadata.tools.components, metadata.component.bom-ref, metadata.component.name," - " components, dependencies, which every document python-sbom writes carries" - ) - ] - - -def test_documented_zipapp_example(tmp_path: Path) -> None: - assert validate(json.dumps(describe_zipapp(tmp_path, "21.0")), None) == [] - - -def test_versionless_root(tmp_path: Path) -> None: - document = describe_zipapp(tmp_path, "21.0") - del document["metadata"]["component"]["version"] - assert validate(json.dumps(document), None) == [] diff --git a/tox.toml b/tox.toml index 177bca1..1ddff03 100644 --- a/tox.toml +++ b/tox.toml @@ -10,7 +10,6 @@ wheel_build_env = ".pkg" dependency_groups = [ "test" ] pass_env = [ "PYTEST_*" ] set_env.COVERAGE_FILE = { replace = "env", name = "COVERAGE_FILE", default = "{env_tmp_dir}{/}.coverage" } -set_env.COVERAGE_FILECOVERAGE_PROCESS_START = "{tox_root}{/}pyproject.toml" commands = [ [ "pytest", From c4bc75c6b9a262e3797627cd1b17ee462d2e4793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 22:08:40 -0800 Subject: [PATCH 015/115] =?UTF-8?q?=F0=9F=93=A6=20build:=20embed=20the=20p?= =?UTF-8?q?roject's=20own=20SBOM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A custom hook loads the shipped hook from the source tree, since the package cannot be a build requirement of itself, and pkg_meta fails when the built wheel lacks .dist-info/sboms/. --- hatch_build.py | 25 +++++++++++++++++++++++++ pyproject.toml | 2 ++ tox.toml | 9 +++++++++ 3 files changed, 36 insertions(+) create mode 100644 hatch_build.py diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 0000000..85f50c4 --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + + +class SelfSbomHook(BuildHookInterface): # simplint: ignore[SMP409] # hatchling loads it from this script + def finalize( + self, + version: str, + build_data: dict[str, Any], # simplint: ignore[SMP602] # hatchling's signature + artifact_path: str, + ) -> None: + # python-sbom cannot be a build requirement of itself, so its hook comes from this source tree; by now the vcs + # hook has written the version module the package imports + sys.path.insert(0, str(Path(self.root) / "src")) + from python_sbom.hatch import SbomBuildHook # ruff: ignore[import-outside-top-level] # see above + + hook = SbomBuildHook( + self.root, self.config, self.build_config, self.metadata, self.directory, self.target_name, self.app + ) + hook.finalize(version, build_data, artifact_path) diff --git a/pyproject.toml b/pyproject.toml index b451fa1..bd6ab9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,6 +118,8 @@ version.source = "vcs" metadata.hooks.fancy-pypi-readme.content-type = "text/markdown" metadata.hooks.fancy-pypi-readme.fragments = [ { path = "README.md" } ] build.hooks.vcs.version-file = "src/python_sbom/version.py" +# writes python-sbom's own SBOM with the hook it ships, after the vcs hook writes the version +build.hooks.custom.path = "hatch_build.py" [tool.uv] dependency-groups.release-news = { requires-python = ">=3.15" } diff --git a/tox.toml b/tox.toml index 1ddff03..1d46288 100644 --- a/tox.toml +++ b/tox.toml @@ -111,6 +111,15 @@ commands = [ "--no-config", "{env_tmp_dir}", ], + [ + "python", + "-c", + # the project's own wheel carries the SBOM its hook writes + """\ + import glob, zipfile; (wheel,) = glob.glob(r'{env_tmp_dir}{/}*.whl'); assert any('/sboms/' in name for name in \ + zipfile.ZipFile(wheel).namelist()), f'no SBOM in {wheel}'\ + """, + ], ] [env.type] From 2891f8aa2f8bab97bc2e808b08156cbab50157cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 22:08:40 -0800 Subject: [PATCH 016/115] =?UTF-8?q?=F0=9F=93=9D=20docs:=20document=20provi?= =?UTF-8?q?ders=20and=20the=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover the content provider, the backends and the pipeline, the source revision chain, embedding after auditwheel, and why build tools are listed down to their files. Redraw the architecture diagram from the import graph, render type aliases and the TypedDicts, and keep hatchling's lifecycle methods out of the API reference. --- README.md | 11 +++ docs/cli.rst | 2 +- docs/conf.py | 32 ++++++--- docs/contributing.rst | 2 + docs/explanation.rst | 103 +++++++++++++++++----------- docs/extend.rst | 155 +++++++++++++++++++++++++----------------- docs/py_api.rst | 135 ++++++++++++++++++++++++++++-------- docs/usage.rst | 74 ++++++++++++++------ 8 files changed, 352 insertions(+), 162 deletions(-) diff --git a/README.md b/README.md index 5592f8c..f99da24 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,23 @@ [![check](https://github.com/tox-dev/python-sbom/actions/workflows/check.yaml/badge.svg)](https://github.com/tox-dev/python-sbom/actions/workflows/check.yaml) [![Documentation Status](https://readthedocs.org/projects/python-sbom/badge/?version=latest)](https://python-sbom.readthedocs.io/en/latest/?badge=latest) [![Downloads](https://static.pepy.tech/badge/python-sbom/month)](https://pepy.tech/project/python-sbom) +[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/tox-dev/python-sbom/badge)](https://scorecard.dev/viewer/?uri=github.com/tox-dev/python-sbom) SBOMs for Python distributions. python-sbom writes a [CycloneDX](https://cyclonedx.org/) SBOM into every wheel you build, where [PEP 770](https://peps.python.org/pep-0770/) tells installers to keep it. The SBOM records your project, the runtime dependencies it declares, any wheels it bundles, and the build toolchain, and a rebuild with the same inputs reproduces it byte for byte. +## Install + +The command line, to embed, convert and validate SBOMs: + +```console +pipx install 'python-sbom[validate]' +``` + +A hatchling project names `python-sbom[hatch]` in its build requirements instead, as shown below. + ## Use With [hatchling](https://hatch.pypa.io/latest/), add the hook to the build requirements and enable it: diff --git a/docs/cli.rst b/docs/cli.rst index dc29159..8d3ba9a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1,5 +1,5 @@ .. sphinx_argparse_cli:: :module: python_sbom._cli._parser - :func: _build_parser + :func: build_parser :title: Command line interface :group_title_prefix: diff --git a/docs/conf.py b/docs/conf.py index f776780..2ee4df6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,7 +49,7 @@ def _inheritance_aliases() -> dict[str, str]: autosectionlabel_maxdepth: Final[int] = 2 html_theme: Final[str] = "furo" -html_title: Final[str] = "python-sbom" +html_title: Final[str] = "python-sbom: SBOMs for Python distributions" html_last_updated_fmt: Final[str] = _NOW.isoformat() html_static_path: Final[list[str]] = ["_static"] html_css_files: Final[list[str]] = ["changelog-toc.css"] @@ -59,12 +59,13 @@ def _inheritance_aliases() -> dict[str, str]: autoclass_content: Final[str] = "both" inheritance_alias: Final[dict[str, str]] = _inheritance_aliases() inheritance_graph_attrs: Final[dict[str, str]] = {"rankdir": "TB"} -autodoc_default_options: Final[dict[str, bool | str]] = { - "members": True, - "member-order": "bysource", - "undoc-members": True, - "show-inheritance": True, -} +# each directive names the members it documents, so hatchling's lifecycle methods and inherited members stay out +autodoc_default_options: Final[dict[str, bool | str]] = {"member-order": "bysource", "show-inheritance": True} +autodoc_type_aliases: Final[dict[str, str]] = {"Toml": "python_sbom.Toml"} +typehints_fully_qualified: Final[bool] = False +always_document_param_types: Final[bool] = True +typehints_document_rtype: Final[bool] = True +typehints_defaults: Final[str] = "comma" intersphinx_mapping: Final[dict[str, tuple[str, None]]] = {"python": ("https://docs.python.org/3.11", None)} extlinks: Final[dict[str, tuple[str, str]]] = { @@ -75,13 +76,20 @@ def _inheritance_aliases() -> dict[str, str]: nitpick_ignore: Final[list[tuple[str, str]]] = [ # hatchling publishes no Sphinx inventory ("py:class", "hatchling.builders.hooks.plugin.interface.BuildHookInterface"), + ("py:class", "hatchling.metadata.core.ProjectMetadata"), + ("py:class", "hatchling.bridge.app.Application"), # the Python documentation describes the protocol without a reference target ("py:class", "importlib.metadata._meta.PackageMetadata"), ] +# parts of the document types that the CycloneDX and SPDX references describe +nitpick_ignore_regex: Final[list[tuple[str, str]]] = [ + ("py:class", r"python_sbom\._model\._types\.(Composition|LicenseInfo|Metadata)"), + ("py:class", r"python_sbom\._formats\._spdx\._\w+"), +] def setup(app: Sphinx) -> None: - # without this hook Sphinx resolves pathlib.Path to an undocumented private alias + # annotations name the private modules that define a type, and pathlib.Path an undocumented private alias app.connect("missing-reference", _resolve_private_path) app.connect("source-read", _include_unreleased) @@ -92,9 +100,13 @@ def _resolve_private_path( node: pending_xref, contnode: Element, ) -> reference | None: - if node["reftarget"] != "pathlib._local.Path": + module, _, name = node["reftarget"].rpartition(".") + if node["reftarget"] == "pathlib._local.Path": + node["reftarget"] = "pathlib.Path" + elif module.startswith("python_sbom._") and name in python_sbom.__all__: + node["reftarget"] = f"python_sbom.{name}" + else: return None - node["reftarget"] = "pathlib.Path" domain = env.get_domain("py") return domain.resolve_xref(env, node["refdoc"], app.builder, node["reftype"], node["reftarget"], node, contnode) diff --git a/docs/contributing.rst b/docs/contributing.rst index a71e5a9..0a1f58d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -26,3 +26,5 @@ they include one. The release workflow links each fragment to its merged pull request and credits its human author. If several people authored a change, put their `GitHub `_ handles in a ``by`` credit in the fragment; omit bots and reviewers. The changelog uses linked names, while ``GitHub`` Release notes use ``@handle``. + +Build the documentation with ``tox r -e docs``; ``tox r -e docs -- -b linkcheck`` checks its external links. diff --git a/docs/explanation.rst b/docs/explanation.rst index b3cd785..b33cd4a 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -4,41 +4,56 @@ How it works Architecture ------------ -The package keeps each concern in its own place, and only the backends know a build system. A backend gathers what -the edge knows: the wheel's metadata and file name, the project's settings, and, through -:class:`~python_sbom.BuildFacts`, the build time, the source revision and the build environment. The core turns those -explicit inputs into the document with pure functions, then serializes it. The hatchling hook hands the result to -hatchling; :class:`~python_sbom.BuiltWheel` writes it into a finished wheel. SPDX rendering and validation read a -document and nothing else. +Each concern has its own subpackage, and only the adapters know a build system or the command line. The hatchling hook +and the ``embed`` command gather what the edge knows, then call one pipeline function, :func:`~python_sbom.build_sbom`, +with explicit inputs: the :class:`~python_sbom.SbomRequest` a :class:`~python_sbom.Backend` returns, the +:class:`~python_sbom.SourceTree` and :class:`~python_sbom.BuildFacts` the collectors read, and a +:class:`~python_sbom.ContentProvider`. The model turns those inputs into the document with pure functions. The diagram +follows the imports between the subpackages; an arrow points at what a package imports. .. mermaid:: - flowchart LR - subgraph backends[Backends] - hatch[python_sbom.hatch
hatchling build hook] - built[BuiltWheel
wheel from any backend] + flowchart TB + subgraph adapters[Adapters] + hook[hatch
hatchling build hook] + cli[_cli
embed, spdx, validate] end - subgraph collect[Collect, at the edge] - settings[Settings
hook table or tool.python-sbom table] - facts[BuildFacts
time, revision, build tools] - source[Source tree
bundled wheels, license files] - end - model[Model
pure document building] - formats[Formats
CycloneDX JSON, SPDX 2.3] - validate[Validate
schema, references, licenses] - hatch & built --> settings & facts & source - settings & facts & source --> model - formats --> hatch & built & validate - model --> formats + generate[_generate
Backend, ContentProvider, build_sbom] + collect[_collect
settings, build facts, source tree] + validate[_validate
schema, references, licenses] + formats[_formats
CycloneDX JSON, SPDX 2.3] + wheel[_wheel
RECORD rows, archive rewrite] + model[_model
inputs, types, pure document building] + hook --> generate + hook --> collect + hook --> wheel + cli --> generate + cli --> collect + cli --> validate + cli --> wheel + generate --> formats + generate --> wheel + generate --> model + collect --> wheel + collect --> model + validate --> formats + validate --> wheel + validate --> model + formats --> model + wheel --> model classDef edge fill:#cfe4ff,stroke:#1f6feb,color:#0b1f3a; classDef core fill:#d8f3dc,stroke:#2d6a4f,color:#081c15; - class hatch,built,settings,facts,source edge; - class model,formats,validate core; + class hook,cli,collect edge; + class generate,validate,formats,wheel,model core; + +The model imports nothing else of the package, and nothing below the adapters imports hatchling, so the library and +the command line work without it. Every package raises subclasses of :class:`~python_sbom.SbomError`, which the command +line reports without a traceback. -Extension points are explicit: a new build system subclasses :class:`~python_sbom.Backend`, a project adds content -through the :class:`~python_sbom.ContentProvider` method its hook or backend implements, and another version control -system plugs in as a :class:`~python_sbom.RevisionReader`. Errors are subclasses of :class:`~python_sbom.SbomError`, so -the command line reports them without a traceback. +Extension points are explicit: a build system plugs in as a :class:`~python_sbom.Backend`, a project adds content as a +:class:`~python_sbom.ContentProvider`, and another version control system as a :class:`~python_sbom.RevisionReader`. +The hatch hook exposes two factories, :meth:`~python_sbom.hatch.SbomBuildHook.content_provider` and +:meth:`~python_sbom.hatch.SbomBuildHook.backend`, for a subclass to override. What the SBOM records --------------------- @@ -54,18 +69,28 @@ declared dependency carries no version, because the installer picks it at instal dependencies incomplete for that reason. An extra's dependencies are left out. The formulation records how the wheel was built: the interpreter's version and implementation, every distribution of -the build environment with its dependencies, the files and hashes of each pure-Python one, and ``SOURCE_DATE_EPOCH``. +the build environment with its dependencies, the files and hashes of each pure-Python one, and ``SOURCE_DATE_EPOCH`` +when it set the build time. Property names start with the project's normalized name, such as ``virtualenv:bundled-wheel``, or with ``python:`` for values copied from core metadata. Reproducibility --------------- -The document holds nothing about the build machine. Its timestamp comes from ``SOURCE_DATE_EPOCH``, or hatchling's -fixed default of 2020-02-02 when that is unset. Its serial number derives from the content, so identical documents -share one and any difference gets a new one. Platform wheels in the build environment appear without their files, -because those differ per operating system and architecture. Installer metadata and console-script launchers are left -out too, since they name the frontend and the environment's path rather than the distribution. +The document holds nothing about the build machine. Its timestamp comes from ``SOURCE_DATE_EPOCH``. When that is +unset, a reproducible hatchling build uses hatchling's fixed 2020-02-02, the time it stamps on the wheel's entries, and +any other build uses the current time. Its serial number derives from the content, so identical documents share one +and any difference gets a new one. + +The build tools are listed down to their files, so a rebuild can check it installed the same bytes. Only what is the +same on every machine goes in: platform wheels appear without their files, because those differ per operating system +and architecture, and editable installs without theirs, because they point at a checkout. Installer metadata, +console-script launchers and uv's cache files are left out too, since they name the frontend, the environment's path or +a time rather than the distribution. + +Hashing each tool's ``RECORD`` instead would be smaller, but that file lists the installer metadata, so two frontends +installing the same tools would give two hashes. The cost is size: python-sbom's own isolated build lists about 600 +files, half of them from setuptools, which setuptools-scm pulls in, for an SBOM of about 400 KB. So a rebuild reproduces the wheel byte for byte on any operating system when these match the original build: the source tree, as a checkout of the same commit; ``SOURCE_DATE_EPOCH``; the Python version; and the versions of the @@ -77,8 +102,10 @@ Limitations - The document describes what the project declares and bundles, not the full tree of runtime dependencies, which depends on the environment the wheel is installed into. - The links to the PyPI release and attestation assume the project publishes to PyPI with attestations. -- A build from an sdist has no git checkout, so its SBOM names no commit. -- ``python-sbom embed`` records the environment it runs in. Unless the wheel was built without isolation in that - environment, the build tool record does not match the real build. +- A build from an sdist has no git checkout, so its SBOM names no commit unless ``PYTHON_SBOM_COMMIT`` or an expanded + ``.git_archival.txt`` supplies it. +- ``python-sbom embed`` records the environment ``--build-env`` names, by default the one it runs in. Unless that is + the environment the wheel was built in, the build tool record does not match the real build. - The SPDX rendering leaves out per-file hashes and the build record, which SPDX 2.3 cannot hold. -- A hook that changes the wheel tag after this one runs leaves the attestation link pointing at the wrong file name. +- A tool that renames the wheel after the build, such as ``auditwheel repair``, leaves the attestation link pointing at + the old file name until ``python-sbom embed`` runs on the renamed wheel. diff --git a/docs/extend.rst b/docs/extend.rst index a14576e..0adb8aa 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -1,25 +1,43 @@ Extend it ========= -The options cover what every project shares. When a project knows more about a bundled wheel than its metadata says, -it subclasses the hook. When it ships an artifact other than a wheel, it builds that artifact's SBOM from the same -pieces the hook uses. +The options cover what every project shares. A project that knows more than its metadata says adds it through a +content provider. A build system other than hatchling plugs in as a backend. An artifact other than a wheel gets its +SBOM from the same public pieces the hook uses. -Add properties to bundled wheels --------------------------------- +Add content with a provider +--------------------------- -:class:`~python_sbom.hatch.SbomBuildHook` calls :meth:`~python_sbom.hatch.SbomBuildHook.bundled_wheel_properties` once per -bundled wheel. Override it in a hatchling `custom build hook `_ -to add properties. They follow ``:bundled-wheel`` and come before those taken from the wheel's metadata. The -options then live in the custom hook's table, and the SBOM lists the script among the build tools, hashed, because the -script shapes the document a rebuild has to reproduce. +A :class:`~python_sbom.ContentProvider` has two methods, each with a default that adds nothing: + +- :meth:`~python_sbom.ContentProvider.bundled_wheel` returns the properties and supplier of one bundled wheel. The + properties follow ``:bundled-wheel`` and come before those taken from the wheel's metadata. +- :meth:`~python_sbom.ContentProvider.contributions` returns extra top-level components, dependency entries, root + properties and build tools, such as the C libraries a native wheel links. They join the document before the root's + dependencies and the serial number are derived from it, so the document stays consistent. + +Name the provider in the ``content-provider`` option, and the hook and ``python-sbom embed`` both load it. When it +lives in the project, the SBOM lists its module among the build tools, hashed, because it shapes the document a rebuild +has to reproduce: + +.. code-block:: python + + from python_sbom import ContentProvider, Contributions + + + class Content(ContentProvider): + def contributions(self, request): + zlib = {'type': 'library', 'bom-ref': 'pkg:generic/zlib@1.3.1', 'name': 'zlib', 'version': '1.3.1'} + return Contributions(components=[zlib], dependencies=[], root_properties=[], tools=[]) Worked example: virtualenv ~~~~~~~~~~~~~~~~~~~~~~~~~~ `virtualenv `_ ships the pip and setuptools wheels it installs into new environments, -under ``src/virtualenv/seed/wheels/embed``. A ``BUNDLE_SUPPORT`` table in that package's ``__init__.py`` decides which -Python versions receive each wheel. Its ``pyproject.toml`` names the supplier and the wheels: +under ``src/virtualenv/seed/wheels/embed``. A ``BUNDLE_SUPPORT`` table in that package's ``__init__.py``, after a +``BUNDLE_FOLDER`` assignment, decides which Python versions receive each wheel. The project already has a hatchling +`custom build hook `_, so it subclasses +:class:`~python_sbom.hatch.SbomBuildHook` there, and the options live in the custom hook's table: .. code-block:: toml @@ -32,41 +50,16 @@ Python versions receive each wheel. Its ``pyproject.toml`` names the supplier an supplier = { name = 'Python Packaging Authority', url = ['https://www.pypa.io'] } bundled-wheels = ['src/virtualenv/seed/wheels/embed/*.whl'] -``hatch_build.py`` turns the table into a ``virtualenv:seeded-for-python`` property per Python version: - -.. code-block:: python - - from __future__ import annotations - - import ast - from pathlib import Path - from typing import TYPE_CHECKING - - from python_sbom.hatch import SbomBuildHook - - if TYPE_CHECKING: - from python_sbom import Property - - - class VirtualenvSbomHook(SbomBuildHook): - def bundled_wheel_properties(self, wheel: Path) -> list[Property]: - embed = Path(self.root) / 'src' / 'virtualenv' / 'seed' / 'wheels' / 'embed' / '__init__.py' - tree = ast.parse(embed.read_text(encoding='utf-8')) - support: dict[str, dict[str, str]] = ast.literal_eval( - next(node.value for node in tree.body if isinstance(node, ast.Assign)) - ) - return [ - {'name': 'virtualenv:seeded-for-python', 'value': python} - for python, wheels in support.items() - if wheel.name in wheels.values() - ] +Its ``hatch_build.py`` overrides :meth:`~python_sbom.hatch.SbomBuildHook.content_provider` to hand over a provider that +reads the table next to the wheels by name, without importing the package. The test suite builds a project with +virtualenv's layout from this script: - - def get_build_hook() -> type[VirtualenvSbomHook]: - return VirtualenvSbomHook +.. literalinclude:: ../tests/_seeder/hatch_build.py + :language: python The ``get_build_hook`` function tells hatchling which class to load, since the module also sees the base class it -imports. The pip component of the wheel's SBOM then carries these properties, in this order: +imports. The SBOM lists the script among the build tools, hashed. The pip component then carries these properties, in +this order: .. list-table:: :header-rows: 1 @@ -83,8 +76,8 @@ imports. The pip component of the wheel's SBOM then carries these properties, in - ``>=3.10`` A project that still builds on a Python older than python-sbom supports, as virtualenv does on 3.9 and 3.10, puts an -environment marker on the requirement, ``python-sbom[hatch]>=0.1; python_version >= '3.11'``. Its script then falls back to -hatchling's plain hook there, so those builds produce a wheel without an SBOM: +environment marker on the requirement, ``python-sbom[hatch]>=0.1; python_version >= '3.11'``. Its script then falls +back to hatchling's plain hook there, so those builds produce a wheel without an SBOM: .. code-block:: python @@ -95,31 +88,65 @@ hatchling's plain hook there, so those builds produce a wheel without an SBOM: else: # python-sbom needs Python 3.11; older builds get no SBOM from hatchling.builders.hooks.plugin.interface import BuildHookInterface as BaseHook +Add another build backend +------------------------- + +A :class:`~python_sbom.Backend` says what a build system knows about the wheel: :meth:`~python_sbom.Backend.request` +returns the project directory, the core metadata, the wheel's file name, the settings and the build scripts. Two +backends ship: + +- :class:`~python_sbom.MetadataBackend` takes the metadata as text, for a build hook of another backend, such as a PDM + or scikit-build-core plugin, that has it before the wheel exists. +- :class:`~python_sbom.BuiltWheel` reads it from a finished wheel, as ``python-sbom embed`` does. + +The pipeline is the same for every backend: + +.. code-block:: python + + from python_sbom import ( + ContentProvider, + MetadataBackend, + build_sbom, + collect_facts, + default_revision, + read_source_tree, + settings_from_pyproject, + ) + + settings = settings_from_pyproject(root, {}) + request = MetadataBackend( + root=root, metadata=core_metadata, wheel_name=wheel_name, settings=settings, build_scripts=() + ).request() + facts = collect_facts(root, default_revision, None, None) + sbom = build_sbom(request, read_source_tree(request), facts, ContentProvider()) + +``sbom.file_name`` and ``sbom.content`` go into the wheel's ``.dist-info/sboms/``; a backend that can only rewrite a +finished wheel passes them to :func:`~python_sbom.embed_sbom`. In the hatch hook, +:meth:`~python_sbom.hatch.SbomBuildHook.backend` returns the backend, so a subclass can swap it. + Describe another artifact ------------------------- -virtualenv also publishes a zipapp. Its SBOM describes what the archive holds and records the same build toolchain as -the wheel's. The pieces the hook uses are public for that: +virtualenv also publishes a zipapp. Its SBOM describes what the archive holds and records the same build toolchain, +time and serial namespace as the wheel's. The pieces are public for that: -- :class:`~python_sbom.BuildFacts` collects the build time from ``SOURCE_DATE_EPOCH``, the source revision and the - interpreter and distributions of the running environment, once, at the edge. -- :func:`~python_sbom.component_from_metadata` turns a distribution's ``METADATA`` into a component, with licenses, - project URLs and metadata properties. -- :func:`~python_sbom.document` adds the CycloneDX header, with a serial number derived from the content. +- :func:`~python_sbom.collect_facts` reads the build time from ``SOURCE_DATE_EPOCH``, the source revision and the + interpreter and distributions of the build environment, once, at the edge. +- :func:`~python_sbom.tool_components` turns those facts into build tool components and their dependencies, and + :func:`~python_sbom.formulation` records the build. +- :func:`~python_sbom.file_component_from_bytes`, :func:`~python_sbom.bundled_wheel_component` and + :func:`~python_sbom.component_from_metadata` describe what the artifact holds. +- :func:`~python_sbom.project_identity` names the project the way its wheel SBOM does, and :func:`~python_sbom.document` + adds the CycloneDX header, with a serial number derived from the content. -This function, which the test suite runs and validates, describes a zipapp with no components yet: +This function, which the test suite runs and validates, describes a zipapp and the files it holds: .. literalinclude:: examples/zipapp.py :language: python :pyobject: describe_zipapp -:func:`~python_sbom.validate` checks the result the way ``python-sbom validate`` checks a wheel's SBOM. - -Add another build backend -------------------------- - -A build system other than hatchling plugs in by subclassing :class:`~python_sbom.Backend`. The subclass supplies the -wheel's metadata, its file name, the project directory and the settings; :meth:`~python_sbom.Backend.sbom` does the -rest. :class:`~python_sbom.BuiltWheel`, the backend behind ``python-sbom embed``, reads all of that from a finished -wheel; subclass it and override :meth:`~python_sbom.Backend.bundled_wheel_properties` to add properties to bundled wheels -without hatchling. +:func:`~python_sbom.validate` takes the JSON text and needs the ``validate`` extra: +``validate(dumps(document).decode(), None)`` checks the result the way ``python-sbom validate`` checks a wheel's SBOM, +and an allowed license set in place of ``None`` adds the license policy. The ``:sbom:file`` metadata property +marks the document as built from these pieces; without it, the validator treats the document as another tool's and +checks the schema alone. diff --git a/docs/py_api.rst b/docs/py_api.rst index 8ccbfc1..8ba4a85 100644 --- a/docs/py_api.rst +++ b/docs/py_api.rst @@ -2,50 +2,112 @@ Python API ========== The core is importable from ``python_sbom`` without hatchling; the hatchling build hook lives in ``python_sbom.hatch``. +:doc:`explanation` shows how the pieces fit together. -.. autodata:: python_sbom.__version__ +.. py:data:: python_sbom.__version__ + :type: str + + The installed version of python-sbom. Hatchling build hook -------------------- .. autoclass:: python_sbom.hatch.SbomBuildHook - :members: bundled_wheel_properties + :members: content_provider, backend -Backends +Pipeline -------- +.. autofunction:: python_sbom.build_sbom + +.. autoclass:: python_sbom.Sbom + :members: content + .. autoclass:: python_sbom.Backend - :members: + :members: request + +.. autoclass:: python_sbom.MetadataBackend .. autoclass:: python_sbom.BuiltWheel - :members: embed .. autoclass:: python_sbom.ContentProvider - :members: + :members: bundled_wheel, contributions + +.. autofunction:: python_sbom.load_content_provider + +.. autofunction:: python_sbom.embed_sbom + +Collecting inputs +----------------- + +.. autofunction:: python_sbom.settings_from_pyproject + +.. autofunction:: python_sbom.settings_from_table + +.. py:data:: python_sbom.Toml + + A value of a TOML table, as :mod:`tomllib` or hatchling hand it over: a string, number, boolean, date or time, or + an array or table of them. + +.. autofunction:: python_sbom.collect_facts + +.. autoclass:: python_sbom.RevisionReader + :special-members: __call__ + +.. autofunction:: python_sbom.default_revision + +.. autofunction:: python_sbom.git_revision + +.. autofunction:: python_sbom.read_source_tree + +.. autofunction:: python_sbom.read_bundled_wheel Inputs ------ -.. autoclass:: python_sbom.Settings - :members: from_table - :exclude-members: supplier, repository, default_branch, bundled_wheels +.. autoclass:: python_sbom.SbomRequest -.. autodata:: python_sbom.Toml +.. autoclass:: python_sbom.Settings .. autoclass:: python_sbom.BuildFacts - :members: collect, timestamp - :exclude-members: epoch, commit, tools, tool_dependencies -.. autoclass:: python_sbom.RevisionReader - :members: __call__ +.. autoclass:: python_sbom.Interpreter -.. autofunction:: python_sbom.git_revision +.. autoclass:: python_sbom.InstalledDistribution + +.. autoclass:: python_sbom.Revision + +.. autoclass:: python_sbom.RecordRow + +.. autoclass:: python_sbom.SourceTree + +.. autoclass:: python_sbom.BundledWheel + +.. autoclass:: python_sbom.BuildScript + +.. autoclass:: python_sbom.BundledWheelInfo + +.. autoclass:: python_sbom.Contributions Building blocks --------------- .. autofunction:: python_sbom.component_from_metadata +.. autofunction:: python_sbom.bundled_wheel_component + +.. autofunction:: python_sbom.file_component_from_bytes + +.. autofunction:: python_sbom.script_tool + +.. autofunction:: python_sbom.tool_components + +.. autofunction:: python_sbom.formulation + +.. autofunction:: python_sbom.project_identity + +.. autoclass:: python_sbom.ProjectIdentity + .. autofunction:: python_sbom.document .. autofunction:: python_sbom.dumps @@ -70,36 +132,53 @@ Errors .. autoexception:: python_sbom.SourceRevisionError +.. autoexception:: python_sbom.MissingExtraError + Types ----- -These are :class:`~typing.TypedDict` types of the JSON documents, for type checking code that builds or reads them. +These are :class:`~typing.TypedDict` types of the JSON documents, for type checking code that builds or reads them. Each +follows the CycloneDX 1.6 JSON object of the same name; the `CycloneDX 1.6 reference +`_ describes every key. + +.. py:class:: python_sbom.Document -.. autoclass:: python_sbom.Document - :no-members: + A complete CycloneDX 1.6 document: the ``$schema``, ``bomFormat``, ``specVersion``, ``serialNumber`` and ``version`` + header, plus the keys of :class:`~python_sbom.Body`. See the `BOM `_ + reference. .. autoclass:: python_sbom.Body - :no-members: + :no-members: -.. autoclass:: python_sbom.Component - :no-members: +.. py:class:: python_sbom.Component + + A distribution, application, platform or file, with ``type``, ``bom-ref`` and ``name`` and the optional + ``version``, ``purl``, ``supplier``, ``authors``, ``description``, ``licenses``, ``copyright``, ``hashes``, + ``externalReferences``, ``properties``, ``evidence`` and nested ``components``. See the `component + `_ reference. .. autoclass:: python_sbom.Dependency - :no-members: + :no-members: + +.. py:class:: python_sbom.Formula + + How an artifact was built: a ``bom-ref`` and one or more ``workflows``, each with the tools it used as + ``resourceReferences``, its ``outputs`` and, optionally, its ``inputs``. See the `formulation + `_ reference. + +.. autoclass:: python_sbom.License + :no-members: .. autoclass:: python_sbom.OrganizationalEntity - :no-members: + :no-members: .. autoclass:: python_sbom.Property - :no-members: + :no-members: .. autoclass:: python_sbom.Spdx - :no-members: + :no-members: Command line ------------ .. autofunction:: python_sbom.main - -.. autoclass:: python_sbom.Options - :members: diff --git a/docs/usage.rst b/docs/usage.rst index 9f7cade..6756d1c 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -5,17 +5,20 @@ With hatchling -------------- Add the hook to ``build-system.requires`` and enable it with an empty table, as the quick start on the :doc:`index` -shows. The ``hatch`` extra brings hatchling 1.28, the first release that puts an SBOM into a wheel; the command line -and the library need no hatchling. The hook acts on wheel builds only; an -sdist carries no SBOM. +shows. The ``hatch`` extra brings hatchling; the command line and the library need none. The hook acts on wheel builds +only: an sdist and an editable wheel carry no SBOM. -The SBOM links to the wheel's PyPI attestation, so it needs the wheel's tag. List the hook after any hook that changes -the tag. +The hook writes the SBOM once hatchling has written the wheel, so the document describes the final file name and +metadata, whatever dependencies or tag other hooks changed. Hatchling runs the hooks in the order of their tables; see +its `build hook documentation `_. The order does not matter +for this hook. Options ~~~~~~~ -Each option lives in the hook's table and is optional. +The options live in a ``[tool.python-sbom]`` table. Each is optional; an unknown key fails the build, so a typo cannot +pass silently. A key in the hook's own table, such as ``[tool.hatch.build.hooks.python-sbom]``, wins over the same key +in ``[tool.python-sbom]``. ``supplier`` The organization that supplies the project: a table with a ``name`` string and an optional ``url`` string array. It @@ -24,7 +27,7 @@ Each option lives in the hook's table and is optional. .. code-block:: toml - [tool.hatch.build.hooks.python-sbom] + [tool.python-sbom] supplier = { name = 'Python Packaging Authority', url = ['https://www.pypa.io'] } ``repository`` @@ -33,7 +36,7 @@ Each option lives in the hook's table and is optional. .. code-block:: toml - [tool.hatch.build.hooks.python-sbom] + [tool.python-sbom] repository = 'https://gitlab.com/example/project' ``default-branch`` @@ -41,7 +44,7 @@ Each option lives in the hook's table and is optional. .. code-block:: toml - [tool.hatch.build.hooks.python-sbom] + [tool.python-sbom] default-branch = 'trunk' ``bundled-wheels`` @@ -50,19 +53,39 @@ Each option lives in the hook's table and is optional. .. code-block:: toml - [tool.hatch.build.hooks.python-sbom] + [tool.python-sbom] bundled-wheels = ['src/project/wheels/*.whl'] -A bundled wheel whose project URLs point to the same forge account as ``repository`` gets the same ``supplier``. Its -vendored packages come from ``_vendor/vendor.txt``, pip's manifest, whose lines must each pin one exact version, and -from any ``.dist-info/METADATA`` under ``_vendor/``. +``content-provider`` + A :class:`~python_sbom.ContentProvider`, as ``module:attribute`` importable from the project root, that adds what + the project knows beyond its metadata; :doc:`extend` shows one. + + .. code-block:: toml + + [tool.python-sbom] + content-provider = 'sbom_content:Content' + +A bundled wheel's files come from its ``RECORD``, and the build fails when a member is missing from it, a row names a +member the wheel lacks, or a digest differs from the member's content. Its vendored packages come from +``_vendor/vendor.txt``, pip's manifest, whose lines must each pin one exact version, and from any +``.dist-info/METADATA`` under ``_vendor/``. A bundled wheel gets a supplier only from the project's content provider, +never from its own metadata. + +The source revision +~~~~~~~~~~~~~~~~~~~ + +The SBOM names the commit the wheel was built from. It takes the first of: the ``PYTHON_SBOM_COMMIT`` environment +variable; git, for a checkout or any directory inside one; and the ``node`` line of an expanded ``.git_archival.txt``, +for a ``git archive``. A checkout with uncommitted changes to tracked files gets a ``:vcs-dirty`` property, and +the link to the revision says so. A wheel built from an sdist, as ``python -m build`` and ``uv build`` do by default, +has no checkout, so set ``PYTHON_SBOM_COMMIT`` or ship ``.git_archival.txt`` to keep the commit. With another build backend -------------------------- -A backend other than hatchling, such as `meson-python `_, cannot run the hook. +A backend other than hatchling, such as `meson-python `_, cannot run the hook. ``python-sbom embed`` writes the same document into a wheel that backend built, adds the ``RECORD`` row, and keeps -every other entry of the archive as it was. It reads the options above from a ``[tool.python-sbom]`` table: +every other entry of the archive as it was. It reads the options from ``[tool.python-sbom]``: .. code-block:: toml @@ -77,15 +100,23 @@ Build, then embed, from the project root: $ python-sbom embed dist/*.whl wrote turbohtml-1.0.dist-info/sboms/turbohtml.cdx.json into turbohtml-1.0-cp314-cp314-linux_x86_64.whl -The build tool record describes the environment ``embed`` runs in, not the isolated environment the backend built in. -To record the build toolchain, build without isolation in the same locked environment: +``--commit`` names the source revision when the wheel came from an sdist, and ``--content-provider`` overrides the +``content-provider`` option. + +The build tool record lists the environment named by ``--build-env``, by default the one ``embed`` runs in. To record +the real toolchain, point ``--build-env`` at the build environment's ``site-packages``, or build without isolation in +the same locked environment. On Linux, meson-python also needs ``patchelf`` there: .. code-block:: console - $ python -m pip install python-sbom meson-python meson ninja + $ python -m pip install python-sbom meson-python meson ninja patchelf $ python -m build --wheel --no-isolation $ python-sbom embed dist/*.whl +``auditwheel repair`` renames a Linux wheel to its ``manylinux`` tag, so the attestation link in an SBOM written +before the repair names the old file. Run ``embed`` on the repaired wheel, after ``auditwheel``, to describe the file +you publish; auditwheel's own SBOM in the same directory stays untouched. + Render it as SPDX ----------------- @@ -105,9 +136,10 @@ number, so both documents name the same build. Check it in CI -------------- -``python-sbom validate`` needs the ``validate`` extra. It checks each wheel's SBOM against the CycloneDX 1.6 schema, -requires the ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, checks that every reference -resolves, and validates the SPDX rendering: +``python-sbom validate`` needs the ``validate`` extra, and says so when it is missing. It checks every CycloneDX SBOM +in each wheel's ``.dist-info/sboms`` against the CycloneDX 1.6 schema. For the SBOM python-sbom wrote, it also requires +the ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, checks that every reference resolves, +and validates the SPDX rendering; an SBOM another tool wrote, such as auditwheel's, gets the schema check alone: .. code-block:: console From 26b526e499f1cf204551ba39e2e6d9c3bdb49838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 22:08:40 -0800 Subject: [PATCH 017/115] =?UTF-8?q?=F0=9F=94=A7=20chore:=20align=20reposit?= =?UTF-8?q?ory=20files=20with=20conventions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trim .gitignore and CODEOWNERS, run simplint between pyproject-fmt and ruff, point Homepage at the documentation and add the Python modules classifier. --- .github/CODEOWNERS | 2 +- .gitignore | 14 +++----------- .pre-commit-config.yaml | 8 ++++---- pyproject.toml | 4 ++-- 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0f9e29b..9b32546 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @gaborbernat +* @gaborbernat diff --git a/.gitignore b/.gitignore index 69be040..1e8968b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,6 @@ -/magic -.idea +*.pyc *.egg-info +dist/ .tox/ -/.venv -.coverage* -coverage.xml -.*_cache -__pycache__ -**.pyc -/build -dist -src/python_sbom/version.py +/src/python_sbom/version.py /uv.lock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cecb70b..467121a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,16 +25,16 @@ repos: rev: "50e0f3aee9dc5747a5f173f79a7417403c18dda6" # frozen: v2.30.3 hooks: - id: pyproject-fmt + - repo: https://github.com/gaborbernat/simplint-pre-commit + rev: "c8719dca99c13381570bc69f8955c9448fc271b7" # frozen: v0.7.1 + hooks: + - id: simplint - repo: https://github.com/astral-sh/ruff-pre-commit rev: "25bcf95026b96e2dd3c0aec3090c3d6ef6ca2e6c" # frozen: v0.17.3 hooks: - id: ruff-format - id: ruff-check args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] - - repo: https://github.com/gaborbernat/simplint-pre-commit - rev: c8719dca99c13381570bc69f8955c9448fc271b7 # frozen: v0.7.1 - hooks: - - id: simplint - repo: https://github.com/google/yamlfmt rev: "21ca5323a9c87ee37a434e0ca908efc0a89daa07" # frozen: v0.21.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index bd6ab9e..e4e0e64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Libraries :: Python Modules", "Typing :: Typed", ] dynamic = [ @@ -48,7 +49,6 @@ dependencies = [ "packaging>=26.3", ] optional-dependencies.hatch = [ - # 1.28 is the first release that puts an SBOM into a wheel "hatchling>=1.28", ] optional-dependencies.validate = [ @@ -57,7 +57,7 @@ optional-dependencies.validate = [ ] urls.Changelog = "https://python-sbom.readthedocs.io/en/latest/changelog.html" urls.Documentation = "https://python-sbom.readthedocs.io" -urls.Homepage = "https://github.com/tox-dev/python-sbom" +urls.Homepage = "https://python-sbom.readthedocs.io" urls.Source = "https://github.com/tox-dev/python-sbom" urls.Tracker = "https://github.com/tox-dev/python-sbom/issues" scripts.python-sbom = "python_sbom._cli._main:main" From c45b6ef3cf27e9989d38ea26eb90099931e0c3ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 22:10:40 -0800 Subject: [PATCH 018/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20break?= =?UTF-8?q?=20the=20CLI=20options=20import=20cycle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/python_sbom/_cli/_commands.py | 2 +- src/python_sbom/_cli/_main.py | 3 ++- src/python_sbom/_cli/_options.py | 26 ++++++++++++++++++++++++++ src/python_sbom/_cli/_parser.py | 20 +------------------- 4 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 src/python_sbom/_cli/_options.py diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py index 9115121..097d1df 100644 --- a/src/python_sbom/_cli/_commands.py +++ b/src/python_sbom/_cli/_commands.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from python_sbom._collect._facts import RevisionReader - from ._parser import Options + from ._options import Options def embed(options: Options) -> bool: diff --git a/src/python_sbom/_cli/_main.py b/src/python_sbom/_cli/_main.py index f489ae5..9efbc1b 100644 --- a/src/python_sbom/_cli/_main.py +++ b/src/python_sbom/_cli/_main.py @@ -5,7 +5,8 @@ from python_sbom._errors import SbomError -from ._parser import Options, build_parser +from ._options import Options +from ._parser import build_parser if TYPE_CHECKING: from collections.abc import Sequence diff --git a/src/python_sbom/_cli/_options.py b/src/python_sbom/_cli/_options.py new file mode 100644 index 0000000..af70cbe --- /dev/null +++ b/src/python_sbom/_cli/_options.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from argparse import Namespace +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + +class Options(Namespace): + handler: Callable[[Options], bool] + wheels: list[Path] + root: Path | None + commit: str | None + build_env: list[str] | None + content_provider: str | None + source: Path + target: Path + paths: list[Path] + allowed_licenses: list[str] | None + + +__all__ = [ + "Options", +] diff --git a/src/python_sbom/_cli/_parser.py b/src/python_sbom/_cli/_parser.py index f5923ac..07af09d 100644 --- a/src/python_sbom/_cli/_parser.py +++ b/src/python_sbom/_cli/_parser.py @@ -1,27 +1,10 @@ from __future__ import annotations -from argparse import ArgumentParser, Namespace +from argparse import ArgumentParser from pathlib import Path -from typing import TYPE_CHECKING from ._commands import embed, spdx, validate -if TYPE_CHECKING: - from collections.abc import Callable - - -class Options(Namespace): - handler: Callable[[Options], bool] - wheels: list[Path] - root: Path | None - commit: str | None - build_env: list[str] | None - content_provider: str | None - source: Path - target: Path - paths: list[Path] - allowed_licenses: list[str] | None - def build_parser() -> ArgumentParser: parser = ArgumentParser(prog="python-sbom", description="Write, convert and check the CycloneDX SBOMs of wheels.") @@ -72,6 +55,5 @@ def build_parser() -> ArgumentParser: __all__ = [ - "Options", "build_parser", ] From af0e85fa770da3ca0e636945805f331ffc992599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 22:13:40 -0800 Subject: [PATCH 019/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20import=20TypeIs?= =?UTF-8?q?=20only=20when=20type=20checking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/python_sbom/_collect/_settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/python_sbom/_collect/_settings.py b/src/python_sbom/_collect/_settings.py index 340abf9..49cb765 100644 --- a/src/python_sbom/_collect/_settings.py +++ b/src/python_sbom/_collect/_settings.py @@ -1,7 +1,7 @@ from __future__ import annotations import tomllib -from typing import TYPE_CHECKING, TypeAlias, TypeIs +from typing import TYPE_CHECKING, TypeAlias from python_sbom._errors import SettingsError from python_sbom._model._inputs import Settings @@ -10,6 +10,7 @@ from collections.abc import Mapping from datetime import date, datetime, time from pathlib import Path + from typing import TypeIs # new in 3.13; annotations here are never evaluated from python_sbom._model._types import OrganizationalEntity From 52381528445b389ce8b757bbc398cd5838c81899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 23:09:23 -0800 Subject: [PATCH 020/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20shrink?= =?UTF-8?q?=20the=20API=20to=20the=20provider=20seam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The public API is now the content provider surface the hatchling hook calls, plus the types its signatures reach; embedding, archives, SPDX and validation run through the command line. A provider receives a context with the project root, settings, identity and a reader for the artifact, so one provider serves the wheel and an archive. The new archive command describes a zipapp with the same bundled-wheel identities the wheel SBOM uses. Address the second review: build facts survive egg-info and broken requirements, git is used only for a tracked project without running a parent repository's hooks, an sdist carries its commit, bundled wheels must match every RECORD row, decoy dist-info and duplicate members are refused, and validation reports hostile documents instead of raising. --- hatch_build.py | 4 + pyproject.toml | 18 +- src/python_sbom/__init__.py | 101 ++--- src/python_sbom/__main__.py | 2 + src/python_sbom/_cli/_commands.py | 189 ++++++++-- src/python_sbom/_cli/_main.py | 7 - src/python_sbom/_cli/_options.py | 5 + src/python_sbom/_cli/_parser.py | 50 +-- src/python_sbom/_collect/_content.py | 55 +++ src/python_sbom/_collect/_facts.py | 142 +++---- src/python_sbom/_collect/_settings.py | 19 - src/python_sbom/_collect/_source.py | 129 ++++--- src/python_sbom/_errors.py | 16 +- src/python_sbom/_formats/_json.py | 7 - src/python_sbom/_formats/_spdx.py | 219 ++++++----- src/python_sbom/_generate/_archive.py | 213 +++++++++++ src/python_sbom/_generate/_backend.py | 115 ------ src/python_sbom/_generate/_content.py | 84 ----- src/python_sbom/_generate/_embed.py | 73 ++++ src/python_sbom/_generate/_pipeline.py | 33 +- src/python_sbom/_model/_bom.py | 58 +-- src/python_sbom/_model/_components.py | 217 +++-------- src/python_sbom/_model/_content.py | 87 +++++ src/python_sbom/_model/_document.py | 110 ++---- src/python_sbom/_model/_identity.py | 49 +++ src/python_sbom/_model/_inputs.py | 127 ++----- src/python_sbom/_model/_licenses.py | 62 ++++ src/python_sbom/_model/_markers.py | 7 +- src/python_sbom/_model/_names.py | 93 +++++ src/python_sbom/_model/_tools.py | 20 +- src/python_sbom/_model/_types.py | 118 ++++-- src/python_sbom/_validate/_artifact.py | 45 +++ src/python_sbom/_validate/_document.py | 140 ++++--- src/python_sbom/_validate/_licenses.py | 21 +- src/python_sbom/_validate/_schemas.py | 20 +- src/python_sbom/_validate/_wheel.py | 39 -- src/python_sbom/_wheel/_archive.py | 125 ++++--- src/python_sbom/_wheel/_record.py | 32 +- src/python_sbom/hatch/_hook.py | 105 +++--- tests/_cli/{test_cli.py => test_commands.py} | 113 ++++-- tests/_collect/test_collect_facts.py | 221 ----------- tests/_collect/test_collect_source.py | 185 --------- .../test_content.py} | 57 ++- tests/_collect/test_facts.py | 351 ++++++++++++++++++ ...t_collect_settings.py => test_settings.py} | 6 +- tests/_collect/test_source.py | 262 +++++++++++++ .../{test_formats_spdx.py => test_spdx.py} | 128 ++++++- tests/_generate/test_archive.py | 199 ++++++++++ tests/_generate/test_embed.py | 49 +++ tests/_generate/test_generate_backend.py | 43 --- tests/_generate/test_pipeline.py | 30 ++ .../_model/{test_model_bom.py => test_bom.py} | 101 +++-- ...model_components.py => test_components.py} | 49 ++- tests/_model/test_content.py | 51 +++ tests/_model/test_document.py | 76 ++++ ...est_model_document.py => test_identity.py} | 29 +- tests/_model/test_licenses.py | 28 ++ tests/_model/test_markers.py | 27 ++ .../{test_model_tools.py => test_tools.py} | 13 +- tests/_seeder/hatch_build.py | 38 +- tests/_seeder/pyproject.toml | 7 +- tests/_validate/test_artifact.py | 95 +++++ tests/_validate/test_document.py | 230 ++++++++++++ tests/_validate/test_licenses.py | 102 +++++ tests/_validate/test_schemas.py | 28 ++ tests/_validate/test_validate_document.py | 184 --------- tests/_validate/test_validate_extra.py | 15 - tests/_validate/test_validate_wheel.py | 40 -- tests/_wheel/test_archive.py | 234 ++++++++++++ tests/_wheel/test_record.py | 80 ++++ tests/_wheel/test_wheel_embed.py | 137 ------- tests/conftest.py | 27 +- tests/hatch/conftest.py | 4 +- .../{test_hatch_hook.py => test_hook.py} | 75 +++- tests/test_examples.py | 24 -- tests/test_public_api.py | 91 +++++ tox.toml | 14 +- 77 files changed, 4062 insertions(+), 2337 deletions(-) create mode 100644 src/python_sbom/_collect/_content.py create mode 100644 src/python_sbom/_generate/_archive.py delete mode 100644 src/python_sbom/_generate/_backend.py delete mode 100644 src/python_sbom/_generate/_content.py create mode 100644 src/python_sbom/_generate/_embed.py create mode 100644 src/python_sbom/_model/_content.py create mode 100644 src/python_sbom/_model/_identity.py create mode 100644 src/python_sbom/_model/_licenses.py create mode 100644 src/python_sbom/_model/_names.py create mode 100644 src/python_sbom/_validate/_artifact.py delete mode 100644 src/python_sbom/_validate/_wheel.py rename tests/_cli/{test_cli.py => test_commands.py} (55%) delete mode 100644 tests/_collect/test_collect_facts.py delete mode 100644 tests/_collect/test_collect_source.py rename tests/{_generate/test_generate_content.py => _collect/test_content.py} (53%) create mode 100644 tests/_collect/test_facts.py rename tests/_collect/{test_collect_settings.py => test_settings.py} (93%) create mode 100644 tests/_collect/test_source.py rename tests/_formats/{test_formats_spdx.py => test_spdx.py} (58%) create mode 100644 tests/_generate/test_archive.py create mode 100644 tests/_generate/test_embed.py delete mode 100644 tests/_generate/test_generate_backend.py create mode 100644 tests/_generate/test_pipeline.py rename tests/_model/{test_model_bom.py => test_bom.py} (85%) rename tests/_model/{test_model_components.py => test_components.py} (85%) create mode 100644 tests/_model/test_content.py create mode 100644 tests/_model/test_document.py rename tests/_model/{test_model_document.py => test_identity.py} (57%) create mode 100644 tests/_model/test_licenses.py create mode 100644 tests/_model/test_markers.py rename tests/_model/{test_model_tools.py => test_tools.py} (76%) create mode 100644 tests/_validate/test_artifact.py create mode 100644 tests/_validate/test_document.py create mode 100644 tests/_validate/test_licenses.py create mode 100644 tests/_validate/test_schemas.py delete mode 100644 tests/_validate/test_validate_document.py delete mode 100644 tests/_validate/test_validate_extra.py delete mode 100644 tests/_validate/test_validate_wheel.py create mode 100644 tests/_wheel/test_archive.py create mode 100644 tests/_wheel/test_record.py delete mode 100644 tests/_wheel/test_wheel_embed.py rename tests/hatch/{test_hatch_hook.py => test_hook.py} (66%) delete mode 100644 tests/test_examples.py create mode 100644 tests/test_public_api.py diff --git a/hatch_build.py b/hatch_build.py index 85f50c4..354f4a2 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -1,3 +1,5 @@ +"""Write python-sbom's own SBOM into its wheel with the hook the package ships.""" + from __future__ import annotations import sys @@ -22,4 +24,6 @@ def finalize( hook = SbomBuildHook( self.root, self.config, self.build_config, self.metadata, self.directory, self.target_name, self.app ) + # as hatchling names a custom hook, so the SBOM lists this script among the build tools + hook.PLUGIN_NAME = self.PLUGIN_NAME hook.finalize(version, build_data, artifact_path) diff --git a/pyproject.toml b/pyproject.toml index e4e0e64..c9ff9c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ optional-dependencies.validate = [ ] urls.Changelog = "https://python-sbom.readthedocs.io/en/latest/changelog.html" urls.Documentation = "https://python-sbom.readthedocs.io" -urls.Homepage = "https://python-sbom.readthedocs.io" +urls.Homepage = "https://github.com/tox-dev/python-sbom" urls.Source = "https://github.com/tox-dev/python-sbom" urls.Tracker = "https://github.com/tox-dev/python-sbom/issues" scripts.python-sbom = "python_sbom._cli._main:main" @@ -137,17 +137,24 @@ lint.select = [ ] lint.ignore = [ "CPY", # The project does not add copyright headers to individual files - "D1", # names and types document the code; simplint asks for public API docs "DOC", # Public API documentation is generated from signatures "incorrect-blank-line-before-class", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible "missing-trailing-comma", # Conflict with formatter "multi-line-summary-first-line", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible "single-line-implicit-string-concatenation", # Conflict with formatter + "undocumented-magic-method", # undocumented-public-api (simplint SMP903) covers the public API instead + "undocumented-public-class", # undocumented-public-api (simplint SMP903) covers the public API instead + "undocumented-public-function", # undocumented-public-api (simplint SMP903) covers the public API instead + "undocumented-public-init", # undocumented-public-api (simplint SMP903) covers the public API instead + "undocumented-public-method", # undocumented-public-api (simplint SMP903) covers the public API instead + "undocumented-public-nested-class", # undocumented-public-api (simplint SMP903) covers the public API instead ] lint.per-file-ignores."tests/**/*.py" = [ "assert", # Tests use assertions + "D", # Test names document behavior "FBT", # Boolean parameters keep parameterized cases compact "implicit-namespace-package", # The tests directory is not a package + "import-private-name", # Unit tests cover the private modules behind the small public API "magic-value-comparison", # Literal expected values keep assertions local "subprocess-without-shell-equals-true", # Entry-point arguments are fixed by the tests "suspicious-subprocess-import", # Entry-point tests invoke a local Python executable @@ -174,8 +181,8 @@ environment.python-version = "3.15" [tool.pytest] ini_options.testpaths = [ "tests" ] -# the tests run the examples under docs/examples that the documentation includes -ini_options.pythonpath = [ "." ] +# test modules mirror the source modules, so their names repeat across directories +ini_options.addopts = [ "--import-mode=importlib" ] ini_options.filterwarnings = [ "error", ] @@ -219,6 +226,9 @@ type = [ { directory = "packaging", name = "Packaging", showcontent = true }, ] +[tool.python-sbom] +supplier = { name = "tox-dev", url = [ "https://github.com/tox-dev" ] } + [tool.simplint] docstring-style = "rst" # the fixture mirrors virtualenv's seed/wheels/embed/__init__.py, which holds the bundle table diff --git a/src/python_sbom/__init__.py b/src/python_sbom/__init__.py index 783bd91..25363ff 100644 --- a/src/python_sbom/__init__.py +++ b/src/python_sbom/__init__.py @@ -2,96 +2,45 @@ from __future__ import annotations -from ._cli._main import main -from ._collect._facts import RevisionReader, collect_facts, default_revision, git_revision -from ._collect._settings import Toml, settings_from_pyproject, settings_from_table -from ._collect._source import read_bundled_wheel, read_source_tree -from ._errors import MissingExtraError, SbomError, SettingsError, SourceRevisionError, WheelError -from ._formats._json import dumps -from ._formats._spdx import Spdx, to_spdx -from ._generate._backend import Backend, BuiltWheel, MetadataBackend -from ._generate._content import ContentProvider, load_content_provider -from ._generate._pipeline import Sbom, build_sbom -from ._model._components import bundled_wheel_component, component_from_metadata, file_component_from_bytes, script_tool -from ._model._document import ProjectIdentity, document, formulation, project_identity -from ._model._inputs import ( - BuildFacts, - BuildScript, - BundledWheel, - BundledWheelInfo, - Contributions, - InstalledDistribution, - Interpreter, - RecordRow, - Revision, - SbomRequest, - Settings, - SourceTree, +from ._errors import SbomError +from ._model._content import ContentProvider, ProviderContext +from ._model._identity import ProjectIdentity +from ._model._inputs import BundledWheelInfo, Contributions, Settings +from ._model._types import ( + Component, + Contact, + Dependency, + Evidence, + ExternalReference, + Hash, + Identity, + IdentityMethod, + License, + LicenseInfo, + OrganizationalEntity, + Property, ) -from ._model._tools import tool_components -from ._model._types import Body, Component, Dependency, Document, Formula, License, OrganizationalEntity, Property -from ._validate._document import validate -from ._validate._wheel import validate_wheel -from ._wheel._archive import embed_sbom from .version import __version__ __all__ = [ - "Backend", - "Body", - "BuildFacts", - "BuildScript", - "BuiltWheel", - "BundledWheel", "BundledWheelInfo", "Component", + "Contact", "ContentProvider", "Contributions", "Dependency", - "Document", - "Formula", - "InstalledDistribution", - "Interpreter", + "Evidence", + "ExternalReference", + "Hash", + "Identity", + "IdentityMethod", "License", - "MetadataBackend", - "MissingExtraError", + "LicenseInfo", "OrganizationalEntity", "ProjectIdentity", "Property", - "RecordRow", - "Revision", - "RevisionReader", - "Sbom", + "ProviderContext", "SbomError", - "SbomRequest", "Settings", - "SettingsError", - "SourceRevisionError", - "SourceTree", - "Spdx", - "Toml", - "WheelError", "__version__", - "build_sbom", - "bundled_wheel_component", - "collect_facts", - "component_from_metadata", - "default_revision", - "document", - "dumps", - "embed_sbom", - "file_component_from_bytes", - "formulation", - "git_revision", - "load_content_provider", - "main", - "project_identity", - "read_bundled_wheel", - "read_source_tree", - "script_tool", - "settings_from_pyproject", - "settings_from_table", - "to_spdx", - "tool_components", - "validate", - "validate_wheel", ] diff --git a/src/python_sbom/__main__.py b/src/python_sbom/__main__.py index 2c7a692..a528915 100644 --- a/src/python_sbom/__main__.py +++ b/src/python_sbom/__main__.py @@ -1,3 +1,5 @@ +"""Main entry point.""" + from __future__ import annotations import sys diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py index 097d1df..16c15a1 100644 --- a/src/python_sbom/_cli/_commands.py +++ b/src/python_sbom/_cli/_commands.py @@ -2,74 +2,187 @@ import json import sys -from pathlib import Path +import zipfile +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING -from python_sbom._collect._facts import collect_facts, default_revision -from python_sbom._collect._settings import settings_from_pyproject -from python_sbom._collect._source import read_source_tree +from python_sbom._collect._content import load_content_provider +from python_sbom._collect._facts import collect_facts +from python_sbom._errors import SbomError from python_sbom._formats._json import dumps from python_sbom._formats._spdx import to_spdx -from python_sbom._generate._backend import BuiltWheel -from python_sbom._generate._content import ContentProvider, load_content_provider -from python_sbom._generate._pipeline import build_sbom -from python_sbom._model._inputs import Revision -from python_sbom._validate._wheel import validate_wheel -from python_sbom._wheel._archive import embed_sbom -from python_sbom.version import __version__ +from python_sbom._generate._archive import describe_archive +from python_sbom._generate._embed import embed_sbom, revision_reader +from python_sbom._validate._artifact import validate as validate_artifact +from python_sbom._validate._document import is_document if TYPE_CHECKING: - from python_sbom._collect._facts import RevisionReader + from argparse import ArgumentParser, _SubParsersAction from ._options import Options -def embed(options: Options) -> bool: +def add_embed(commands: _SubParsersAction[ArgumentParser]) -> None: + parser = commands.add_parser( + "embed", + help="write the SBOM into wheels another build backend built", + description="Write the SBOM the hatchling hook writes into wheels another build backend built, such as" + " meson-python. Options come from the [tool.python-sbom] table of the project's pyproject.toml.", + ) + parser.set_defaults(handler=_embed) + parser.add_argument("wheels", type=Path, nargs="+", metavar="WHEEL", help="wheel to rewrite in place") + _add_build_options(parser) + + +def _embed(options: Options) -> bool: root = Path.cwd() if options.root is None else options.root - settings = settings_from_pyproject(root, {}) - content, script = ( - (ContentProvider(), None) - if (spec := options.content_provider or settings.content_provider) is None - else load_content_provider(spec, root) + content = None if options.content_provider is None else load_content_provider(options.content_provider, root) + for wheel in options.wheels: + written = embed_sbom(wheel, root, content=content, commit=options.commit, build_env=options.build_env) + sys.stdout.write(f"wrote {written} into {wheel.name}\n") + return True + + +def _add_build_options(parser: ArgumentParser) -> None: + parser.add_argument( + "--root", type=Path, metavar="DIR", help="project directory of the artifact (default: current directory)" ) - revision: RevisionReader = ( - default_revision if (commit := options.commit) is None else lambda _root: Revision(commit=commit, dirty=False) + parser.add_argument( + "--commit", metavar="SHA", help="source revision to record, for an artifact built from an sdist" ) - facts = collect_facts(root, revision, options.build_env, None) - for wheel in options.wheels: - request = BuiltWheel( - wheel=wheel, root=root, settings=settings, build_scripts=() if script is None else (script,) - ).request() - sbom = build_sbom(request, read_source_tree(request), facts, content) - sys.stdout.write(f"wrote {embed_sbom(wheel, sbom.file_name, sbom.content, facts.epoch)} into {wheel.name}\n") + parser.add_argument( + "--build-env", + action="append", + metavar="DIR", + help="site-packages directory of the environment that built the artifact; repeat it per directory" + " (default: this interpreter's)", + ) + parser.add_argument( + "--content-provider", + metavar="MODULE:ATTRIBUTE", + help="content provider, overriding the content-provider option", + ) + + +def add_archive(commands: _SubParsersAction[ArgumentParser]) -> None: + parser = commands.add_parser( + "archive", + help="write the SBOM of a zip archive that is not a wheel, such as a zipapp", + description="Describe a zip archive that holds the project's .dist-info at its top level, such as a zipapp:" + " the archive's own files, the project's distribution, and every wheel it bundles, as a file or unpacked," + " described as the wheel's SBOM describes them. Options come from [tool.python-sbom].", + ) + parser.set_defaults(handler=_archive) + parser.add_argument("archive", type=Path, metavar="ARCHIVE", help="archive to describe") + _add_build_options(parser) + parser.add_argument( + "--build-script", + dest="build_scripts", + action="append", + metavar="PATH", + help="script, relative to the project, that built the archive; repeat it per script", + ) + parser.add_argument( + "--output", type=Path, metavar="FILE", help="SBOM file to write (default: .cdx.json next to it)" + ) + parser.add_argument( + "--embed", action="store_true", help="also add the SBOM to the archive, at its top level as .cdx.json" + ) + + +def _archive(options: Options) -> bool: + root = Path.cwd() if options.root is None else options.root + facts = collect_facts(root, revision_reader(options.commit), path=options.build_env) + document = describe_archive( + root, + options.archive, + facts=facts, + content=None if options.content_provider is None else load_content_provider(options.content_provider, root), + build_scripts=tuple(PurePosixPath(script) for script in options.build_scripts or []), + ) + name = f"{options.archive.name}.cdx.json" + (target := options.output or options.archive.with_name(name)).write_bytes(content := dumps(document)) + sys.stdout.write(f"wrote {target}\n") + if options.embed: + with zipfile.ZipFile(options.archive, "a") as handle: + if name in handle.namelist(): + msg = f"{options.archive.name} already holds {name}" + raise SbomError(msg) + # a fixed time and Unix attributes, so the archive's bytes are the same whichever system adds the entry + entry = zipfile.ZipInfo(name, datetime.fromtimestamp(max(facts.epoch, 315532800), tz=UTC).timetuple()[:6]) + entry.create_system, entry.external_attr = 3, 0o644 << 16 + handle.writestr(entry, content) + sys.stdout.write(f"wrote {name} into {options.archive.name}\n") return True -def spdx(options: Options) -> bool: - options.target.write_bytes(dumps(to_spdx(json.loads(options.source.read_text(encoding="utf-8")), __version__))) +def add_spdx(commands: _SubParsersAction[ArgumentParser]) -> None: + parser = commands.add_parser("spdx", help="render a CycloneDX SBOM as SPDX 2.3 JSON") + parser.set_defaults(handler=_spdx) + parser.add_argument("source", type=Path, metavar="SOURCE", help="CycloneDX JSON file to read") + parser.add_argument("target", type=Path, metavar="TARGET", help="SPDX JSON file to write") + + +def _spdx(options: Options) -> bool: + if not is_document(loaded := json.loads(options.source.read_text(encoding="utf-8"))): + msg = f"{options.source} is not a CycloneDX document python-sbom can render" + raise SbomError(msg) + try: + rendered = to_spdx(loaded) + except (KeyError, TypeError) as error: + # the CLI renders without the schema check, which needs the validate extra + msg = f"{options.source} is not a CycloneDX document python-sbom can render: {error!r}" + raise SbomError(msg) from error + # bytes, so the line endings are the same on every operating system + options.target.write_bytes(rendered.encode()) return True -def validate(options: Options) -> bool: +def add_validate(commands: _SubParsersAction[ArgumentParser]) -> None: + parser = commands.add_parser( + "validate", + help="validate the SBOMs in wheels and archives", + description="Check every CycloneDX SBOM a wheel or archive holds against the CycloneDX 1.6 schema; for one" + " python-sbom wrote, also its serial number, references, SPDX rendering and, with --allow-license, the licenses" + " of what it ships. Needs the validate extra: python-sbom[validate].", + ) + parser.set_defaults(handler=_validate) + parser.add_argument( + "paths", type=Path, nargs="+", metavar="PATH", help="wheel, archive, or directory holding wheels" + ) + parser.add_argument( + "--allow-license", + dest="allowed_licenses", + action="append", + metavar="SPDX-ID", + help="SPDX license id shipped components may use; repeat it per id, leave it out to skip the license policy", + ) + + +def _validate(options: Options) -> bool: allowed = None if options.allowed_licenses is None else frozenset(options.allowed_licenses) if not ( - wheels := sorted(wheel for path in options.paths for wheel in (path.glob("*.whl") if path.is_dir() else [path])) + artifacts := sorted( + artifact for path in options.paths for artifact in (path.glob("*.whl") if path.is_dir() else [path]) + ) ): sys.stderr.write(f"no wheel found in {', '.join(map(str, options.paths))}\n") return False valid = True - for wheel in wheels: - for problem in (problems := validate_wheel(wheel, allowed)): - sys.stderr.write(f"SBOM invalid: {wheel.name}: {problem}\n") + for artifact in artifacts: + for problem in (problems := validate_artifact(artifact, allowed)): + sys.stderr.write(f"SBOM invalid: {artifact.name}: {problem}\n") if problems: valid = False else: - sys.stdout.write(f"SBOM in {wheel.name} is valid\n") + sys.stdout.write(f"SBOM in {artifact.name} is valid\n") return valid __all__ = [ - "embed", - "spdx", - "validate", + "add_archive", + "add_embed", + "add_spdx", + "add_validate", ] diff --git a/src/python_sbom/_cli/_main.py b/src/python_sbom/_cli/_main.py index 9efbc1b..2c7d71b 100644 --- a/src/python_sbom/_cli/_main.py +++ b/src/python_sbom/_cli/_main.py @@ -13,13 +13,6 @@ def main(args: Sequence[str] | None = None) -> None: - """ - Run the command-line interface. - - :param args: command-line arguments, or ``sys.argv[1:]`` when ``None`` - :raises SystemExit: with status 1 when a command fails - - """ options = build_parser().parse_args(args, namespace=Options()) try: succeeded = options.handler(options) diff --git a/src/python_sbom/_cli/_options.py b/src/python_sbom/_cli/_options.py index af70cbe..5d21063 100644 --- a/src/python_sbom/_cli/_options.py +++ b/src/python_sbom/_cli/_options.py @@ -9,12 +9,17 @@ class Options(Namespace): + # each subcommand sets only its own fields; its handler reads only those handler: Callable[[Options], bool] wheels: list[Path] + archive: Path root: Path | None commit: str | None build_env: list[str] | None + build_scripts: list[str] | None content_provider: str | None + output: Path | None + embed: bool source: Path target: Path paths: list[Path] diff --git a/src/python_sbom/_cli/_parser.py b/src/python_sbom/_cli/_parser.py index 07af09d..2f1576f 100644 --- a/src/python_sbom/_cli/_parser.py +++ b/src/python_sbom/_cli/_parser.py @@ -1,56 +1,18 @@ from __future__ import annotations from argparse import ArgumentParser -from pathlib import Path -from ._commands import embed, spdx, validate +from python_sbom.version import __version__ + +from ._commands import add_archive, add_embed, add_spdx, add_validate def build_parser() -> ArgumentParser: parser = ArgumentParser(prog="python-sbom", description="Write, convert and check the CycloneDX SBOMs of wheels.") + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") commands = parser.add_subparsers(required=True) - embed_parser = commands.add_parser( - "embed", - help="write the SBOM into wheels another build backend built", - description="Write the SBOM the hatchling hook writes into wheels another build backend built, such as" - " meson-python. Options come from the [tool.python-sbom] table of the project's pyproject.toml.", - ) - embed_parser.set_defaults(handler=embed) - embed_parser.add_argument("wheels", type=Path, nargs="+", metavar="wheel", help="wheel to rewrite in place") - embed_parser.add_argument( - "--root", type=Path, help="project directory the wheels were built from (default: current directory)" - ) - embed_parser.add_argument("--commit", help="source revision to record, for a wheel built from an sdist") - embed_parser.add_argument( - "--build-env", - action="append", - metavar="dir", - help="site-packages directory of the environment that built the wheels; repeat it per directory" - " (default: this interpreter's)", - ) - embed_parser.add_argument( - "--content-provider", - metavar="module:attribute", - help="content provider, overriding the content-provider option", - ) - spdx_parser = commands.add_parser("spdx", help="render a CycloneDX SBOM as SPDX 2.3 JSON") - spdx_parser.set_defaults(handler=spdx) - spdx_parser.add_argument("source", type=Path, help="CycloneDX JSON file to read") - spdx_parser.add_argument("target", type=Path, help="SPDX JSON file to write") - validate_parser = commands.add_parser( - "validate", help="validate the SBOMs in wheels", description="Needs the validate extra: python-sbom[validate]." - ) - validate_parser.set_defaults(handler=validate) - validate_parser.add_argument( - "paths", type=Path, nargs="+", metavar="path", help="wheel, or directory holding wheels" - ) - validate_parser.add_argument( - "--allow-license", - dest="allowed_licenses", - action="append", - metavar="spdx-id", - help="SPDX license id bundled components may use; repeat it per id, leave it out to skip the license policy", - ) + for add in (add_embed, add_archive, add_spdx, add_validate): + add(commands) return parser diff --git a/src/python_sbom/_collect/_content.py b/src/python_sbom/_collect/_content.py new file mode 100644 index 0000000..57544f5 --- /dev/null +++ b/src/python_sbom/_collect/_content.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import importlib +import inspect +import sys +from pathlib import Path, PurePosixPath + +from python_sbom._errors import SettingsError +from python_sbom._model._content import ContentProvider + + +def load_content_provider(spec: str | None, root: Path) -> ContentProvider: + if spec is None: + return ContentProvider() + module_name, _, attribute = spec.partition(":") + if not (module_name and attribute): + msg = f"content provider {spec!r} must be `module:attribute`" + raise SettingsError(msg) + sys.path.insert(0, str(root)) + try: + module = importlib.import_module(module_name) + except ImportError as error: + msg = f"content provider {spec!r} does not import from {root}: {error}" + raise SettingsError(msg) from error + finally: + sys.path.remove(str(root)) + provider = getattr(module, attribute, None) + if isinstance(provider, type) and issubclass(provider, ContentProvider): + provider = provider() + if not isinstance(provider, ContentProvider): + msg = f"content provider {spec!r} is not a ContentProvider subclass or instance" + raise SettingsError(msg) + return provider + + +def provider_script(provider: ContentProvider, root: Path) -> PurePosixPath | None: + # a provider defined in the project shapes the document, so the SBOM lists its module among the build tools + # a method's code names the file even for a module hatchling loaded outside the import system, such as a custom + # hook script + cls = type(provider) + methods = ( + function.__code__.co_filename + for _, function in inspect.getmembers(cls, inspect.isfunction) + if function.__qualname__.startswith(f"{cls.__qualname__}.") + ) + if (source := next(methods, getattr(sys.modules.get(cls.__module__), "__file__", None))) is None: + return None + resolved, base = Path(source).resolve(), root.resolve() + return PurePosixPath(resolved.relative_to(base).as_posix()) if resolved.is_relative_to(base) else None + + +__all__ = [ + "load_content_provider", + "provider_script", +] diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index cb6f1dc..0770589 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -8,55 +8,35 @@ import subprocess # ruff: ignore[suspicious-subprocess-import] # runs git to read the source revision import sys import time +import warnings from email.parser import Parser from importlib.metadata import distributions from typing import TYPE_CHECKING, Final, Protocol -from packaging.requirements import Requirement +from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name -from python_sbom._errors import SettingsError, SourceRevisionError +from python_sbom._errors import SettingsError from python_sbom._model._inputs import BuildFacts, InstalledDistribution, Interpreter, RecordRow, Revision from python_sbom._wheel._record import decode_digest if TYPE_CHECKING: - from collections.abc import Sequence - from importlib.metadata import Distribution + from collections.abc import Container, Sequence + from importlib.metadata import Distribution, PackageMetadata from pathlib import Path +#: the file an sdist carries its source revision in, as ``node: ``, for the wheel built from it +REVISION_FILE: Final[str] = ".python-sbom-revision" +_COMMIT: Final[re.Pattern[str]] = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}") -class RevisionReader(Protocol): - """Reads the source revision of a project directory.""" - - def __call__(self, root: Path, /) -> Revision | None: - """ - Read the revision. - - :param root: the project directory - :returns: the revision, or ``None`` when it is unknown - :raises SourceRevisionError: when version control holds the directory but cannot read its revision - """ +class RevisionReader(Protocol): + def __call__(self, root: Path, /) -> Revision | None: ... def collect_facts( - root: Path, revision: RevisionReader, path: Sequence[str] | None, default_epoch: int | None + root: Path, revision: RevisionReader, *, path: Sequence[str] | None = None, default_epoch: int | None = None ) -> BuildFacts: - """ - Gather what the environment says about a build. - - Nothing about the machine goes in: the same toolchain, revision and build time give the same facts on any - operating system. - - :param root: the project directory - :param revision: reads the source revision, such as :func:`default_revision` - :param path: the directories to find the build environment's distributions in, or ``None`` for :data:`sys.path` - :param default_epoch: the build time when ``SOURCE_DATE_EPOCH`` is unset, such as the fixed time of a reproducible - hatchling build, or ``None`` for the current time - :returns: the facts - :raises SettingsError: when ``SOURCE_DATE_EPOCH`` is not an integer - - """ if from_environment := bool(raw := os.environ.get("SOURCE_DATE_EPOCH", "")): try: epoch = int(raw) @@ -79,22 +59,24 @@ def collect_facts( def _installed(path: Sequence[str] | None) -> list[InstalledDistribution]: - found: Final[dict[str, Distribution]] = {} + found: Final[dict[str, tuple[Distribution, PackageMetadata]]] = {} for distribution in distributions(path=list(path)) if path is not None else distributions(): + try: + metadata = distribution.metadata + # a .dist-info without METADATA, which newer Pythons refuse to read + except FileNotFoundError: # pragma: >=3.15 cover + continue # the first on the path is the one imports load, so a shadowed copy further down is left out - found.setdefault(canonicalize_name(distribution.metadata["Name"]), distribution) + if "Name" in metadata and "Version" in metadata: + found.setdefault(canonicalize_name(metadata["Name"]), (distribution, metadata)) return [ InstalledDistribution( - metadata=Parser().parsestr(distribution.read_text("METADATA") or ""), + name=name, + metadata=metadata, files=_recorded_files(distribution), - depends_on=sorted({ - name - for requirement in map(Requirement, distribution.requires or []) - if (name := canonicalize_name(requirement.name)) in found - and (requirement.marker is None or requirement.marker.evaluate()) - }), + depends_on=_depends_on(name, distribution, found), ) - for distribution in found.values() + for name, (distribution, metadata) in found.items() ] @@ -124,64 +106,82 @@ def _describes_distribution(path: str, parent_suffix: str, name: str) -> bool: ) -def default_revision(root: Path, /) -> Revision | None: - """ - Read the source revision from the first source that knows it. - - The sources, in order: the ``PYTHON_SBOM_COMMIT`` environment variable, for builds from an sdist; git, for a - checkout or a directory inside one; and the ``node`` of an expanded ``.git_archival.txt``, for a ``git archive``. +def _depends_on(name: str, distribution: Distribution, installed: Container[str]) -> list[str]: + names: Final[set[str]] = set() + for line in distribution.requires or []: + try: + requirement = Requirement(line) + except InvalidRequirement as error: + # old distributions declare requirements, such as "pytz (>dev)", that today's syntax rejects + warnings.warn(f"left out the requirement {line!r} of {name}: {error}", stacklevel=2) + continue + if (required := canonicalize_name(requirement.name)) in installed and ( + requirement.marker is None or requirement.marker.evaluate() + ): + names.add(required) + return sorted(names) - :param root: the project directory - :returns: the revision, or ``None`` when no source knows it - :raises SourceRevisionError: when a git checkout holds the directory but git cannot read its revision - """ +def default_revision(root: Path, /) -> Revision | None: + # the variable exists for builds from an sdist, so it wins over whatever checkout the sdist was unpacked into if commit := os.environ.get("PYTHON_SBOM_COMMIT"): - return Revision(commit=commit, dirty=False) + return commit_revision(commit, "PYTHON_SBOM_COMMIT") if (found := git_revision(root)) is not None: return found + # the sdist build wrote it from the revision it resolved, so it is at least as exact as an archival file + if (recorded := root / REVISION_FILE).is_file(): + return commit_revision(recorded.read_text(encoding="utf-8").strip().removeprefix("node: "), REVISION_FILE) archival = root / ".git_archival.txt" if archival.is_file() and ( - match := re.search(r"^node: ([0-9a-f]{40,64})$", archival.read_text(encoding="utf-8"), re.MULTILINE) + match := re.search(rf"^node: ({_COMMIT.pattern})$", archival.read_text(encoding="utf-8"), re.MULTILINE) ): return Revision(commit=match[1], dirty=False) return None -def git_revision(root: Path, /) -> Revision | None: - """ - Read the commit of the git checkout that holds a directory. +def commit_revision(commit: str, origin: str) -> Revision: + # the commit lands in the source link, so anything but a full hash could point the link elsewhere + if not _COMMIT.fullmatch(commit): + msg = f"{origin} must be a full lowercase hex commit hash of 40 or 64 characters, got {commit!r}" + raise SettingsError(msg) + return Revision(commit=commit, dirty=False) - :param root: the project directory, or any directory inside a checkout - :returns: the commit and whether tracked files changed, or ``None`` outside a checkout, without git, or before the - first commit - :raises SourceRevisionError: when git cannot read the checkout, such as one it considers unsafe - """ +def git_revision(root: Path, /) -> Revision | None: resolved = root.resolve() if (git := shutil.which("git")) is None or not any( (directory / ".git").exists() for directory in (resolved, *resolved.parents) ): return None - head = _git(git, root, "rev-parse", "--verify", "--quiet", "HEAD") - if head.returncode == 1: # a checkout before its first commit + try: + # a project unpacked into another checkout, such as an sdist under its build directory, is not its source + _git(git, root, "ls-files", "--error-unmatch", "--", "pyproject.toml") + commit = _git(git, root, "rev-parse", "--verify", "--quiet", "HEAD") + # untracked files count, since the build packages every file .gitignore lets through + changes = _git(git, root, "status", "--porcelain", "--", ".") + except subprocess.CalledProcessError as error: + if error.returncode != 1: # 1: the project is untracked, or the checkout has no commit yet + warnings.warn(f"git cannot read the source revision in {root}: {error.stderr.strip()}", stacklevel=2) return None - if head.returncode: - msg = f"git rev-parse HEAD failed in {root}: {head.stderr.strip()}" - raise SourceRevisionError(msg) - status = _git(git, root, "status", "--porcelain", "--untracked-files=no") - return Revision(commit=head.stdout.strip(), dirty=bool(status.stdout.strip())) + return Revision(commit=commit, dirty=bool(changes)) -def _git(git: str, root: Path, *arguments: str) -> subprocess.CompletedProcess[str]: +def _git(git: str, root: Path, *arguments: str) -> str: + # git first runs in whichever repository encloses the project, whose fsmonitor hook would run a command from its + # config; no optional locks, so status never rewrites the index under another git process return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] # git from PATH with fixed arguments - [git, "-C", str(root), *arguments], check=False, capture_output=True, text=True - ) + [git, "-c", "core.fsmonitor=false", "--no-optional-locks", "-C", str(root), *arguments], + check=True, + capture_output=True, + text=True, + ).stdout.strip() __all__ = [ + "REVISION_FILE", "RevisionReader", "collect_facts", + "commit_revision", "default_revision", "git_revision", ] diff --git a/src/python_sbom/_collect/_settings.py b/src/python_sbom/_collect/_settings.py index 49cb765..b90ac18 100644 --- a/src/python_sbom/_collect/_settings.py +++ b/src/python_sbom/_collect/_settings.py @@ -19,15 +19,6 @@ def settings_from_pyproject(root: Path, overrides: Mapping[str, Toml]) -> Settings: - """ - Read the options from ``[tool.python-sbom]`` of a project's ``pyproject.toml``. - - :param root: the project directory - :param overrides: options that win over the table, such as a hatchling hook's own table - :returns: the settings - :raises SettingsError: when ``pyproject.toml`` is missing or not TOML, or an option is unknown or has the wrong type - - """ pyproject = root / "pyproject.toml" try: loaded = tomllib.loads(pyproject.read_text(encoding="utf-8")) @@ -42,16 +33,6 @@ def settings_from_pyproject(root: Path, overrides: Mapping[str, Toml]) -> Settin def settings_from_table(table: Mapping[str, Toml], where: str) -> Settings: - """ - Read the options from a configuration table. - - :param table: the table, with any of ``supplier``, ``repository``, ``default-branch``, ``bundled-wheels`` and - ``content-provider`` - :param where: where the table lives, named in error messages - :returns: the settings - :raises SettingsError: when a key is unknown or an option has the wrong type - - """ known = {"bundled-wheels", "content-provider", "default-branch", "repository", "supplier"} if unknown := sorted(table.keys() - known): msg = f"{where} has unknown option(s) {', '.join(unknown)}; known: {', '.join(sorted(known))}" diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index 431760b..3de8a9d 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -1,33 +1,27 @@ from __future__ import annotations import hashlib -import zipfile +from collections import Counter from email.parser import Parser +from io import BytesIO from typing import TYPE_CHECKING, Final from packaging.requirements import InvalidRequirement, Requirement from python_sbom._errors import WheelError -from python_sbom._model._inputs import BuildScript, BundledWheel, SourceTree -from python_sbom._wheel._archive import find_dist_info, member_sha256, read_metadata_member -from python_sbom._wheel._record import parse_record +from python_sbom._model._inputs import BuildScript, BundledWheel, SbomRequest, SourceTree +from python_sbom._wheel._archive import member_digest, open_archive, read_member, read_wheel_metadata +from python_sbom._wheel._record import HASHLIB_NAMES, UNHASHED, parse_record if TYPE_CHECKING: + import zipfile from email.message import Message - from pathlib import Path + from pathlib import Path, PurePosixPath - from python_sbom._model._inputs import SbomRequest + from python_sbom._model._inputs import RecordRow, Settings def read_source_tree(request: SbomRequest) -> SourceTree: - """ - Read what the document takes from the project tree: the copyright line, the bundled wheels and the build scripts. - - :param request: what the backend knows about the wheel - :returns: the source tree - :raises WheelError: when a bundled wheel is malformed - - """ root = request.root return SourceTree( copyright=next( @@ -41,11 +35,12 @@ def read_source_tree(request: SbomRequest) -> SourceTree: None, ), bundled=[ - read_bundled_wheel(root, wheel) - for wheel in sorted( - {path for pattern in request.settings.bundled_wheels for path in root.glob(pattern)}, - key=lambda path: path.relative_to(root).as_posix(), - ) + read_bundled_wheel((root / relative).read_bytes(), relative) + for relative in sorted({ + wheel.relative_to(root).as_posix() + for pattern in request.settings.bundled_wheels + for wheel in root.glob(pattern) + }) ], # str, so a Windows drive in an absolute path survives the join with the native root build_scripts=[ @@ -55,40 +50,24 @@ def read_source_tree(request: SbomRequest) -> SourceTree: ) -def read_bundled_wheel(root: Path, wheel: Path) -> BundledWheel: - """ - Read a wheel a project ships as a data file, checking every member against its ``RECORD``. - - :param root: the project directory - :param wheel: the wheel, under ``root`` - :returns: the wheel's metadata, files and vendored packages - :raises WheelError: when the wheel lacks metadata, its ``RECORD`` disagrees with its members, or it vendors a - package without one exact version - - """ - where = wheel.relative_to(root).as_posix() - with zipfile.ZipFile(wheel) as archive: +def read_bundled_wheel(data: bytes, path: str) -> BundledWheel: + where: Final[str] = f"bundled wheel {path}" + with open_archive(BytesIO(data), where) as archive: members: Final[list[str]] = sorted(name for name in archive.namelist() if not name.endswith("/")) - if (dist_info := find_dist_info(members)) is None: - msg = f"bundled wheel {where} has no .dist-info/METADATA" - raise WheelError(msg) - metadata = Parser().parsestr(read_metadata_member(archive, f"{dist_info}/METADATA", where).decode("utf-8")) - if not (metadata["Name"] and metadata["Version"]): - msg = f"bundled wheel {where}: {dist_info}/METADATA lacks Name or Version" - raise WheelError(msg) - files = parse_record(read_metadata_member(archive, f"{dist_info}/RECORD", where).decode("utf-8"), where) - _check_record(archive, where, {row.path: row.digest for row in files if row.algorithm == "SHA-256"}, members) + dist_info, metadata = read_wheel_metadata(archive, where) + _check_named(metadata, where, f"{dist_info}/METADATA") + files: Final[list[RecordRow]] = _verified_record(archive, where, dist_info) vendored: Final[list[tuple[str, Message]]] = [ *_vendor_manifest(archive, where, members), *( - (name, Parser().parsestr(read_metadata_member(archive, name, where).decode("utf-8"))) + (name, _vendored_metadata(archive, where, name)) for name in members if "/_vendor/" in name and name.endswith(".dist-info/METADATA") ), ] return BundledWheel( - path=where, - sha256=hashlib.sha256(wheel.read_bytes()).hexdigest(), + path=path, + sha256=hashlib.sha256(data).hexdigest(), metadata_name=f"{dist_info}/METADATA", metadata=metadata, files=files, @@ -96,40 +75,78 @@ def read_bundled_wheel(root: Path, wheel: Path) -> BundledWheel: ) -def _check_record(archive: zipfile.ZipFile, where: str, digests: dict[str, str], members: list[str]) -> None: - # the SBOM copies these digests, so a RECORD that misstates the wheel would make the SBOM misstate it too - unhashed = {member for member in members if not member.endswith(("/RECORD", "/RECORD.jws", "/RECORD.p7s"))} - if missing := sorted(unhashed - digests.keys()): - msg = f"bundled wheel {where}: RECORD lacks a SHA-256 for {', '.join(missing)}" +def _check_named(metadata: Message, where: str, name: str) -> None: + if not (metadata["Name"] and metadata["Version"]): + msg = f"{where}: {name} lacks Name or Version" + raise WheelError(msg) + + +def _verified_record(archive: zipfile.ZipFile, where: str, dist_info: str) -> list[RecordRow]: + # the SBOM copies these rows, so a RECORD that misstates the wheel would make the SBOM misstate it too + rows: Final[list[RecordRow]] = parse_record( + read_member(archive, f"{dist_info}/RECORD", where).decode("utf-8"), where + ) + if repeated := sorted(path for path, count in Counter(row.path for row in rows).items() if count > 1): + msg = f"{where}: RECORD lists {', '.join(repeated)} more than once" raise WheelError(msg) - if extra := sorted(digests.keys() - set(members)): - msg = f"bundled wheel {where}: RECORD lists {', '.join(extra)}, which the wheel does not hold" + members: Final[dict[str, zipfile.ZipInfo]] = { + info.filename: info for info in archive.infolist() if not info.is_dir() + } + recorded: Final[set[str]] = {row.path for row in rows} + if missing := sorted(members.keys() - recorded - {f"{dist_info}/{name}" for name in UNHASHED}): + msg = f"{where}: RECORD lacks a digest for {', '.join(missing)}" raise WheelError(msg) - if changed := [name for name in sorted(digests) if member_sha256(archive, name) != digests[name]]: - msg = f"bundled wheel {where}: the SHA-256 in RECORD differs from the content of {', '.join(changed)}" + if extra := sorted(recorded - members.keys()): + msg = f"{where}: RECORD lists {', '.join(extra)}, which the wheel does not hold" raise WheelError(msg) + if resized := [row.path for row in rows if row.size != str(members[row.path].file_size)]: + msg = f"{where}: the size in RECORD differs from the size of {', '.join(resized)}" + raise WheelError(msg) + if changed := [ + row.path + for row in rows + if member_digest(archive, members[row.path], HASHLIB_NAMES[row.algorithm]) != row.digest + ]: + msg = f"{where}: the digest in RECORD differs from the content of {', '.join(changed)}" + raise WheelError(msg) + return rows def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) -> list[tuple[str, Message]]: # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info found: Final[list[tuple[str, Message]]] = [] for name in (member for member in members if member.endswith("/_vendor/vendor.txt")): - for line in read_metadata_member(archive, name, where).decode("utf-8").splitlines(): + for line in read_member(archive, name, where).decode("utf-8").splitlines(): if not (text := line.partition("#")[0].strip()): continue try: pins = list((requirement := Requirement(text)).specifier) except InvalidRequirement as error: - msg = f"bundled wheel {where}: vendored requirement in {name} does not parse: {text}" + msg = f"{where}: vendored requirement in {name} does not parse: {text}" raise WheelError(msg) from error if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: - msg = f"bundled wheel {where}: vendored requirement in {name} pins no one exact version: {text}" + msg = f"{where}: vendored requirement in {name} pins no one exact version: {text}" raise WheelError(msg) found.append((name, Parser().parsestr(f"Name: {requirement.name}\nVersion: {pins[0].version}\n"))) return found +def _vendored_metadata(archive: zipfile.ZipFile, where: str, name: str) -> Message: + metadata: Final[Message] = Parser().parsestr(read_member(archive, name, where).decode("utf-8")) + _check_named(metadata, where, name) + return metadata + + +def wheel_request(wheel: Path, root: Path, settings: Settings, build_scripts: tuple[PurePosixPath, ...]) -> SbomRequest: + with open_archive(wheel, str(wheel)) as archive: + _, metadata = read_wheel_metadata(archive, str(wheel)) + return SbomRequest( + root=root, metadata=metadata, wheel_name=wheel.name, settings=settings, build_scripts=build_scripts + ) + + __all__ = [ "read_bundled_wheel", "read_source_tree", + "wheel_request", ] diff --git a/src/python_sbom/_errors.py b/src/python_sbom/_errors.py index 7d27a42..154f689 100644 --- a/src/python_sbom/_errors.py +++ b/src/python_sbom/_errors.py @@ -5,26 +5,20 @@ class SbomError(Exception): """Base of the errors this package raises; the command line reports them without a traceback.""" -class SettingsError(SbomError, TypeError): - """A setting has the wrong type.""" +# callers catch these as SbomError, or as the built-in some of them also are; SettingsError covers every option, +# file or variable the project sets wrongly +class SettingsError(SbomError): ... # simplint: ignore[SMP704] # see above -class WheelError(SbomError, ValueError): - """A wheel, built or bundled, lacks what its SBOM needs to describe it.""" +class WheelError(SbomError, ValueError): ... # simplint: ignore[SMP704] # see above -class SourceRevisionError(SbomError): - """The project is a git checkout, but git cannot read its revision.""" - - -class MissingExtraError(SbomError, ImportError): - """A feature needs an optional extra that is not installed.""" +class MissingExtraError(SbomError, ImportError): ... # simplint: ignore[SMP704] # see above __all__ = [ "MissingExtraError", "SbomError", "SettingsError", - "SourceRevisionError", "WheelError", ] diff --git a/src/python_sbom/_formats/_json.py b/src/python_sbom/_formats/_json.py index c65b888..4ec244d 100644 --- a/src/python_sbom/_formats/_json.py +++ b/src/python_sbom/_formats/_json.py @@ -10,13 +10,6 @@ def dumps(document: Document | Spdx) -> bytes: - """ - Serialize a document the way python-sbom writes it: two-space indented JSON and a final newline. - - :param document: a CycloneDX or SPDX document - :returns: UTF-8 bytes, with the same line endings on every operating system - - """ return f"{json.dumps(document, indent=2)}\n".encode() diff --git a/src/python_sbom/_formats/_spdx.py b/src/python_sbom/_formats/_spdx.py index 7173b5b..69d63f5 100644 --- a/src/python_sbom/_formats/_spdx.py +++ b/src/python_sbom/_formats/_spdx.py @@ -1,14 +1,49 @@ from __future__ import annotations import hashlib +import json import re from datetime import UTC, datetime from typing import TYPE_CHECKING, Final, NotRequired, TypedDict -from packaging.utils import canonicalize_name +from python_sbom._model._names import SBOM_FILE, SBOM_REPOSITORY, property_prefix, sbom_property +from python_sbom.version import __version__ + +from ._json import dumps if TYPE_CHECKING: - from python_sbom._model._types import Component, Document, License + from python_sbom._model._types import ( + Component, + Document, + ExternalReference, + License, + Metadata, + OrganizationalEntity, + Tools, + ) + +# SPDX 2.3 JSON names every CycloneDX 1.6 hash algorithm, but by names of its own +_HASH_ALGORITHMS: Final[dict[str, str]] = { + "MD5": "MD5", + "SHA-1": "SHA1", + "SHA-256": "SHA256", + "SHA-384": "SHA384", + "SHA-512": "SHA512", + "SHA3-256": "SHA3-256", + "SHA3-384": "SHA3-384", + "SHA3-512": "SHA3-512", + "BLAKE2b-256": "BLAKE2b-256", + "BLAKE2b-384": "BLAKE2b-384", + "BLAKE2b-512": "BLAKE2b-512", + "BLAKE3": "BLAKE3", +} +_LICENSE_REF: Final[re.Pattern[str]] = re.compile(r"LicenseRef-[A-Za-z0-9.-]+") + + +class _LegacyTool(TypedDict): + vendor: NotRequired[str] + name: NotRequired[str] + version: NotRequired[str] class _Checksum(TypedDict): @@ -59,22 +94,6 @@ class _ExtractedLicense(TypedDict): class Spdx(TypedDict): - """ - An SPDX 2.3 JSON document, as :func:`to_spdx` renders it. - - :ivar spdxVersion: ``SPDX-2.3`` - :ivar dataLicense: ``CC0-1.0`` - :ivar SPDXID: ``SPDXRef-DOCUMENT`` - :ivar name: the root component's name and version - :ivar documentNamespace: a URI that ends in the CycloneDX serial number - :ivar comment: which CycloneDX document it was rendered from - :ivar creationInfo: the build time and the creators - :ivar packages: one package per component, files aside - :ivar relationships: containment, dependency and build tool relationships - :ivar hasExtractedLicensingInfos: one entry per ``LicenseRef-`` a package declares - - """ - spdxVersion: str dataLicense: str SPDXID: str @@ -87,76 +106,44 @@ class Spdx(TypedDict): hasExtractedLicensingInfos: NotRequired[list[_ExtractedLicense]] -def to_spdx(bom: Document, tool_version: str) -> Spdx: - """ - Render a CycloneDX document as SPDX 2.3. - - Packages keep their hashes, licenses and purls, and the containment, dependency and build tool relationships carry - over. Per-file hashes stay behind, because SPDX 2.3 requires a SHA-1 for every file and wheel ``RECORD`` files hold - SHA-256, and so does the build formulation, which SPDX 2.3 has no field for. +def to_spdx(document: Document) -> str: + return dumps(render_spdx(document, __version__)).decode() - :param bom: a CycloneDX document; python-sbom's own documents name their file and repository, others fall back - :param tool_version: the python-sbom version to name as a creator - :returns: the SPDX document, whose namespace ends in the CycloneDX serial number so both name the same build - """ - root: Final[Component] = bom["metadata"]["component"] - tools: Final[list[Component]] = bom["metadata"]["tools"]["components"] - prefix = f"{canonicalize_name(root['name'])}:" - recorded = {prop["name"]: prop["value"] for prop in bom["metadata"].get("properties", [])} +def render_spdx(bom: Document, tool_version: str) -> Spdx: + metadata: Final[Metadata] = bom["metadata"] + root: Final[Component] = metadata["component"] + project: Final[str] = property_prefix(root["name"]) + recorded: Final[dict[str, str]] = {prop["name"]: prop.get("value", "") for prop in metadata.get("properties", [])} name: Final[str] = f"{root['name']}-{root['version']}" if "version" in root else root["name"] - serial = bom["serialNumber"].removeprefix("urn:uuid:") - components: Final[list[Component]] = _walk([root, *bom["components"], *tools]) - packages = [_package(component, prefix) for component in components] + # without a serial number the content names the document, so rendering it twice gives the same namespace + source: Final[str] = ( + bom["serialNumber"] + if "serialNumber" in bom + else f"sha256:{hashlib.sha256(json.dumps(bom, sort_keys=True).encode()).hexdigest()}" + ) + repository: Final[str] = recorded.get(sbom_property(project, SBOM_REPOSITORY), "https://spdx.org/spdxdocs") + sbom_file: Final[str] = recorded.get(sbom_property(project, SBOM_FILE), "CycloneDX JSON") + tools, tool_creators = _tools(metadata) + components: Final[list[Component]] = _walk([root, *bom.get("components", []), *tools]) + packages: Final[list[_Package]] = [_package(component, f"{project}:") for component in components] spdx: Final[Spdx] = { "spdxVersion": "SPDX-2.3", "dataLicense": "CC0-1.0", "SPDXID": "SPDXRef-DOCUMENT", "name": name, - "documentNamespace": f"{recorded.get(f'{prefix}sbom:repository', 'https://spdx.org/spdxdocs')}/sboms/{name}-{serial}", - "comment": f"Rendered from the CycloneDX SBOM {bom['serialNumber']}" - f" ({recorded.get(f'{prefix}sbom:file', 'CycloneDX JSON')}), which also carries per-file hashes and the build" - " formulation.", - "creationInfo": { - "created": datetime - .fromisoformat(bom["metadata"]["timestamp"]) - .astimezone(UTC) - .strftime("%Y-%m-%dT%H:%M:%SZ"), - "creators": [ - *( - [f"Organization: {supplier['name']}"] - if (supplier := bom["metadata"].get("supplier")) is not None - else [] - ), - *( - f"Tool: {tool['name']}-{tool['version']}" if "version" in tool else f"Tool: {tool['name']}" - for tool in tools - if tool["type"] == "application" - ), - f"Tool: python-sbom-{tool_version}", - ], - }, + "documentNamespace": f"{repository}/sboms/{name}-{source.removeprefix('urn:uuid:')}", + "comment": f"Rendered from the CycloneDX SBOM {source} ({sbom_file}), which also carries per-file hashes and" + " the build formulation.", + "creationInfo": _creation_info(metadata, tool_creators, tool_version), "packages": packages, - "relationships": [ - _relationship("SPDXRef-DOCUMENT", "DESCRIBES", _spdx_id(root["bom-ref"])), - *( - _relationship(_spdx_id(parent["bom-ref"]), "CONTAINS", _spdx_id(child["bom-ref"])) - for parent in components - for child in parent.get("components", []) - if child["type"] != "file" - ), - *( - _relationship(_spdx_id(entry["ref"]), "DEPENDS_ON", _spdx_id(dependency)) - for entry in bom["dependencies"] - for dependency in entry.get("dependsOn", []) - ), - *(_relationship(_spdx_id(tool["bom-ref"]), "BUILD_TOOL_OF", _spdx_id(root["bom-ref"])) for tool in tools), - ], + "relationships": _relationships(bom, components, tools), } # SPDX requires the text of every LicenseRef a package declares; the metadata names them, it holds no text - if references := sorted({ - token for package in packages for token in re.findall(r"LicenseRef-[A-Za-z0-9.-]+", package["licenseDeclared"]) - }): + references: Final[list[str]] = sorted({ + token for package in packages for token in _LICENSE_REF.findall(package["licenseDeclared"]) + }) + if references: spdx["hasExtractedLicensingInfos"] = [ {"licenseId": token, "extractedText": f"{token}, as named in package metadata", "name": token} for token in references @@ -164,11 +151,27 @@ def to_spdx(bom: Document, tool_version: str) -> Spdx: return spdx +def _tools(metadata: Metadata) -> tuple[list[Component], list[str]]: + # CycloneDX 1.5 lists the tools as components; earlier versions as a bare array, whose entries carry no bom-ref, + # so they become creators alone + tools: Final[Tools | list[_LegacyTool] | None] = metadata.get("tools") + if isinstance(tools, list): + return [], [_creator(name, tool.get("version")) for tool in tools if (name := tool.get("name"))] + components: Final[list[Component]] = [] if tools is None else tools.get("components", []) + return components, [ + _creator(tool["name"], tool.get("version")) for tool in components if tool["type"] == "application" + ] + + +def _creator(name: str, version: str | None) -> str: + return f"Tool: {name}" if version is None else f"Tool: {name}-{version}" + + def _walk(components: list[Component]) -> list[Component]: # iterative, so a deeply nested hostile document cannot exhaust the stack; file components mirror wheel RECORD # entries, which carry no SHA-1 and so cannot become SPDX files found: Final[list[Component]] = [] - pending = list(reversed(components)) + pending: Final[list[Component]] = list(reversed(components)) while pending: if (component := pending.pop())["type"] != "file": found.append(component) @@ -194,13 +197,16 @@ def _package(component: Component, property_prefix: str) -> _Package: } if version := component.get("version"): package["versionInfo"] = version - if supplier := component.get("supplier"): - package["supplier"] = f"Organization: {supplier['name']}" - if hashes := component.get("hashes"): - package["checksums"] = [ - {"algorithm": entry["alg"].replace("-", ""), "checksumValue": entry["content"]} for entry in hashes - ] - references = component.get("externalReferences", []) + if (supplier := component.get("supplier")) and (organization := supplier.get("name")): + package["supplier"] = f"Organization: {organization}" + checksums: Final[list[_Checksum]] = [ + {"algorithm": _HASH_ALGORITHMS[entry["alg"]], "checksumValue": entry["content"]} + for entry in component.get("hashes", []) + if entry["alg"] in _HASH_ALGORITHMS + ] + if checksums: + package["checksums"] = checksums + references: Final[list[ExternalReference]] = component.get("externalReferences", []) if homepage := next((reference["url"] for reference in references if reference["type"] == "website"), None): package["homepage"] = homepage if copyright_text := component.get("copyright"): @@ -208,7 +214,7 @@ def _package(component: Component, property_prefix: str) -> _Package: if description := component.get("description"): package["summary"] = description if comment := "\n".join( - f"{prop['name']}: {prop['value']}" + f"{prop['name']}: {prop.get('value', '')}" for prop in component.get("properties", []) if prop["name"].startswith(property_prefix) ): @@ -224,18 +230,49 @@ def _package(component: Component, property_prefix: str) -> _Package: return package -def _relationship(element: str, kind: str, related: str) -> _Relationship: - return {"spdxElementId": element, "relationshipType": kind, "relatedSpdxElement": related} - - def _spdx_id(bom_ref: str) -> str: # SPDX ids allow letters, digits, "." and "-"; the digest keeps two refs that differ only in other characters, # such as the markers of two conditional requirements, apart - readable = re.sub(r"[^A-Za-z0-9.]+", "-", bom_ref).strip("-") + readable: Final[str] = re.sub(r"[^A-Za-z0-9.]+", "-", bom_ref).strip("-") return f"SPDXRef-{readable}-{hashlib.sha256(bom_ref.encode()).hexdigest()[:8]}" +def _creation_info(metadata: Metadata, tool_creators: list[str], tool_version: str) -> _CreationInfo: + timestamp: Final[str | None] = metadata.get("timestamp") + # SPDX requires a creation time; without the CycloneDX one, the time of this rendering is the true one + created: Final[datetime] = datetime.now(UTC) if timestamp is None else datetime.fromisoformat(timestamp) + supplier: Final[OrganizationalEntity | None] = metadata.get("supplier") + organization: Final[str] = "" if supplier is None else supplier.get("name", "") + creators: Final[list[str]] = [f"Organization: {organization}"] if organization else [] + creators.extend(tool_creators) + creators.append(f"Tool: python-sbom-{tool_version}") + return {"created": created.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "creators": creators} + + +def _relationships(bom: Document, components: list[Component], tools: list[Component]) -> list[_Relationship]: + root_id: Final[str] = _spdx_id(bom["metadata"]["component"]["bom-ref"]) + relationships: Final[list[_Relationship]] = [_relationship("SPDXRef-DOCUMENT", "DESCRIBES", root_id)] + relationships.extend( + _relationship(_spdx_id(parent["bom-ref"]), "CONTAINS", _spdx_id(child["bom-ref"])) + for parent in components + for child in parent.get("components", []) + if child["type"] != "file" + ) + relationships.extend( + _relationship(_spdx_id(entry["ref"]), "DEPENDS_ON", _spdx_id(dependency)) + for entry in bom.get("dependencies", []) + for dependency in entry.get("dependsOn", []) + ) + relationships.extend(_relationship(_spdx_id(tool["bom-ref"]), "BUILD_TOOL_OF", root_id) for tool in tools) + return relationships + + +def _relationship(element: str, kind: str, related: str) -> _Relationship: + return {"spdxElementId": element, "relationshipType": kind, "relatedSpdxElement": related} + + __all__ = [ "Spdx", + "render_spdx", "to_spdx", ] diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py new file mode 100644 index 0000000..7a0762a --- /dev/null +++ b/src/python_sbom/_generate/_archive.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import hashlib +import tempfile +from dataclasses import dataclass +from email.parser import Parser +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final +from urllib.parse import quote + +from python_sbom._collect._content import load_content_provider, provider_script +from python_sbom._collect._settings import settings_from_pyproject +from python_sbom._collect._source import read_bundled_wheel +from python_sbom._errors import WheelError +from python_sbom._model._components import ( + bundled_wheel_component, + component_from_metadata, + file_component_from_bytes, + script_tool, + vendored_dependencies, +) +from python_sbom._model._content import ProviderContext +from python_sbom._model._document import contributed_properties, formulation, timestamp, with_header +from python_sbom._model._identity import project_identity +from python_sbom._model._names import SBOM_FILE, sbom_property +from python_sbom._model._tools import tool_components +from python_sbom._model._types import Dependency +from python_sbom._wheel._archive import open_archive + +if TYPE_CHECKING: + from collections.abc import Sequence + from email.message import Message + + from python_sbom._model._content import ContentProvider + from python_sbom._model._identity import ProjectIdentity + from python_sbom._model._inputs import BuildFacts, Settings + from python_sbom._model._types import Component, Document, Metadata + + +def describe_archive( + root: Path, + archive: Path, + *, + facts: BuildFacts, + content: ContentProvider | None = None, + build_scripts: Sequence[PurePosixPath] = (), +) -> Document: + # the archive's top-level .dist-info names the project; every other .dist-info marks a wheel bundled unpacked into + # its parent directory and every .whl member a bundled wheel; other files belong to the deepest distribution that + # holds them, or, at the top level, to the archive + settings = settings_from_pyproject(root, {}) + content = content or load_content_provider(settings.content_provider, root) + with tempfile.TemporaryDirectory() as directory: + held = _read_archive(root, archive, Path(directory), settings, content) + contributions = content.contributions(held.reader.context) + held.root["properties"] = list(contributed_properties(contributions.root_properties, held.project.property_prefix)) + if facts.revision is not None: + held.root["properties"].append({ + "name": f"{held.project.property_prefix}:vcs-commit", + "value": facts.revision.commit, + }) + components: Final[list[Component]] = [*held.components, *contributions.components] + tools: Final[list[Component]] = [ + *( + script_tool( + path, + hashlib.sha256((root / str(path)).read_bytes()).hexdigest(), + held.version, + None + if held.project.repository is None + else f"{held.project.repository}/blob/{settings.default_branch}/{path}", + ) + for path in [*build_scripts, *filter(None, [provider_script(content, root)])] + ), + *(environment := tool_components(facts))[0], + *contributions.tools, + ] + metadata: Final[Metadata] = { + "timestamp": timestamp(facts), + "lifecycles": [{"phase": "build"}], + "tools": {"components": tools}, + "component": held.root, + # names the document's file, which marks it as one python-sbom built, so validation checks it in full + "properties": [ + {"name": sbom_property(held.project.property_prefix, SBOM_FILE), "value": f"{archive.name}.cdx.json"} + ], + } + if settings.supplier is not None: + held.root["supplier"] = metadata["manufacturer"] = metadata["supplier"] = settings.supplier + return with_header( + { + "metadata": metadata, + "components": components, + "dependencies": [ + Dependency(ref=held.root["bom-ref"], dependsOn=[component["bom-ref"] for component in components]), + *vendored_dependencies(components), + *environment[1], + *contributions.dependencies, + ], + "formulation": [ + formulation("archive-build", f"build {archive.name} and this SBOM", tools, held.root["bom-ref"], facts) + ], + }, + held.project.serial_namespace, + ) + + +def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, content: ContentProvider) -> _Held: + # unpacked, so a content provider reads the archive's files the way it reads the project's + with open_archive(archive, archive.name) as handle: + members = sorted( + name for name in handle.namelist() if not name.endswith("/") and name != f"{archive.name}.cdx.json" + ) + handle.extractall(unpacked) + # each distribution by the directory that holds its .dist-info, the project's own at the top level + metadata = { + f"{name.rsplit('/', 2)[0]}/" if name.count("/") > 1 else "": Parser().parsestr( + (unpacked / name).read_text(encoding="utf-8") + ) + for name in members + if name.endswith(".dist-info/METADATA") + } + if (top := sum(name.count("/") == 1 and name.endswith(".dist-info/METADATA") for name in members)) != 1: + msg = f"{archive.name} holds {top} top-level .dist-info/METADATA, not the one naming the project" + raise WheelError(msg) + owned: Final[dict[str, list[str]]] = {prefix: [] for prefix in metadata} + loose: Final[list[str]] = [] + for name in members: + # the deepest distribution directory that holds the file; the project's own holds every other subdirectory + if owner := max((prefix for prefix in metadata if prefix and name.startswith(prefix)), key=len, default=""): + owned[owner].append(name) + elif "/" in name: + owned[""].append(name) + else: + loose.append(name) + project = project_identity(metadata[""], settings) + reader = _Reader( + unpacked=unpacked, + project=project, + settings=settings, + content=content, + context=ProviderContext(root=root, settings=settings, identity=project, artifact=unpacked), + ) + version = metadata[""]["Version"] + ref = f"pkg:generic/{quote(archive.name, safe='')}@{quote(version, safe='')}" + return _Held( + reader=reader, + project=reader.project, + version=version, + root={ + "type": "application", + "bom-ref": ref, + "name": archive.name, + "version": version, + "purl": ref, + "components": [reader.file(ref, name) for name in loose], + }, + components=[reader.distribution(prefix, metadata[prefix], owned[prefix]) for prefix in sorted(metadata)], + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _Reader: + unpacked: Path + project: ProjectIdentity + settings: Settings + content: ContentProvider + context: ProviderContext + + def distribution(self, prefix: str, metadata: Message, names: list[str]) -> Component: + component = component_from_metadata(metadata) + if prefix: + # a wheel bundled unpacked: the directory tells copies of one release apart, as the path does for a .whl + directory = PurePosixPath(prefix.rstrip("/")) + info = self.content.bundled_wheel(self.context, directory) + component["bom-ref"] = f"bundled:{directory}" + if info.supplier is not None: + component["supplier"] = info.supplier + component["properties"] = [ + {"name": f"{self.project.property_prefix}:bundled-wheel", "value": str(directory)}, + *info.properties, + *component["properties"], + ] + component["components"] = [ + self.wheel(name) if name.endswith(".whl") else self.file(component["bom-ref"], name) for name in names + ] + return component + + def wheel(self, name: str) -> Component: + info = self.content.bundled_wheel(self.context, PurePosixPath(name)) + return bundled_wheel_component( + read_bundled_wheel((self.unpacked / name).read_bytes(), name), + self.project.property_prefix, + info.properties, + info.supplier, + ) + + def file(self, parent: str, name: str) -> Component: + return file_component_from_bytes(parent, name, (self.unpacked / name).read_bytes()) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _Held: + reader: _Reader + project: ProjectIdentity + version: str + root: Component + components: list[Component] + + +__all__ = [ + "describe_archive", +] diff --git a/src/python_sbom/_generate/_backend.py b/src/python_sbom/_generate/_backend.py deleted file mode 100644 index 4879e5b..0000000 --- a/src/python_sbom/_generate/_backend.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -import zipfile -from abc import ABC, abstractmethod -from email.parser import Parser -from typing import TYPE_CHECKING - -from python_sbom._errors import WheelError -from python_sbom._model._inputs import SbomRequest -from python_sbom._wheel._archive import find_dist_info, read_metadata_member - -if TYPE_CHECKING: - from pathlib import Path, PurePosixPath - - from python_sbom._model._inputs import Settings - - -class Backend(ABC): - """Adapts a build system to python-sbom: it says what it knows about the wheel it builds or rewrites.""" - - @abstractmethod - def request(self) -> SbomRequest: - """ - Describe the wheel. - - :returns: the project directory, the wheel's metadata and file name, the settings and the build scripts - - """ - - -class MetadataBackend(Backend): - """ - A backend for build systems that hand over the core metadata before the wheel exists, such as a build hook. - - :param root: the project directory - :param metadata: the wheel's ``METADATA`` text - :param wheel_name: the wheel's file name - :param settings: the project's options - :param build_scripts: scripts, relative to ``root``, that shape the document - - """ - - def __init__( - self, - *, - root: Path, - metadata: str, - wheel_name: str, - settings: Settings, - build_scripts: tuple[PurePosixPath, ...], - ) -> None: - self._request = SbomRequest( - root=root, - metadata=Parser().parsestr(metadata), - wheel_name=wheel_name, - settings=settings, - build_scripts=build_scripts, - ) - - def request(self) -> SbomRequest: - """ - Return the request built from the metadata. - - :returns: the request - - """ - return self._request - - -class BuiltWheel(Backend): - """ - A wheel some build backend already built, such as meson-python, and the project it came from. - - :param wheel: the built wheel file - :param root: the project directory the wheel was built from - :param settings: the project's options - :param build_scripts: scripts, relative to ``root``, that shape the document - - """ - - def __init__( - self, *, wheel: Path, root: Path, settings: Settings, build_scripts: tuple[PurePosixPath, ...] - ) -> None: - self._wheel = wheel - self._root = root - self._settings = settings - self._build_scripts = build_scripts - - def request(self) -> SbomRequest: - """ - Read the wheel's core metadata. - - :returns: the request - :raises WheelError: when the archive is not a wheel - - """ - with zipfile.ZipFile(self._wheel) as archive: - if (dist_info := find_dist_info(archive.namelist())) is None: - msg = f"{self._wheel} has no .dist-info/METADATA, so it is not a wheel" - raise WheelError(msg) - metadata = read_metadata_member(archive, f"{dist_info}/METADATA", str(self._wheel)).decode("utf-8") - return SbomRequest( - root=self._root, - metadata=Parser().parsestr(metadata), - wheel_name=self._wheel.name, - settings=self._settings, - build_scripts=self._build_scripts, - ) - - -__all__ = [ - "Backend", - "BuiltWheel", - "MetadataBackend", -] diff --git a/src/python_sbom/_generate/_content.py b/src/python_sbom/_generate/_content.py deleted file mode 100644 index f02ddd6..0000000 --- a/src/python_sbom/_generate/_content.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import annotations - -import importlib -import sys -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING - -from python_sbom._errors import SettingsError -from python_sbom._model._inputs import BundledWheelInfo, Contributions - -if TYPE_CHECKING: - from python_sbom._model._inputs import BundledWheel, SbomRequest - - -def load_content_provider(spec: str, root: Path) -> tuple[ContentProvider, PurePosixPath | None]: - """ - Import the content provider a ``content-provider`` option names. - - :param spec: ``module:attribute``, importable from ``root``; the attribute is a :class:`ContentProvider` subclass or - instance - :param root: the project directory, searched first for the module - :returns: the provider, and the module's path relative to ``root`` when it lives there, which the SBOM lists among - the build tools since it shapes the document - :raises SettingsError: when the spec is malformed, the import fails, or the attribute is no content provider - - """ - module_name, _, attribute = spec.partition(":") - if not (module_name and attribute): - msg = f"content provider {spec!r} must be `module:attribute`" - raise SettingsError(msg) - sys.path.insert(0, str(root)) - try: - module = importlib.import_module(module_name) - except ImportError as error: - msg = f"content provider {spec!r} does not import from {root}: {error}" - raise SettingsError(msg) from error - finally: - sys.path.remove(str(root)) - provider = getattr(module, attribute, None) - if isinstance(provider, type) and issubclass(provider, ContentProvider): - provider = provider() - if not isinstance(provider, ContentProvider): - msg = f"content provider {spec!r} is not a ContentProvider subclass or instance" - raise SettingsError(msg) - source = Path(module.__file__ or "").resolve() - script = ( - PurePosixPath(source.relative_to(root.resolve()).as_posix()) if source.is_relative_to(root.resolve()) else None - ) - return provider, script - - -class ContentProvider: - """ - Adds project-specific content to an SBOM; subclass it and override what the project needs. - - Name a subclass in the ``content-provider`` option, or pass an instance to :func:`build_sbom`. - - """ - - def bundled_wheel(self, wheel: BundledWheel) -> BundledWheelInfo: # ruff: ignore[unused-method-argument, no-self-use] # the no-op default - """ - Describe what the project does with a wheel it bundles. - - :param wheel: the bundled wheel - :returns: its extra properties and supplier; none by default - - """ - return BundledWheelInfo(properties=[], supplier=None) - - def contributions(self, request: SbomRequest) -> Contributions: # ruff: ignore[unused-method-argument, no-self-use] # the no-op default - """ - Add components, dependencies, root properties or tools, before references and the serial are derived. - - :param request: what the backend knows about the wheel - :returns: the contributions; none by default - - """ - return Contributions(components=[], dependencies=[], root_properties=[], tools=[]) - - -__all__ = [ - "ContentProvider", - "load_content_provider", -] diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py new file mode 100644 index 0000000..66cf3b0 --- /dev/null +++ b/src/python_sbom/_generate/_embed.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +from python_sbom._collect._content import load_content_provider, provider_script +from python_sbom._collect._facts import collect_facts, commit_revision, default_revision +from python_sbom._collect._settings import settings_from_pyproject +from python_sbom._collect._source import read_source_tree, wheel_request +from python_sbom._model._content import ProviderContext +from python_sbom._model._identity import project_identity +from python_sbom._wheel._archive import member_digest, open_archive, write_sbom + +from ._pipeline import build_sbom + +if TYPE_CHECKING: + from collections.abc import Sequence + from pathlib import Path + + from python_sbom._collect._facts import RevisionReader + from python_sbom._model._content import ContentProvider + from python_sbom._model._inputs import BuildFacts, SbomRequest + + +def embed_sbom( + wheel: Path, + root: Path, + *, + content: ContentProvider | None, + commit: str | None = None, + build_env: Sequence[str] | None, +) -> str: + settings = settings_from_pyproject(root, {}) + content = content or load_content_provider(settings.content_provider, root) + scripts = tuple(filter(None, [provider_script(content, root)])) + return embed_into( + wheel, + wheel_request(wheel, root, settings, scripts), + content, + collect_facts(root, revision_reader(commit), path=build_env), + ) + + +def embed_into(wheel: Path, request: SbomRequest, content: ContentProvider, facts: BuildFacts) -> str: + source = read_source_tree(request) + with open_archive(wheel, str(wheel)) as archive: + inside = {member_digest(archive, name, "sha256"): name for name in archive.namelist() if name.endswith(".whl")} + # a bundled wheel the built wheel holds is named by its path in there, as an archive SBOM names it, so both + # documents give one wheel the same identity + source = dataclasses.replace( + source, + bundled=[ + dataclasses.replace(bundled, path=inside.get(bundled.sha256, bundled.path)) for bundled in source.bundled + ], + ) + project = project_identity(request.metadata, request.settings) + context = ProviderContext(root=request.root, settings=request.settings, identity=project, artifact=wheel) + sbom = build_sbom(request, source, facts, content, context) + return write_sbom(wheel, sbom.file_name, sbom.content, facts.epoch) + + +def revision_reader(commit: str | None) -> RevisionReader: + if commit is None: + return default_revision + revision = commit_revision(commit, "--commit") + return lambda _root: revision + + +__all__ = [ + "embed_into", + "embed_sbom", + "revision_reader", +] diff --git a/src/python_sbom/_generate/_pipeline.py b/src/python_sbom/_generate/_pipeline.py index 7395583..b722441 100644 --- a/src/python_sbom/_generate/_pipeline.py +++ b/src/python_sbom/_generate/_pipeline.py @@ -1,58 +1,41 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import PurePosixPath from typing import TYPE_CHECKING from python_sbom._formats._json import dumps from python_sbom._model._bom import describe -from python_sbom._model._components import sbom_file_name +from python_sbom._model._names import sbom_file_name if TYPE_CHECKING: + from python_sbom._model._content import ContentProvider, ProviderContext from python_sbom._model._inputs import BuildFacts, SbomRequest, SourceTree from python_sbom._model._types import Document - from ._content import ContentProvider - -def build_sbom(request: SbomRequest, source: SourceTree, facts: BuildFacts, content: ContentProvider) -> Sbom: - """ - Build the SBOM of a wheel from what the edge gathered. - - :param request: what the backend knows about the wheel, from :meth:`Backend.request` - :param source: the project tree, from :func:`read_source_tree` - :param facts: the build facts, from :func:`collect_facts` - :param content: what the project adds - :returns: the SBOM - - """ +def build_sbom( + request: SbomRequest, source: SourceTree, facts: BuildFacts, content: ContentProvider, context: ProviderContext +) -> Sbom: return Sbom( file_name=sbom_file_name(request.metadata["Name"]), document=describe( request, source, facts, - {wheel.path: content.bundled_wheel(wheel) for wheel in source.bundled}, - content.contributions(request), + {wheel.path: content.bundled_wheel(context, PurePosixPath(wheel.path)) for wheel in source.bundled}, + content.contributions(context), ), ) @dataclass(frozen=True, slots=True, kw_only=True) class Sbom: - """ - An SBOM ready to write. - - :param file_name: its file name under ``.dist-info/sboms/`` - :param document: the CycloneDX document - - """ - file_name: str document: Document @property def content(self) -> bytes: - """The document as python-sbom writes it: indented JSON with a final newline.""" return dumps(self.document) diff --git a/src/python_sbom/_model/_bom.py b/src/python_sbom/_model/_bom.py index a2840f5..01beebd 100644 --- a/src/python_sbom/_model/_bom.py +++ b/src/python_sbom/_model/_bom.py @@ -6,23 +6,17 @@ from packaging.requirements import Requirement -from ._components import ( - bundled_wheel_component, - component_from_metadata, - purl, - pypi_url, - sbom_file_name, - script_tool, -) -from ._document import document, formulation, project_identity, timestamp +from ._components import bundled_wheel_component, component_from_metadata, script_tool, vendored_dependencies +from ._document import contributed_properties, formulation, timestamp, with_header +from ._identity import project_identity from ._markers import applies_without_extras +from ._names import SBOM_FILE, SBOM_REPOSITORY, purl, pypi_url, sbom_file_name, sbom_property from ._tools import tool_components -from ._types import Dependency if TYPE_CHECKING: from collections.abc import Mapping - from ._document import ProjectIdentity + from ._identity import ProjectIdentity from ._inputs import BuildFacts, BundledWheelInfo, Contributions, SbomRequest, Settings, SourceTree from ._types import Component, Document, ExternalReference, Metadata, Property @@ -36,12 +30,20 @@ def describe( ) -> Document: project = project_identity(request.metadata, request.settings) root = _root_component(request, source, facts, project, contributions) - bundled = [bundled_wheel_component(wheel, project.namespace, bundled_info[wheel.path]) for wheel in source.bundled] + bundled = [ + bundled_wheel_component( + wheel, + project.property_prefix, + bundled_info[wheel.path].properties, + bundled_info[wheel.path].supplier, + ) + for wheel in source.bundled + ] components = [ *bundled, # hatchling repeats a Requires-Dist when a hook re-adds a dependency, and each bom-ref must be unique *( - _declared_dependency(text, requirement, project.namespace) + _declared_dependency(text, requirement, project.property_prefix) for text in dict.fromkeys(request.metadata.get_all("Requires-Dist", [])) if applies_without_extras(requirement := Requirement(text)) ), @@ -61,21 +63,13 @@ def describe( *environment, *contributions.tools, ] - return document( + return with_header( { "metadata": _metadata(root, tools, request, project, facts), "components": components, "dependencies": [ {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in components]}, - *( - Dependency(ref=component["bom-ref"], dependsOn=vendored) - for component in bundled - if ( - vendored := [ - child["bom-ref"] for child in component.get("components", []) if child["type"] == "library" - ] - ) - ), + *vendored_dependencies(bundled), *tool_dependencies, *contributions.dependencies, ], @@ -112,7 +106,10 @@ def _root_component( link := _file_link(project, settings, license_files[0]) ): references.append({"type": "license", "url": link}) - properties: Final[list[Property]] = [*root["properties"], *contributions.root_properties] + properties: Final[list[Property]] = [ + *root["properties"], + *contributed_properties(contributions.root_properties, project.property_prefix), + ] if (revision := facts.revision) is not None: if project.repository: comment = "source revision, with uncommitted changes" if revision.dirty else "exact source revision" @@ -121,9 +118,9 @@ def _root_component( "url": f"{project.repository}/tree/{revision.commit}", "comment": comment, }) - properties.append({"name": f"{project.namespace}:vcs-commit", "value": revision.commit}) + properties.append({"name": f"{project.property_prefix}:vcs-commit", "value": revision.commit}) if revision.dirty: - properties.append({"name": f"{project.namespace}:vcs-dirty", "value": "true"}) + properties.append({"name": f"{project.property_prefix}:vcs-dirty", "value": "true"}) root["externalReferences"] = references root["properties"] = properties return root @@ -165,11 +162,14 @@ def _metadata( if "licenses" in root: metadata["licenses"] = root["licenses"] properties: Final[list[Property]] = [ - {"name": f"{project.namespace}:sbom:scope", "value": _scope(settings)}, - {"name": f"{project.namespace}:sbom:file", "value": sbom_file_name(request.metadata["Name"])}, + {"name": sbom_property(project.property_prefix, "sbom:scope"), "value": _scope(settings)}, + {"name": sbom_property(project.property_prefix, SBOM_FILE), "value": sbom_file_name(request.metadata["Name"])}, ] if project.repository is not None: - properties.append({"name": f"{project.namespace}:sbom:repository", "value": project.repository}) + properties.append({ + "name": sbom_property(project.property_prefix, SBOM_REPOSITORY), + "value": project.repository, + }) metadata["properties"] = properties return metadata diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index f72bef3..8936b61 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -7,54 +7,28 @@ from typing import TYPE_CHECKING, Final from urllib.parse import urlsplit -from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression -from packaging.utils import canonicalize_name +from ._inputs import RecordRow +from ._licenses import licenses +from ._names import project_urls, purl, pypi_url, reference_type, tool_ref +from ._types import Dependency if TYPE_CHECKING: from email.message import Message from importlib.metadata import PackageMetadata from pathlib import PurePosixPath - from ._inputs import BundledWheel, BundledWheelInfo, RecordRow - from ._types import Component, Contact, ExternalReference, Identity, License + from ._inputs import BundledWheel + from ._types import Component, Contact, ExternalReference, Identity, OrganizationalEntity, Property -# the label normalization and mapping cyclonedx-py applies to Project-URL entries, plus "source", so the same metadata -# yields the same reference types whichever generator produced the document -_URL_LABEL_TO_REFERENCE_TYPE: Final[dict[str, str]] = { - "bugtracker": "issue-tracker", - "issuetracker": "issue-tracker", - "issues": "issue-tracker", - "bugreports": "issue-tracker", - "tracker": "issue-tracker", - "home": "website", - "homepage": "website", - "download": "distribution", - "documentation": "documentation", - "docs": "documentation", - "changelog": "release-notes", - "changes": "release-notes", - "source": "vcs", - "repository": "vcs", - "github": "vcs", - "chat": "chat", -} - -def bundled_wheel_component(wheel: BundledWheel, namespace: str, info: BundledWheelInfo) -> Component: - """ - Describe a wheel a project ships as a data file. - - :param wheel: the wheel, as :func:`~python_sbom.read_bundled_wheel` read it - :param namespace: the project's normalized name, which prefixes the properties this package writes - :param info: what the project adds to the component - :returns: the component, with a file component per ``RECORD`` row and a component per vendored package - - """ +def bundled_wheel_component( + wheel: BundledWheel, namespace: str, properties: list[Property], supplier: OrganizationalEntity | None +) -> Component: component = component_from_metadata(wheel.metadata) # the purl names the package; the path tells apart copies of one release bundled in two places component["bom-ref"] = f"bundled:{wheel.path}" - if info.supplier is not None: - component["supplier"] = info.supplier + if supplier is not None: + component["supplier"] = supplier component["hashes"] = [{"alg": "SHA-256", "content": wheel.sha256}] component["externalReferences"] = [ *component["externalReferences"], @@ -62,13 +36,13 @@ def bundled_wheel_component(wheel: BundledWheel, namespace: str, info: BundledWh ] component["properties"] = [ {"name": f"{namespace}:bundled-wheel", "value": wheel.path}, - *info.properties, + *properties, *component["properties"], ] component["evidence"] = { "identity": [ - _identity("purl", "manifest-analysis", wheel.metadata_name), - _identity("hash", "hash-comparison", wheel.sha256), + _evidence("purl", "manifest-analysis", wheel.metadata_name), + _evidence("hash", "hash-comparison", wheel.sha256), ] } vendored: Final[dict[str, Component]] = {} @@ -76,7 +50,7 @@ def bundled_wheel_component(wheel: BundledWheel, namespace: str, info: BundledWh child = component_from_metadata(metadata) child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['bom-ref']}" child["properties"] = [*child["properties"], {"name": f"{namespace}:vendored-manifest", "value": source}] - child["evidence"] = {"identity": [_identity("purl", "manifest-analysis", source)]} + child["evidence"] = {"identity": [_evidence("purl", "manifest-analysis", source)]} # a vendored .dist-info comes after pip's manifest line naming the same package, so its metadata wins vendored[purl(child["name"], child["version"])] = child component["components"] = [ @@ -87,38 +61,22 @@ def bundled_wheel_component(wheel: BundledWheel, namespace: str, info: BundledWh def component_from_metadata(metadata: PackageMetadata | Message) -> Component: - """ - Describe a distribution from its core metadata, as a ``library`` component. - - :param metadata: the distribution's ``METADATA`` - :returns: the component, with its purl as ``bom-ref``, authors, licenses, project URLs and metadata properties - - """ name, version = metadata["Name"], metadata["Version"] component: Final[Component] = {"type": "library", "bom-ref": purl(name, version), "name": name, "version": version} component["purl"] = component["bom-ref"] if summary := metadata.get("Summary"): component["description"] = summary # maintainers are who users reach today, so they win over the original authors when both are named - if people := next( - ( - found - for names, addresses in (("Maintainer", "Maintainer-email"), ("Author", "Author-email")) - if ( - found := [ - contact - for contact in ( - *(_contact(name, "") for name in metadata.get_all(names, [])), - *starmap(_contact, getaddresses(metadata.get_all(addresses, []))), - ) - if contact - ] - ) - ), - None, - ): - component["authors"] = people - if declared := _licenses(metadata): + for names, addresses in (("Maintainer", "Maintainer-email"), ("Author", "Author-email")): + named = (_contact(name, "") for name in metadata.get_all(names, [])) + if people := [ + contact + for contact in (*named, *starmap(_contact, getaddresses(metadata.get_all(addresses, [])))) + if contact + ]: + component["authors"] = people + break + if declared := licenses(metadata): component["licenses"] = declared component["externalReferences"] = _project_references(metadata) # core metadata headers copied verbatim, so a reader sees what the distribution declared @@ -142,47 +100,6 @@ def _contact(name: str, email: str) -> Contact: return contact -def _licenses(metadata: PackageMetadata | Message) -> list[License]: - # a License-Expression is taken as declared; otherwise each License field and License classifier name is - # declared, and the SPDX ids they map to are added as concluded - if expression := metadata.get("License-Expression"): - return [{"expression": expression, "acknowledgement": "declared"}] - names = [ - entry.rsplit(" :: ", 1)[-1] for entry in metadata.get_all("Classifier", []) if entry.startswith("License :: ") - ] - if (declared := metadata.get("License")) and "\n" not in declared: - names.append(declared) - found: Final[list[License]] = [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] - # a license policy judges SPDX ids only, so the names stay bare when any one of them does not resolve, and the - # policy reports that instead of judging a partial set - identifiers = [identifier for name in names if (identifier := _spdx_license(name)) is not None] - if len(identifiers) == len(names): - found.extend( - {"license": {"id": identifier, "acknowledgement": "concluded"}} for identifier in dict.fromkeys(identifiers) - ) - return found - - -def _spdx_license(name: str) -> str | None: - # only the license classifiers that name exactly one SPDX license; "BSD License" or "Apache Software License" leave - # the version or variant open - if identifier := { - "ISC License (ISCL)": "ISC", - "MIT License": "MIT", - "MIT No Attribution License (MIT-0)": "MIT-0", - "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", - "Python Software Foundation License": "PSF-2.0", - "The Unlicense (Unlicense)": "Unlicense", - }.get(name): - return identifier - try: - canonical = canonicalize_license_expression(name) - except InvalidLicenseExpression: - return None - # a CycloneDX license id holds one entry of the SPDX license list, never an expression or a LicenseRef - return canonical if re.fullmatch(r"[A-Za-z0-9.-]+", canonical) and not canonical.startswith("LicenseRef-") else None - - def _project_references(metadata: PackageMetadata | Message) -> list[ExternalReference]: # a home page that is not an http(s) URL, such as UNKNOWN, is left out references: Final[list[ExternalReference]] = [] @@ -195,29 +112,10 @@ def _project_references(metadata: PackageMetadata | Message) -> list[ExternalRef return references -def reference_type(label: str) -> str: - return _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") - - -def project_urls(metadata: PackageMetadata | Message) -> list[tuple[str, str]]: - return [ - (label.strip(), url.strip()) - for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", []) if "," in entry) - ] - - -def pypi_url(name: str, version: str | None = None) -> str: - return f"https://pypi.org/project/{name}/{version}/" if version else f"https://pypi.org/project/{name}/" - - -def _identity(field: str, technique: str, value: str) -> Identity: +def _evidence(field: str, technique: str, value: str) -> Identity: return {"field": field, "confidence": 1, "methods": [{"technique": technique, "confidence": 1, "value": value}]} -def purl(name: str, version: str | None = None) -> str: - return f"pkg:pypi/{canonicalize_name(name)}@{version}" if version else f"pkg:pypi/{canonicalize_name(name)}" - - def file_component(parent_ref: str, row: RecordRow) -> Component: return { "type": "file", @@ -228,36 +126,33 @@ def file_component(parent_ref: str, row: RecordRow) -> Component: } -def file_component_from_bytes(parent_ref: str, name: str, content: bytes) -> Component: - """ - Describe one file from its bytes, such as a member of a zipapp. +def vendored_dependencies(components: list[Component]) -> list[Dependency]: + # each bundled wheel depends on the packages it vendors, wherever it sits in the document + found: Final[list[Dependency]] = [] + pending = list(components) + while pending: + component = pending.pop(0) + children = component.get("components", []) + if component["bom-ref"].startswith("bundled:") and ( + vendored := [ + child["bom-ref"] + for child in children + if child["type"] == "library" and "#vendored/" in child["bom-ref"] + ] + ): + found.append(Dependency(ref=component["bom-ref"], dependsOn=vendored)) + pending.extend(children) + return found - :param parent_ref: the ``bom-ref`` of the component that holds the file - :param name: the file's path - :param content: the file's bytes - :returns: a ``file`` component with the file's SHA-256 and size - """ - return { - "type": "file", - "bom-ref": f"{parent_ref}#{name}", - "name": name, - "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(content).hexdigest()}], - "properties": [{"name": "size", "value": str(len(content))}], - } +def file_component_from_bytes(parent_ref: str, name: str, content: bytes) -> Component: + return file_component( + parent_ref, + RecordRow(path=name, algorithm="SHA-256", digest=hashlib.sha256(content).hexdigest(), size=str(len(content))), + ) def script_tool(path: PurePosixPath, sha256: str, version: str, link: str | None) -> Component: - """ - Describe a build script as a build tool, since a rebuild has to match the script that shaped the document. - - :param path: the script's path relative to the project root - :param sha256: the SHA-256 of the script - :param version: the version of the artifact the script builds - :param link: a URL of the script in the repository, or ``None`` - :returns: an ``application`` component - - """ tool: Final[Component] = { "type": "application", "bom-ref": tool_ref(str(path)), @@ -270,25 +165,11 @@ def script_tool(path: PurePosixPath, sha256: str, version: str, link: str | None return tool -def tool_ref(ref: str) -> str: - # prefixed because the same distribution can be both a build tool and a component of the artifact - return f"tool:{ref}" - - -def sbom_file_name(name: str) -> str: - return f"{canonicalize_name(name)}.cdx.json" - - __all__ = [ "bundled_wheel_component", "component_from_metadata", "file_component", "file_component_from_bytes", - "project_urls", - "purl", - "pypi_url", - "reference_type", - "sbom_file_name", "script_tool", - "tool_ref", + "vendored_dependencies", ] diff --git a/src/python_sbom/_model/_content.py b/src/python_sbom/_model/_content.py new file mode 100644 index 0000000..d712d97 --- /dev/null +++ b/src/python_sbom/_model/_content.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import zipfile +from typing import TYPE_CHECKING + +from ._inputs import BundledWheelInfo, Contributions + +if TYPE_CHECKING: + from pathlib import Path, PurePosixPath + + from ._identity import ProjectIdentity + from ._inputs import Settings + + +class ProviderContext: + """ + What a content provider may use about the build it contributes to. + + :param root: the project directory + :param settings: the project's options + :param identity: how the project's SBOMs name it, including the prefix of its properties + :param artifact: the built wheel, or the directory an archive is unpacked into, which :meth:`read` reads + :ivar root: the project directory + :ivar settings: the project's options + :ivar identity: how the project's SBOMs name it, including the prefix of its properties + + """ + + def __init__(self, *, root: Path, settings: Settings, identity: ProjectIdentity, artifact: Path) -> None: + self.root = root + self.settings = settings + self.identity = identity + self._artifact = artifact + + def read(self, path: PurePosixPath) -> bytes: + """ + Read a file of the artifact being described: a member of the built wheel or of the archive. + + :param path: the file's path inside the artifact, as a bundled wheel's path is given + :returns: the file's bytes + :raises KeyError: when the artifact holds no such file + + """ + if self._artifact.is_dir(): + if not (target := self._artifact / str(path)).is_file(): + raise KeyError(str(path)) + return target.read_bytes() + with zipfile.ZipFile(self._artifact) as archive: + return archive.read(str(path)) + + +class ContentProvider: + """ + Adds what a project knows beyond its metadata to its SBOM; subclass it and override what the project needs. + + Name a subclass in the ``content-provider`` option, which the hook, ``python-sbom embed`` and ``python-sbom + archive`` load, or return an instance from :meth:`python_sbom.hatch.SbomBuildHook.content_provider`. When the class + lives in the project, the SBOM lists its module among the build tools. + + """ + + def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: # ruff: ignore[unused-method-argument, no-self-use] # the no-op default + """ + Describe what the project does with a wheel it bundles, as a file or unpacked into a directory. + + :param context: the project and the artifact being described + :param wheel: the wheel's path inside the artifact + :returns: its extra properties, and whether the project's supplier supplies it; neither by default + + """ + return BundledWheelInfo() + + def contributions(self, context: ProviderContext) -> Contributions: # ruff: ignore[unused-method-argument, no-self-use] # the no-op default + """ + Add components, their dependencies, root properties and build tools, before references and the serial derive. + + :param context: the project and the artifact being described + :returns: the contributions; none by default + + """ + return Contributions() + + +__all__ = [ + "ContentProvider", + "ProviderContext", +] diff --git a/src/python_sbom/_model/_document.py b/src/python_sbom/_model/_document.py index 0de9c39..ced9142 100644 --- a/src/python_sbom/_model/_document.py +++ b/src/python_sbom/_model/_document.py @@ -2,67 +2,36 @@ import json import uuid -from dataclasses import dataclass +from collections import Counter from datetime import UTC, datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final -from packaging.utils import canonicalize_name - -from ._components import project_urls, reference_type +from python_sbom._errors import SbomError if TYPE_CHECKING: - from email.message import Message - - from ._inputs import BuildFacts, Settings - from ._types import Body, Component, Document, Formula - - -def project_identity(metadata: Message, settings: Settings) -> ProjectIdentity: - """ - Name a project the way its documents do, so other documents about it, such as a zipapp's, agree. - - :param metadata: the project's core metadata - :param settings: the project's options, whose ``repository`` wins over the project URLs - :returns: the identity - - """ - namespace = canonicalize_name(metadata["Name"]) - repository = settings.repository or next( - (url.rstrip("/") for label, url in project_urls(metadata) if reference_type(label) == "vcs"), None - ) - base = repository or f"https://pypi.org/project/{namespace}" - return ProjectIdentity( - namespace=namespace, - repository=repository, - serial_namespace=uuid.uuid5(uuid.NAMESPACE_URL, f"{base}/sboms"), - ) - - -@dataclass(frozen=True, slots=True, kw_only=True) -class ProjectIdentity: - """ - How documents name a project. - - :param namespace: the normalized project name, which prefixes the properties this package writes - :param repository: the repository URL, or ``None`` - :param serial_namespace: the UUID namespace of the serial numbers of the project's documents - - """ - - namespace: str - repository: str | None - serial_namespace: uuid.UUID - - -def document(body: Body, namespace: uuid.UUID) -> Document: - """ - Add the CycloneDX 1.6 header to a document body. - - :param body: the metadata, components, dependencies and formulation - :param namespace: the UUID namespace of the serial number, see :func:`project_identity` - :returns: the document; its serial number derives from the body, so identical bodies share one - - """ + from ._inputs import BuildFacts + from ._types import Body, Component, Document, Formula, Property + + +def with_header(body: Body, namespace: uuid.UUID) -> Document: + # a provider contributes refs of its own, so the document is checked before it is written rather than by validate + refs: Final[Counter[str]] = Counter() + pending: Final[list[Component]] = [ + body["metadata"]["component"], + *body["components"], + *body["metadata"]["tools"]["components"], + ] + while pending: + component = pending.pop() + refs[component["bom-ref"]] += 1 + pending.extend(component.get("components", [])) + if repeated := sorted(ref for ref, count in refs.items() if count > 1): + msg = f"bom-ref values repeat: {', '.join(repeated)}" + raise SbomError(msg) + named = {ref for entry in body["dependencies"] for ref in (entry["ref"], *entry.get("dependsOn", []))} + if unknown := sorted(named - refs.keys()): + msg = f"dependencies name bom-refs no component has: {', '.join(unknown)}" + raise SbomError(msg) return { "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", "bomFormat": "CycloneDX", @@ -73,19 +42,15 @@ def document(body: Body, namespace: uuid.UUID) -> Document: } -def formulation(uid: str, name: str, tools: list[Component], output_ref: str, facts: BuildFacts) -> Formula: - """ - Record how an artifact was built: one workflow with its tools, its output and the build time it read. +def contributed_properties(properties: list[Property], prefix: str) -> list[Property]: + if claimed := sorted(prop["name"] for prop in properties if prop["name"].startswith(f"{prefix}:")): + msg = f"contributed root properties {', '.join(claimed)} use the {prefix}: prefix python-sbom writes" + raise SbomError(msg) + return properties - :param uid: the workflow's identifier, such as ``wheel-build`` - :param name: what the workflow does - :param tools: the build tools it used - :param output_ref: the ``bom-ref`` of the artifact it produced - :param facts: the build facts; ``SOURCE_DATE_EPOCH`` is listed as an input only when it set the build time - :returns: the formula - """ - workflow: Formula = { +def formulation(uid: str, name: str, tools: list[Component], output_ref: str, facts: BuildFacts) -> Formula: + formula: Formula = { "bom-ref": f"formula:{uid}", "workflows": [ { @@ -99,10 +64,10 @@ def formulation(uid: str, name: str, tools: list[Component], output_ref: str, fa ], } if facts.epoch_from_environment: - workflow["workflows"][0]["inputs"] = [ + formula["workflows"][0]["inputs"] = [ {"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": str(facts.epoch)}]} ] - return workflow + return formula def timestamp(facts: BuildFacts) -> str: @@ -110,9 +75,8 @@ def timestamp(facts: BuildFacts) -> str: __all__ = [ - "ProjectIdentity", - "document", + "contributed_properties", "formulation", - "project_identity", "timestamp", + "with_header", ] diff --git a/src/python_sbom/_model/_identity.py b/src/python_sbom/_model/_identity.py new file mode 100644 index 0000000..064d292 --- /dev/null +++ b/src/python_sbom/_model/_identity.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from ._names import project_urls, property_prefix, reference_type + +if TYPE_CHECKING: + from email.message import Message + + from ._inputs import Settings + + +def project_identity(metadata: Message, settings: Settings) -> ProjectIdentity: + project = property_prefix(metadata["Name"]) + repository = settings.repository or next( + (url.rstrip("/") for label, url in project_urls(metadata) if reference_type(label) == "vcs"), None + ) + base = repository or f"https://pypi.org/project/{project}" + return ProjectIdentity( + property_prefix=project, + repository=repository, + serial_namespace=uuid.uuid5(uuid.NAMESPACE_URL, f"{base}/sboms"), + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ProjectIdentity: + """ + How every SBOM of a project names it. + + :param property_prefix: the project's normalized name, which starts the names of the properties python-sbom writes, + such as ``virtualenv`` in ``virtualenv:bundled-wheel`` + :param repository: the repository URL, from the options or the first ``Source``, ``Repository`` or ``GitHub`` + project URL, or ``None`` + :param serial_namespace: the UUID namespace of the serial numbers of the project's SBOMs + + """ + + property_prefix: str + repository: str | None + serial_namespace: uuid.UUID + + +__all__ = [ + "ProjectIdentity", + "project_identity", +] diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py index b47190d..9b10f4f 100644 --- a/src/python_sbom/_model/_inputs.py +++ b/src/python_sbom/_model/_inputs.py @@ -1,10 +1,11 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING if TYPE_CHECKING: from email.message import Message + from importlib.metadata import PackageMetadata from pathlib import Path, PurePosixPath from ._types import Component, Dependency, OrganizationalEntity, Property @@ -12,16 +13,6 @@ @dataclass(frozen=True, slots=True, kw_only=True) class RecordRow: - """ - One file of a wheel, as its ``RECORD`` lists it. - - :param path: the file's path inside the wheel - :param algorithm: the hash algorithm as CycloneDX names it, such as ``SHA-256`` - :param digest: the file's digest as lowercase hex - :param size: the file's size in bytes, as ``RECORD`` writes it - - """ - path: str algorithm: str digest: str @@ -30,15 +21,6 @@ class RecordRow: @dataclass(frozen=True, slots=True, kw_only=True) class Interpreter: - """ - The Python that runs the build. - - :param name: the implementation name, such as ``cpython`` - :param version: the full version, such as ``3.15.0`` - :param implementation: the implementation as :func:`platform.python_implementation` names it, such as ``CPython`` - - """ - name: str version: str implementation: str @@ -46,47 +28,20 @@ class Interpreter: @dataclass(frozen=True, slots=True, kw_only=True) class InstalledDistribution: - """ - One distribution of the build environment. - - :param metadata: its core metadata - :param files: its files, for a pure-Python distribution; empty for a platform one, whose files differ per machine - :param depends_on: the normalized names of the installed distributions it requires in this environment - - """ - - metadata: Message + name: str + metadata: PackageMetadata | Message files: list[RecordRow] depends_on: list[str] @dataclass(frozen=True, slots=True, kw_only=True) class Revision: - """ - The source revision a build comes from. - - :param commit: the commit identifier - :param dirty: whether tracked files differed from the commit when the build ran - - """ - commit: str dirty: bool @dataclass(frozen=True, slots=True, kw_only=True) class BuildFacts: - """ - What the environment says about the build, collected once per build at the edge. - - :param epoch: the build time as a Unix timestamp - :param epoch_from_environment: whether ``SOURCE_DATE_EPOCH`` set the build time - :param revision: the source revision, or ``None`` when it is unknown - :param interpreter: the Python that runs the build - :param distributions: every distribution of the build environment - - """ - epoch: int epoch_from_environment: bool revision: Revision | None @@ -97,14 +52,14 @@ class BuildFacts: @dataclass(frozen=True, slots=True, kw_only=True) class Settings: """ - The options a project sets, in a hatchling hook's table or in ``[tool.python-sbom]``. + The project's options, from ``[tool.python-sbom]``. :param supplier: the organization that supplies the project, or ``None`` to name none :param repository: the repository URL, or ``None`` to take the first ``Source``, ``Repository`` or ``GitHub`` project URL :param default_branch: the branch in links to files in the repository - :param bundled_wheels: glob patterns, relative to the project root, of wheels the project ships as data files - :param content_provider: ``module:attribute`` of a content provider importable from the project root, or ``None`` + :param bundled_wheels: glob patterns, relative to the project directory, of wheels the project ships as data files + :param content_provider: ``module:attribute`` of the content provider, or ``None`` """ @@ -117,17 +72,6 @@ class Settings: @dataclass(frozen=True, slots=True, kw_only=True) class SbomRequest: - """ - What a backend knows about the wheel it builds or rewrites. - - :param root: the project directory - :param metadata: the wheel's core metadata - :param wheel_name: the wheel's file name, for its attestation link - :param settings: the project's options - :param build_scripts: scripts, relative to ``root``, that shape the document, listed among the build tools - - """ - root: Path metadata: Message wheel_name: str @@ -137,18 +81,6 @@ class SbomRequest: @dataclass(frozen=True, slots=True, kw_only=True) class BundledWheel: - """ - A wheel the project ships as a data file. - - :param path: its path relative to the project root - :param sha256: the SHA-256 of its bytes, as lowercase hex - :param metadata_name: the path of its ``METADATA`` inside it - :param metadata: its core metadata - :param files: its files, from its ``RECORD`` - :param vendored: the packages it vendors, each with the file that names it - - """ - path: str sha256: str metadata_name: str @@ -159,29 +91,12 @@ class BundledWheel: @dataclass(frozen=True, slots=True, kw_only=True) class BuildScript: - """ - A build script that shapes the document, so a rebuild has to match it. - - :param path: its path relative to the project root - :param sha256: the SHA-256 of its bytes, as lowercase hex - - """ - path: PurePosixPath sha256: str @dataclass(frozen=True, slots=True, kw_only=True) class SourceTree: - """ - What the document takes from the project tree besides the wheel's metadata. - - :param copyright: the first ``Copyright`` line of the license files, or ``None`` - :param bundled: the wheels the project ships as data files - :param build_scripts: the scripts that shape the document - - """ - copyright: str | None bundled: list[BundledWheel] build_scripts: list[BuildScript] @@ -192,13 +107,16 @@ class BundledWheelInfo: """ What a project adds to the component of one bundled wheel. - :param properties: properties after ``:bundled-wheel``, before those from the wheel's metadata - :param supplier: the wheel's supplier, or ``None`` to name none; only the project decides it, never the wheel + Both fields default to empty, so a provider names only what it adds. + + :param properties: properties after ``:bundled-wheel``, before those from the wheel's metadata + :param supplier: the wheel's supplier, or ``None`` to name none; only the project decides a bundled wheel's + supplier, never the wheel's own metadata """ - properties: list[Property] - supplier: OrganizationalEntity | None + properties: list[Property] = field(default_factory=list) + supplier: OrganizationalEntity | None = None @dataclass(frozen=True, slots=True, kw_only=True) @@ -206,17 +124,22 @@ class Contributions: """ What a project adds to the document beyond its bundled wheels, merged before references and the serial are derived. - :param components: extra top-level components, such as vendored C libraries; the root depends on each + Each field defaults to empty, so a provider names only what it adds. The build fails when a ``bom-ref`` repeats or + a dependency names one that does not exist. + + :param components: extra top-level components, such as vendored C libraries; the root depends on each, and a + component's own parts nest under its ``components`` :param dependencies: extra dependency entries, such as the dependencies of those components - :param root_properties: extra properties of the root component + :param root_properties: extra properties of the root component; their names must not start with the project's + prefix, which python-sbom's own properties use :param tools: extra build tools """ - components: list[Component] - dependencies: list[Dependency] - root_properties: list[Property] - tools: list[Component] + components: list[Component] = field(default_factory=list) + dependencies: list[Dependency] = field(default_factory=list) + root_properties: list[Property] = field(default_factory=list) + tools: list[Component] = field(default_factory=list) __all__ = [ diff --git a/src/python_sbom/_model/_licenses.py b/src/python_sbom/_model/_licenses.py new file mode 100644 index 0000000..22877d8 --- /dev/null +++ b/src/python_sbom/_model/_licenses.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Final + +from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression + +if TYPE_CHECKING: + from email.message import Message + from importlib.metadata import PackageMetadata + + from ._types import License + + +def licenses(metadata: PackageMetadata | Message) -> list[License]: + # a License-Expression is taken as declared; otherwise each License field and License classifier name is + # declared, and the SPDX ids they map to are added as concluded + if expression := metadata.get("License-Expression"): + try: + return [{"expression": canonicalize_license_expression(expression), "acknowledgement": "declared"}] + except InvalidLicenseExpression: + # an expression SPDX rejects, such as "MIT WITH GPL-3.0-only", stays as the name the metadata gives + return [{"license": {"name": expression, "acknowledgement": "declared"}}] + names = [ + entry.rsplit(" :: ", 1)[-1] for entry in metadata.get_all("Classifier", []) if entry.startswith("License :: ") + ] + if (declared := metadata.get("License")) and "\n" not in declared: + names.append(declared) + found: Final[list[License]] = [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] + # a license policy judges SPDX ids only, so the names stay bare when any one of them does not resolve, and the + # policy reports that instead of judging a partial set + identifiers = [identifier for name in names if (identifier := _spdx_license(name)) is not None] + if len(identifiers) == len(names): + found.extend( + {"license": {"id": identifier, "acknowledgement": "concluded"}} for identifier in dict.fromkeys(identifiers) + ) + return found + + +def _spdx_license(name: str) -> str | None: + # only the license classifiers that name exactly one SPDX license; "BSD License" or "Apache Software License" leave + # the version or variant open + if identifier := { + "ISC License (ISCL)": "ISC", + "MIT License": "MIT", + "MIT No Attribution License (MIT-0)": "MIT-0", + "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", + "Python Software Foundation License": "PSF-2.0", + "The Unlicense (Unlicense)": "Unlicense", + }.get(name): + return identifier + try: + canonical = canonicalize_license_expression(name) + except InvalidLicenseExpression: + return None + # a CycloneDX license id holds one entry of the SPDX license list, never an expression or a LicenseRef + return canonical if re.fullmatch(r"[A-Za-z0-9.-]+", canonical) and not canonical.startswith("LicenseRef-") else None + + +__all__ = [ + "licenses", +] diff --git a/src/python_sbom/_model/_markers.py b/src/python_sbom/_model/_markers.py index fd09dde..0c8f8fc 100644 --- a/src/python_sbom/_model/_markers.py +++ b/src/python_sbom/_model/_markers.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final from packaging._parser import Op, Value, Variable # ruff: ignore[import-private-name] # packaging exposes the parsed marker tree only privately from packaging.markers import Marker @@ -34,8 +34,9 @@ def _extra_verdict(markers: MarkerList | MarkerAtom) -> bool | None: groups[-1].append(Marker(f"{value!r} {operator} extra").evaluate({"extra": ""})) case _: groups[-1].append(None) - verdicts = [False if False in group else (None if None in group else True) for group in groups] - return True if True in verdicts else (None if None in verdicts else False) + # ranked False < unknown < True: "and" takes the lowest verdict of a group, "or" the highest of the groups + verdicts: Final[tuple[bool | None, ...]] = (False, None, True) + return verdicts[max(min(map(verdicts.index, group), default=2) for group in groups)] __all__ = [ diff --git a/src/python_sbom/_model/_names.py b/src/python_sbom/_model/_names.py new file mode 100644 index 0000000..f740b0c --- /dev/null +++ b/src/python_sbom/_model/_names.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Final + +from packaging.utils import canonicalize_name + +if TYPE_CHECKING: + from email.message import Message + from importlib.metadata import PackageMetadata + +#: the metadata property in which a document names its own file; validation checks such documents in full +SBOM_FILE: Final[str] = "sbom:file" +#: the metadata property in which a document names the project's repository +SBOM_REPOSITORY: Final[str] = "sbom:repository" +# the label normalization and mapping cyclonedx-py applies to Project-URL entries, so the same metadata yields the same +# reference types whichever generator produced the document, plus the well-known labels of PEP 753 it lacks; funding +# labels have no CycloneDX type and fall to "other" +_URL_LABEL_TO_REFERENCE_TYPE: Final[dict[str, str]] = { + "bugtracker": "issue-tracker", + "issuetracker": "issue-tracker", + "issues": "issue-tracker", + "issue": "issue-tracker", + "bugs": "issue-tracker", + "bugreports": "issue-tracker", + "tracker": "issue-tracker", + "home": "website", + "homepage": "website", + "download": "distribution", + "documentation": "documentation", + "docs": "documentation", + "changelog": "release-notes", + "changes": "release-notes", + "whatsnew": "release-notes", + "history": "release-notes", + "releasenotes": "release-notes", + "source": "vcs", + "sourcecode": "vcs", + "repository": "vcs", + "github": "vcs", + "chat": "chat", +} + + +def sbom_property(project: str, key: str) -> str: + return f"{project}:{key}" + + +def sbom_file_name(name: str) -> str: + return f"{property_prefix(name)}.cdx.json" + + +def property_prefix(name: str) -> str: + # the normalized project name, which prefixes the properties this package writes + return canonicalize_name(name) + + +def purl(name: str, version: str | None = None) -> str: + return f"pkg:pypi/{property_prefix(name)}@{version}" if version else f"pkg:pypi/{property_prefix(name)}" + + +def pypi_url(name: str, version: str | None = None) -> str: + return f"https://pypi.org/project/{name}/{version}/" if version else f"https://pypi.org/project/{name}/" + + +def tool_ref(ref: str) -> str: + # prefixed because the same distribution can be both a build tool and a component of the artifact + return f"tool:{ref}" + + +def reference_type(label: str) -> str: + return _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") + + +def project_urls(metadata: PackageMetadata | Message) -> list[tuple[str, str]]: + return [ + (label.strip(), url.strip()) + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", []) if "," in entry) + ] + + +__all__ = [ + "SBOM_FILE", + "SBOM_REPOSITORY", + "project_urls", + "property_prefix", + "purl", + "pypi_url", + "reference_type", + "sbom_file_name", + "sbom_property", + "tool_ref", +] diff --git a/src/python_sbom/_model/_tools.py b/src/python_sbom/_model/_tools.py index beb75ea..b10759a 100644 --- a/src/python_sbom/_model/_tools.py +++ b/src/python_sbom/_model/_tools.py @@ -2,9 +2,8 @@ from typing import TYPE_CHECKING, Final -from packaging.utils import canonicalize_name - -from ._components import component_from_metadata, file_component, purl, tool_ref +from ._components import component_from_metadata, file_component +from ._names import purl, tool_ref if TYPE_CHECKING: from ._inputs import BuildFacts @@ -12,13 +11,6 @@ def tool_components(facts: BuildFacts) -> tuple[list[Component], list[Dependency]]: - """ - Describe the build environment as build tools: the interpreter first, then every distribution by name. - - :param facts: the collected build facts - :returns: the tool components and the dependency entries between the distributions - - """ interpreter = facts.interpreter package = f"pkg:generic/{interpreter.name}@{interpreter.version}" tools: Final[list[Component]] = [ @@ -32,15 +24,13 @@ def tool_components(facts: BuildFacts) -> tuple[list[Component], list[Dependency } ] refs: Final[dict[str, str]] = { - canonicalize_name(distribution.metadata["Name"]): tool_ref( - purl(distribution.metadata["Name"], distribution.metadata["Version"]) - ) + distribution.name: tool_ref(purl(distribution.name, distribution.metadata["Version"])) for distribution in facts.distributions } dependencies: Final[list[Dependency]] = [] - for distribution in sorted(facts.distributions, key=lambda entry: canonicalize_name(entry.metadata["Name"])): + for distribution in sorted(facts.distributions, key=lambda entry: entry.name): component = component_from_metadata(distribution.metadata) - component["bom-ref"] = refs[canonicalize_name(distribution.metadata["Name"])] + component["bom-ref"] = refs[distribution.name] if distribution.files: component["components"] = [file_component(component["bom-ref"], row) for row in distribution.files] tools.append(component) diff --git a/src/python_sbom/_model/_types.py b/src/python_sbom/_model/_types.py index 8d68600..0fcdfd5 100644 --- a/src/python_sbom/_model/_types.py +++ b/src/python_sbom/_model/_types.py @@ -17,11 +17,27 @@ class OrganizationalEntity(TypedDict): class Contact(TypedDict): + """ + A person who wrote or maintains a component. + + :ivar name: the person's name + :ivar email: the person's email address + + """ + name: NotRequired[str] email: NotRequired[str] -class _Hash(TypedDict): +class Hash(TypedDict): + """ + A digest of a component's bytes. + + :ivar alg: the algorithm as CycloneDX names it, such as ``SHA-256`` + :ivar content: the digest as lowercase hex + + """ + alg: str content: str @@ -30,7 +46,7 @@ class Property(TypedDict): """ A name and value pair attached to a component or the document. - :ivar name: the property's name; names this package writes start with the project name or ``python:`` + :ivar name: the property's name; names this package writes start with the project's prefix or ``python:`` :ivar value: the property's value """ @@ -40,12 +56,30 @@ class Property(TypedDict): class ExternalReference(TypedDict): + """ + A link from a component to something about it. + + :ivar type: the CycloneDX reference type, such as ``vcs`` or ``website`` + :ivar url: the link + :ivar comment: what the link is, such as the Project-URL label it came from + + """ + type: str url: str comment: NotRequired[str] class LicenseInfo(TypedDict): + """ + One license, by SPDX id or by name. + + :ivar id: the SPDX license id + :ivar name: the license's name, when it has no SPDX id + :ivar acknowledgement: ``declared`` when the metadata names it, ``concluded`` when python-sbom resolved it + + """ + id: NotRequired[str] name: NotRequired[str] acknowledgement: str @@ -53,9 +87,9 @@ class LicenseInfo(TypedDict): class License(TypedDict): """ - One license entry: either a license by SPDX id or name, or an SPDX expression. + One license entry of a component: a license or an SPDX expression. - :ivar license: the license, by ``id`` or ``name`` + :ivar license: the license, by SPDX id or by name :ivar expression: an SPDX license expression :ivar acknowledgement: ``declared`` or ``concluded``, for an expression @@ -66,23 +100,51 @@ class License(TypedDict): acknowledgement: NotRequired[str] -class _Method(TypedDict): +class IdentityMethod(TypedDict): + """ + How a component's identity was established. + + :ivar technique: the CycloneDX technique, such as ``manifest-analysis`` or ``hash-comparison`` + :ivar confidence: from 0 to 1 + :ivar value: what the technique read, such as the path of a ``METADATA`` file or a digest + + """ + technique: str confidence: int value: str class Identity(TypedDict): + """ + The evidence for one field of a component's identity. + + :ivar field: the field it supports, such as ``purl`` or ``hash`` + :ivar confidence: from 0 to 1 + :ivar methods: how it was established + + """ + field: str confidence: int - methods: list[_Method] + methods: list[IdentityMethod] + + +class Evidence(TypedDict): + """ + Why a component is what it says it is. + + :ivar identity: the evidence per field + """ -class _Evidence(TypedDict): identity: list[Identity] -#: A CycloneDX component: a distribution, an application, the interpreter or a file. +#: A CycloneDX 1.6 component: a distribution, an application, the interpreter or a file, as a dictionary with the +#: keys the CycloneDX component object defines, of which python-sbom uses ``type``, ``bom-ref``, ``name``, +#: ``version``, ``purl``, ``supplier``, ``authors``, ``description``, ``licenses``, ``copyright``, ``hashes``, +#: ``externalReferences``, ``properties``, ``evidence`` and the nested ``components``. Component = TypedDict( "Component", { @@ -96,10 +158,10 @@ class _Evidence(TypedDict): "description": NotRequired[str], "licenses": NotRequired[list[License]], "copyright": NotRequired[str], - "hashes": NotRequired[list[_Hash]], + "hashes": NotRequired[list[Hash]], "externalReferences": NotRequired[list[ExternalReference]], "properties": NotRequired[list[Property]], - "evidence": NotRequired[_Evidence], + "evidence": NotRequired[Evidence], "components": NotRequired[list["Component"]], }, ) @@ -175,17 +237,6 @@ class Composition(TypedDict): class Body(TypedDict): - """ - The content of a document, everything but the header :func:`~python_sbom.document` adds. - - :ivar metadata: the document's metadata, with the described component at ``component`` - :ivar components: the components the described component holds or depends on - :ivar dependencies: the dependency graph, by ``bom-ref`` - :ivar compositions: how complete the inventory is - :ivar formulation: how the described component was built - - """ - metadata: Metadata components: list[Component] dependencies: list[Dependency] @@ -193,24 +244,16 @@ class Body(TypedDict): formulation: NotRequired[list[Formula]] -#: A complete CycloneDX 1.6 JSON document. -Document = TypedDict( - "Document", - { - "$schema": str, - "bomFormat": str, - "specVersion": str, - "serialNumber": str, - "version": int, - "metadata": Metadata, - "components": list[Component], - "dependencies": list[Dependency], - "compositions": NotRequired[list[Composition]], - "formulation": NotRequired[list[Formula]], - }, +_Header = TypedDict( + "_Header", + {"$schema": str, "bomFormat": str, "specVersion": str, "serialNumber": str, "version": int}, ) +class Document(Body, _Header): + pass + + __all__ = [ "Body", "Component", @@ -218,9 +261,12 @@ class Body(TypedDict): "Contact", "Dependency", "Document", + "Evidence", "ExternalReference", "Formula", + "Hash", "Identity", + "IdentityMethod", "License", "LicenseInfo", "Lifecycle", diff --git a/src/python_sbom/_validate/_artifact.py b/src/python_sbom/_validate/_artifact.py new file mode 100644 index 0000000..cf77593 --- /dev/null +++ b/src/python_sbom/_validate/_artifact.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import zipfile +from typing import TYPE_CHECKING, Final + +from python_sbom._wheel._archive import open_archive, read_member + +from ._document import validate_document + +if TYPE_CHECKING: + from pathlib import Path + +# an SBOM lists every file of its wheel at a few hundred bytes each, so even a wheel with a hundred thousand files stays +# well under this, while one archive member can make validation hold no more than this in memory +_SBOM_LIMIT: Final[int] = 64 * 1024 * 1024 + + +def validate(path: Path, allowed_licenses: frozenset[str] | None) -> list[str]: + if not zipfile.is_zipfile(path): + return validate_document(path.read_text(encoding="utf-8"), allowed_licenses) + with open_archive(path, path.name) as archive: + found: Final[dict[str, str]] = { + name: read_member(archive, name, path.name, _SBOM_LIMIT).decode("utf-8") + for name in sorted(archive.namelist()) + if _is_sbom(name) + } + if not found: + return [f"no SBOM found in {path.name}"] + return [f"{name}: {problem}" for name, raw in found.items() for problem in validate_document(raw, allowed_licenses)] + + +def _is_sbom(name: str) -> bool: + # every top-level .dist-info counts, so a decoy one sorting first cannot hide the SBOM of another + match name.split("/"): + case [file]: + return file.endswith(".cdx.json") + case [dist_info, "sboms", file]: + return dist_info.endswith(".dist-info") and file.endswith(".cdx.json") + case _: + return False + + +__all__ = [ + "validate", +] diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index da78693..d9d926c 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -5,17 +5,21 @@ from collections import Counter from typing import TYPE_CHECKING, Final, TypeAlias -from python_sbom._formats._spdx import to_spdx +from python_sbom._errors import MissingExtraError +from python_sbom._formats._spdx import render_spdx +from python_sbom._model._names import SBOM_FILE from python_sbom.version import __version__ from ._licenses import license_problems if TYPE_CHECKING: + from typing import TypeIs # new in 3.13; annotations here are never evaluated + from python_sbom._model._types import Component, Document # a JSON value, as json.loads hands it over _Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" -# the schema leaves these optional, yet the checks below and the SPDX rendering read them +# the schema leaves these optional, yet every document python-sbom writes carries them and the checks below read them _REQUIRED: Final[tuple[tuple[str, ...], ...]] = ( ("metadata", "timestamp"), ("metadata", "tools", "components"), @@ -24,50 +28,86 @@ ("components",), ("dependencies",), ) - - -def validate(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: - """ - Check one CycloneDX SBOM; needs the ``validate`` extra. - - Every document must conform to the CycloneDX 1.6 schema. A document python-sbom wrote must also carry the - ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, unique and resolvable ``bom-ref`` - values, a root that depends on exactly the top-level components, an SPDX 2.3 rendering that validates, and, with - ``allowed_licenses``, licenses the policy allows; documents other tools wrote get the schema check alone. - - :param raw: the JSON text of the document - :param allowed_licenses: SPDX license ids the shipped components may use, or ``None`` to skip the license policy - :returns: one message per problem, empty when the document is valid - :raises MissingExtraError: when the ``validate`` extra is not installed - - """ +_SERIAL: Final[re.Pattern[str]] = re.compile(r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +# the archive reader stops at the same size +_MAX_CHARACTERS: Final[int] = 16 * 1024 * 1024 +# the schema's uniqueItems compares every pair of items, so an array this long already takes half a minute +_MAX_ITEMS: Final[int] = 10_000 +# an exception's message can quote the hostile value that caused it +_MAX_TEXT: Final[int] = 200 + + +def validate_document(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: + if len(raw) > _MAX_CHARACTERS: + return [f"is {len(raw)} characters long, over the {_MAX_CHARACTERS} character limit"] + # one document's problem never stops the others from being checked, so whatever the checks raise is reported + try: + return _problems(raw, allowed_licenses) + except MissingExtraError: + raise + except json.JSONDecodeError as error: + return [f"is not JSON: {error}"] + except RecursionError: + # the JSON parser and the schema validator recurse once per level, how deep depends on the Python version + return ["does not conform to the CycloneDX 1.6 schema: it nests too deeply to check"] + except Exception as error: # ruff: ignore[blind-except] # a hostile document may trip any check, see above + return [f"could not be checked, {type(error).__name__}: {str(error)[:_MAX_TEXT]}"] + + +def _problems(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: + # every document must conform to the CycloneDX 1.6 schema; one python-sbom wrote must also carry the urn:uuid serial + # actions/attest needs, resolvable and unique bom-refs and a valid SPDX rendering, and meet the license policy from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # the optional extra loads on first use + loaded: Final = json.loads(raw) + if census := _census_problems(loaded): + return census if error := schema_error(raw): return [f"does not conform to the CycloneDX 1.6 schema: {error}"] - loaded = json.loads(raw) # python-sbom names its file in a metadata property, so documents other tools wrote get the schema check alone; the # schema already guarantees the shape of what this reads if not any( - prop.get("name", "").endswith(":sbom:file") for prop in loaded.get("metadata", {}).get("properties", []) + prop.get("name", "").endswith(f":{SBOM_FILE}") for prop in loaded.get("metadata", {}).get("properties", []) ): return [] - if missing := [".".join(path) for path in _REQUIRED if not _has(loaded, path)]: + missing: Final[list[str]] = [".".join(path) for path in _REQUIRED if not _has(loaded, path)] + # is_document holds once nothing is missing; asking it narrows the loaded JSON to a document + if missing or not is_document(loaded): return [f"lacks {', '.join(missing)}, which every document python-sbom writes carries"] # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize the document # as CycloneDX at all, and silently rejects anything missing it as an unknown format - if not re.fullmatch( - r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", - serial := loaded.get("serialNumber", ""), - ): + if not _SERIAL.fullmatch(serial := loaded.get("serialNumber", "")): return [f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {serial!r}"] # the SPDX rendering of a document whose references are broken only repeats those problems - problems = _reference_problems(loaded) or spdx_problems(to_spdx(loaded, __version__)) + problems: Final[list[str]] = _reference_problems(loaded) or spdx_problems(render_spdx(loaded, __version__)) if allowed_licenses is not None: problems.extend(license_problems(loaded["components"], allowed_licenses)) return problems +def _census_problems(document: _Json) -> list[str]: + # counts the longest components array and every bom-ref before the schema vouches for any shape; iterative, so + # deep nesting cannot exhaust the stack + largest = 0 + refs: Final[Counter[str]] = Counter() + pending: Final[list[_Json]] = [document] + while pending: + node = pending.pop() + if isinstance(node, list): + pending.extend(node) + elif isinstance(node, dict): + if isinstance(ref := node.get("bom-ref"), str): + refs[ref] += 1 + if isinstance(components := node.get("components"), list): + largest = max(largest, len(components)) + pending.extend(node.values()) + if largest > _MAX_ITEMS: + return [f"lists {largest} components in one array, over the {_MAX_ITEMS} the schema check can take"] + if duplicates := sorted(ref for ref, count in refs.items() if count > 1): + return [f"bom-ref values are not unique: {duplicates}"] + return [] + + def _has(value: _Json, path: tuple[str, ...]) -> bool: for key in path: if not isinstance(value, dict) or key not in value: @@ -76,39 +116,53 @@ def _has(value: _Json, path: tuple[str, ...]) -> bool: return True +def is_document(value: object) -> TypeIs[Document]: + # what the SPDX rendering reads without a fallback; the schema leaves even these optional + return isinstance(value, dict) and all(_has(value, ("metadata", "component", key)) for key in ("bom-ref", "name")) + + def _reference_problems(document: Document) -> list[str]: - root_ref = document["metadata"]["component"]["bom-ref"] - tools = document["metadata"]["tools"]["components"] - every_ref = _bom_refs([document["metadata"]["component"], *document["components"], *tools]) - if duplicates := sorted(ref for ref, count in Counter(every_ref).items() if count > 1): - return [f"bom-ref values are not unique: {duplicates}"] + root: Final[Component] = document["metadata"]["component"] + tools: Final[list[Component]] = document["metadata"]["tools"]["components"] + components: Final[list[Component]] = _walk([root, *document["components"], *tools]) + if nameless := sorted(component["name"] for component in components if "bom-ref" not in component): + return [f"components lack a bom-ref: {nameless}"] problems: Final[list[str]] = [] - dependencies = {entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"]} - if unknown := (set(dependencies) | set().union(*dependencies.values())) - set(every_ref): + dependencies: Final[dict[str, set[str]]] = { + entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"] + } + refs: Final[set[str]] = {component["bom-ref"] for component in components} + if unknown := (set(dependencies) | set().union(*dependencies.values())) - refs: problems.append(f"dependencies reference bom-refs that do not exist: {sorted(unknown)}") - if dependencies.get(root_ref) != (component_refs := {component["bom-ref"] for component in document["components"]}): + direct: Final[set[str]] = {component["bom-ref"] for component in document["components"]} + if dependencies.get(root["bom-ref"]) != direct: problems.append( - f"root dependsOn {sorted(dependencies.get(root_ref, []))} != components {sorted(component_refs)}" + f"root dependsOn {sorted(dependencies.get(root['bom-ref'], []))} != components {sorted(direct)}" ) - tool_refs = {tool["bom-ref"] for tool in tools} + tool_refs: Final[set[str]] = {tool["bom-ref"] for tool in tools} for formula in document.get("formulation", []): for workflow in formula.get("workflows", []): - if unknown := {reference["ref"] for reference in workflow["resourceReferences"]} - tool_refs: + # a workflow may also name an external resource, which no tool can be + referenced = { + reference["ref"] for reference in workflow.get("resourceReferences", []) if "ref" in reference + } + if unknown := referenced - tool_refs: problems.append(f"workflow {workflow['uid']} references unknown tools: {sorted(unknown)}") return problems -def _bom_refs(components: list[Component]) -> list[str]: +def _walk(components: list[Component]) -> list[Component]: # iterative, so a deeply nested hostile document cannot exhaust the stack - found: Final[list[str]] = [] - pending = list(components) + found: Final[list[Component]] = [] + pending: Final[list[Component]] = list(components) while pending: component = pending.pop() - found.append(component["bom-ref"]) + found.append(component) pending.extend(component.get("components", [])) return found __all__ = [ - "validate", + "is_document", + "validate_document", ] diff --git a/src/python_sbom/_validate/_licenses.py b/src/python_sbom/_validate/_licenses.py index 9ce2970..3c92ee9 100644 --- a/src/python_sbom/_validate/_licenses.py +++ b/src/python_sbom/_validate/_licenses.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +from itertools import accumulate from typing import TYPE_CHECKING, Final if TYPE_CHECKING: @@ -9,11 +10,13 @@ # SPDX matches operators only in upper case _OPERATORS: Final[frozenset[str]] = frozenset({"AND", "OR", "WITH", "(", ")"}) _TOKEN: Final[re.Pattern[str]] = re.compile(r"[()]|[^\s()]+") +# the parser spends three stack frames per parenthesis, and real expressions nest a level or two +_MAX_NESTING: Final[int] = 32 def license_problems(components: list[Component], allowed: frozenset[str]) -> list[str]: return [ - f"{component['bom-ref']}: {problem}" + f"{component.get('bom-ref', component['name'])}: {problem}" for component in _shipped(components) if (problem := _judge(component, allowed)) ] @@ -24,7 +27,7 @@ def _shipped(components: list[Component]) -> list[Component]: # library a bundled wheel vendors ships under that wheel's terms; iterative, so deep nesting cannot exhaust the # stack found: Final[list[Component]] = [] - pending = list(reversed(components)) + pending: Final[list[Component]] = list(reversed(components)) while pending: component = pending.pop() if component["type"] == "library" and ( @@ -59,15 +62,23 @@ def _judge(component: Component, allowed: frozenset[str]) -> str | None: def _allowed(expression: str, allowed: frozenset[str]) -> bool: - # reversed, so the parser takes the next token with a cheap pop from the end - tokens: Final[list[str]] = _TOKEN.findall(expression)[::-1] + tokens: Final[list[str]] = _tokens(expression) verdict: Final[bool] = _license_any_of(tokens, allowed) if tokens: - msg = f"unexpected {tokens[-1]!r}" + msg: Final[str] = f"unexpected {tokens[-1]!r}" raise ValueError(msg) return verdict +def _tokens(expression: str) -> list[str]: + found: Final[list[str]] = _TOKEN.findall(expression) + if (depth := max(accumulate({"(": 1, ")": -1}.get(token, 0) for token in found), default=0)) > _MAX_NESTING: + msg: Final[str] = f"it nests {depth} levels deep, over the {_MAX_NESTING} level limit" + raise ValueError(msg) + # reversed, so the parser takes the next token with a cheap pop from the end + return found[::-1] + + def _license_any_of(tokens: list[str], allowed: frozenset[str]) -> bool: verdict = _all_of(tokens, allowed) while tokens and tokens[-1] == "OR": diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index 8b280fa..a744482 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -6,7 +6,7 @@ try: from cyclonedx.schema import SchemaVersion - from cyclonedx.validation.json import JsonStrictValidator + from cyclonedx.validation.json import JsonStrictValidator, JsonValidationError from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document @@ -15,24 +15,28 @@ raise MissingExtraError(_MESSAGE) from error if TYPE_CHECKING: + from jsonschema.exceptions import ValidationError + from python_sbom._formats._spdx import Spdx # loading the schema takes about a second, so the validator is built once _VALIDATOR: Final[JsonStrictValidator] = JsonStrictValidator(SchemaVersion.V1_6) +# the message quotes the offending value, which for a document root runs to megabytes +_MAX_TEXT: Final[int] = 200 def schema_error(raw: str) -> str | None: - try: - error = _VALIDATOR.validate_str(raw, all_errors=False) - except RecursionError: - # the schema validator recurses once per nesting level, so a hostile document can exhaust the stack - return "it nests too deeply to check" - return None if error is None else str(error) + error: Final = _VALIDATOR.validate_str(raw, all_errors=False) + # all_errors=False returns one error or None, but under a runtime check type checkers do not follow + if not isinstance(error, JsonValidationError): + return None + detail: Final[ValidationError] = error.data + return f"{detail.message[:_MAX_TEXT]} at {detail.json_path[:_MAX_TEXT]}" def spdx_problems(spdx: Spdx) -> list[str]: try: - parsed = JsonLikeDictParser().parse(dict(spdx)) + parsed: Final = JsonLikeDictParser().parse(dict(spdx)) except SPDXParsingError as error: return [f"SPDX rendering does not parse: {message}" for message in error.get_messages()] return [f"SPDX rendering invalid: {message.validation_message}" for message in validate_full_spdx_document(parsed)] diff --git a/src/python_sbom/_validate/_wheel.py b/src/python_sbom/_validate/_wheel.py deleted file mode 100644 index 1042571..0000000 --- a/src/python_sbom/_validate/_wheel.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -import zipfile -from typing import TYPE_CHECKING - -from python_sbom._wheel._archive import find_dist_info - -from ._document import validate - -if TYPE_CHECKING: - from pathlib import Path - - -def validate_wheel(wheel: Path, allowed_licenses: frozenset[str] | None) -> list[str]: - """ - Check every CycloneDX SBOM a wheel carries under ``.dist-info/sboms/``; needs the ``validate`` extra. - - :param wheel: the wheel to read - :param allowed_licenses: SPDX license ids the shipped components may use, or ``None`` to skip the license policy - :returns: one message per problem, empty when the wheel carries at least one SBOM and every SBOM is valid - :raises MissingExtraError: when the ``validate`` extra is not installed - - """ - with zipfile.ZipFile(wheel) as archive: - names = archive.namelist() - prefix = f"{find_dist_info(names)}/sboms/" - sboms = { - name: archive.read(name).decode("utf-8") - for name in sorted(names) - if name.startswith(prefix) and name.endswith(".cdx.json") - } - if not sboms: - return ["no .dist-info/sboms/*.cdx.json found in the wheel"] - return [f"{name}: {problem}" for name, raw in sboms.items() for problem in validate(raw, allowed_licenses)] - - -__all__ = [ - "validate_wheel", -] diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index ad1d1dc..8a098d6 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -6,41 +6,32 @@ import tempfile import time import zipfile +import zlib +from collections import Counter +from contextlib import contextmanager +from email.parser import Parser from pathlib import Path -from typing import TYPE_CHECKING, Final +from typing import IO, TYPE_CHECKING, Final from python_sbom._errors import WheelError from ._record import record_line if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Generator, Iterable + from email.message import Message # metadata files beyond this size are refused, so a hostile wheel cannot exhaust memory _METADATA_LIMIT: Final[int] = 16 * 1024 * 1024 +_CHUNK: Final[int] = 1024 * 1024 -def embed_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: - """ - Write an SBOM into a built wheel under ``.dist-info/sboms/``, with its ``RECORD`` row. - - An SBOM of the same name is replaced. Every other entry keeps its name, order, timestamp and attributes; the new - entry takes ``epoch`` as its timestamp and fixed attributes, so the result is the same on every operating system. - The wheel keeps its permissions, and a failure leaves it untouched with no temporary file behind. - - :param wheel: the wheel to rewrite in place - :param name: the SBOM's file name, such as ``project.cdx.json`` - :param content: the SBOM's bytes - :param epoch: the Unix timestamp of the new entry - :returns: the SBOM's path inside the wheel - :raises WheelError: when the archive is not a wheel - - """ - with zipfile.ZipFile(wheel) as archive: +def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: + with open_archive(wheel, str(wheel)) as archive: entries: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [ (info, archive.read(info)) for info in archive.infolist() ] - if (dist_info := find_dist_info(info.filename for info, _ in entries)) is None: + if (dist_info := _find_dist_info(info.filename for info, _ in entries)) is None: msg = f"{wheel} has no .dist-info/METADATA, so it is not a wheel" raise WheelError(msg) record: Final[str] = f"{dist_info}/RECORD" @@ -48,17 +39,6 @@ def embed_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: msg = f"{wheel} has no {record}" raise WheelError(msg) sbom: Final[str] = f"{dist_info}/sboms/{name}" - lines: Final[list[str]] = [ - line - for info, data in entries - if info.filename == record - for line in data.decode("utf-8").splitlines(keepends=True) - if not line.startswith(f"{sbom},") - ] - newline = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" - # the row of RECORD itself stays last - at = next((index for index, line in enumerate(lines) if line.startswith(f"{record},")), len(lines)) - lines.insert(at, f"{record_line(sbom, content)}{newline}") # zip timestamps start at 1980 entry = zipfile.ZipInfo(sbom, time.gmtime(max(epoch, 315532800))[:6]) entry.compress_type = zipfile.ZIP_DEFLATED @@ -68,7 +48,7 @@ def embed_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: members: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [] for info, data in entries: if info.filename == record: - members.extend([(entry, content), (info, "".join(lines).encode())]) + members.extend([(entry, content), (info, _rewritten_record(data, record, sbom, content))]) elif info.filename != sbom: members.append((info, data)) # a name no "dist/*.whl" glob matches, in the wheel's directory so the final rename stays on one file system @@ -86,36 +66,79 @@ def embed_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: return sbom -def find_dist_info(names: Iterable[str]) -> str | None: - return next( - (name.split("/")[0] for name in names if name.count("/") == 1 and name.endswith(".dist-info/METADATA")), - None, - ) +@contextmanager +def open_archive(file: Path | IO[bytes], where: str) -> Generator[zipfile.ZipFile, None, None]: + try: + with zipfile.ZipFile(file) as archive: + # installers extract the first of two same-named members and zipfile reads the last, and a file system that + # ignores case merges names that differ only by case, so such names would hide what gets installed + if clashes := sorted( + name for name, count in Counter(name.casefold() for name in archive.namelist()).items() if count > 1 + ): + msg = f"{where} holds more than one member named {', '.join(clashes)}, ignoring case" + raise WheelError(msg) + yield archive + except (zipfile.BadZipFile, zlib.error) as error: + msg = f"{where} is not a valid zip archive: {error}" + raise WheelError(msg) from error + + +def _find_dist_info(names: Iterable[str]) -> str | None: + found = [name.split("/")[0] for name in names if name.count("/") == 1 and name.endswith(".dist-info/METADATA")] + # a wheel holds one .dist-info; a second one is a decoy that could otherwise decide whose metadata counts + if len(found) > 1: + msg = f"the archive holds {len(found)} .dist-info directories, {', '.join(sorted(found))}, not one" + raise WheelError(msg) + return next(iter(found), None) + + +def _rewritten_record(record: bytes, name: str, sbom: str, content: bytes) -> bytes: + lines: Final[list[str]] = [ + line for line in record.decode("utf-8").splitlines(keepends=True) if not line.startswith(f"{sbom},") + ] + newline: Final[str] = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" + # the row of RECORD itself stays last + at: Final[int] = next((index for index, line in enumerate(lines) if line.startswith(f"{name},")), len(lines)) + lines.insert(at, f"{record_line(sbom, content)}{newline}") + return "".join(lines).encode() + + +def read_wheel_metadata(archive: zipfile.ZipFile, where: str) -> tuple[str, Message]: + if (dist_info := _find_dist_info(archive.namelist())) is None: + msg = f"{where} has no .dist-info/METADATA, so it is not a wheel" + raise WheelError(msg) + return dist_info, Parser().parsestr(read_member(archive, f"{dist_info}/METADATA", where).decode("utf-8")) -def read_metadata_member(archive: zipfile.ZipFile, name: str, where: str) -> bytes: +def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _METADATA_LIMIT) -> bytes: try: info = archive.getinfo(name) except KeyError as error: msg = f"{where} has no {name}" raise WheelError(msg) from error - if info.file_size > _METADATA_LIMIT: - msg = f"{where}: {name} is {info.file_size} bytes, over the {_METADATA_LIMIT} byte limit" - raise WheelError(msg) - return archive.read(info) - - -def member_sha256(archive: zipfile.ZipFile, name: str) -> str: - digest = hashlib.sha256() + # counted as it decompresses, since the size the archive states may lie and one read decompresses it all at once + content: Final[bytearray] = bytearray() + with archive.open(info) as member: + while chunk := member.read(_CHUNK): + content.extend(chunk) + if len(content) > limit: + msg = f"{where}: {name} is over the {limit} byte limit" + raise WheelError(msg) + return bytes(content) + + +def member_digest(archive: zipfile.ZipFile, name: str | zipfile.ZipInfo, algorithm: str) -> str: + digest: Final = hashlib.new(algorithm) with archive.open(name) as member: - for chunk in iter(lambda: member.read(1024 * 1024), b""): + while chunk := member.read(_CHUNK): digest.update(chunk) return digest.hexdigest() __all__ = [ - "embed_sbom", - "find_dist_info", - "member_sha256", - "read_metadata_member", + "member_digest", + "open_archive", + "read_member", + "read_wheel_metadata", + "write_sbom", ] diff --git a/src/python_sbom/_wheel/_record.py b/src/python_sbom/_wheel/_record.py index b2701fd..e16adf3 100644 --- a/src/python_sbom/_wheel/_record.py +++ b/src/python_sbom/_wheel/_record.py @@ -5,6 +5,7 @@ import csv import hashlib from io import StringIO +from pathlib import PurePosixPath from typing import Final from python_sbom._errors import WheelError @@ -18,21 +19,30 @@ "sha384": "SHA-384", "sha512": "SHA-512", } +# the hashlib name of each algorithm, by its CycloneDX spelling +HASHLIB_NAMES: Final[dict[str, str]] = {cyclonedx: name for name, cyclonedx in _ALGORITHMS.items()} +# the files of a .dist-info the wheel specification leaves without a digest: RECORD and its signatures +UNHASHED: Final[frozenset[str]] = frozenset({"RECORD", "RECORD.jws", "RECORD.p7s"}) def parse_record(text: str, where: str) -> list[RecordRow]: - # the row of RECORD itself has no digest and is left out + # the rows without a digest, which only RECORD and its signatures may have, are left out rows: Final[list[RecordRow]] = [] for number, columns in enumerate(csv.reader(StringIO(text, newline="")), start=1): - if not columns: - continue - if len(columns) != 3: # ruff: ignore[magic-value-comparison] # path, digest and size, as the wheel specification lays out a row - msg = f"{where}: RECORD row {number} has {len(columns)} columns, not 3" - raise WheelError(msg) - path, digest, size = columns - if digest: - algorithm, hexdigest = decode_digest(digest, f"{where}: RECORD row {number}") - rows.append(RecordRow(path=path, algorithm=algorithm, digest=hexdigest, size=size)) + match columns: + case []: + pass + case [path, "", _] if PurePosixPath(path).name in UNHASHED: + pass + case [path, "", _]: + msg = f"{where}: RECORD row {number} gives no digest for {path}" + raise WheelError(msg) + case [path, digest, size]: + algorithm, hexdigest = decode_digest(digest, f"{where}: RECORD row {number}") + rows.append(RecordRow(path=path, algorithm=algorithm, digest=hexdigest, size=size)) + case _: + msg = f"{where}: RECORD row {number} has {len(columns)} columns, not 3" + raise WheelError(msg) return rows @@ -55,6 +65,8 @@ def record_line(name: str, content: bytes) -> str: __all__ = [ + "HASHLIB_NAMES", + "UNHASHED", "decode_digest", "parse_record", "record_line", diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index fbff7be..735bf04 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -1,5 +1,7 @@ from __future__ import annotations +import tempfile +from functools import cached_property from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Final @@ -7,16 +9,15 @@ from hatchling.builders.utils import get_reproducible_timestamp from hatchling.utils.constants import DEFAULT_BUILD_SCRIPT -from python_sbom._collect._facts import collect_facts, default_revision +from python_sbom._collect._content import load_content_provider, provider_script +from python_sbom._collect._facts import REVISION_FILE, collect_facts, default_revision from python_sbom._collect._settings import settings_from_table -from python_sbom._collect._source import read_source_tree -from python_sbom._generate._backend import BuiltWheel -from python_sbom._generate._content import ContentProvider, load_content_provider -from python_sbom._generate._pipeline import build_sbom -from python_sbom._wheel._archive import embed_sbom +from python_sbom._collect._source import wheel_request +from python_sbom._errors import SettingsError +from python_sbom._generate._embed import embed_into if TYPE_CHECKING: - from python_sbom._generate._backend import Backend + from python_sbom._model._content import ContentProvider from python_sbom._model._inputs import Settings # options of every hatchling hook table, which hatchling itself reads @@ -34,7 +35,9 @@ class SbomBuildHook(BuildHookInterface): Hatchling build hook that writes a CycloneDX SBOM into the wheel's ``.dist-info/sboms/``, as PEP 770 describes. It runs once hatchling has written the wheel, so the SBOM describes the wheel's final metadata and file name - whatever other hooks changed. Options come from ``[tool.python-sbom]``, overridden by the hook's own table. + whatever other hooks changed. Options come from ``[tool.python-sbom]``, the one place ``python-sbom embed`` and + ``python-sbom archive`` read them too. For an sdist it records the source revision, so a wheel built from that sdist + still names its commit. :cvar PLUGIN_NAME: the hook's name in ``[tool.hatch.build.hooks]`` @@ -42,6 +45,26 @@ class SbomBuildHook(BuildHookInterface): PLUGIN_NAME = "python-sbom" + def initialize( + self, + version: str, # ruff: ignore[unused-method-argument] # hatchling's signature + build_data: dict[str, Any], # simplint: ignore[SMP602] # hatchling's signature + ) -> None: + """ + Record the source revision in an sdist. + + :param version: hatchling's build variant + :param build_data: hatchling's build data, which receives the revision file + + """ + # a dirty checkout has no commit that reproduces the sdist, so a wheel built from it names none + if self.target_name != "sdist" or (revision := default_revision(Path(self.root))) is None or revision.dirty: + return + # outside the project, so the build leaves nothing behind in it + written = Path(tempfile.mkdtemp()) / REVISION_FILE + written.write_text(f"node: {revision.commit}\n", encoding="utf-8") + build_data["force_include"][str(written)] = REVISION_FILE + def finalize( self, version: str, @@ -52,59 +75,51 @@ def finalize( Write the SBOM into the wheel hatchling built. :param version: hatchling's build variant; an editable wheel gets no SBOM, since it carries no code - :param build_data: unused + :param build_data: hatchling's build data :param artifact_path: the built wheel """ if self.target_name != "wheel" or version == "editable": return root = Path(self.root) - # hatchling has parsed pyproject.toml already - settings = settings_from_table( - { - **self.metadata.config.get("tool", {}).get("python-sbom", {}), - **{key: value for key, value in self.config.items() if key not in _HATCHLING_OPTIONS}, - }, - f"[tool.python-sbom] or the {self.PLUGIN_NAME} hook's table", - ) - content, provider_scripts = self.content_provider(settings) - # a custom hook script shapes the document, so the SBOM lists it among the build tools - own_script = ( - (PurePosixPath(self.config.get("path", DEFAULT_BUILD_SCRIPT)),) if self.PLUGIN_NAME == "custom" else () - ) - request = self.backend(Path(artifact_path), settings, (*own_script, *provider_scripts)).request() + content = self.content_provider() + # a custom hook script shapes the document, as a provider defined in the project does, so the SBOM lists both + # among the build tools + own = [PurePosixPath(self.config.get("path", DEFAULT_BUILD_SCRIPT))] if self.PLUGIN_NAME == "custom" else [] + scripts = tuple(dict.fromkeys([*own, *filter(None, [provider_script(content, root)])])) # hatchling's own fixed time for a reproducible build when SOURCE_DATE_EPOCH is unset, so the SBOM agrees with # the zip entries hatchling wrote facts = collect_facts( - root, default_revision, None, get_reproducible_timestamp() if self.build_config.reproducible else None + root, + default_revision, + default_epoch=get_reproducible_timestamp() if self.build_config.reproducible else None, ) - sbom = build_sbom(request, read_source_tree(request), facts, content) - embed_sbom(Path(artifact_path), sbom.file_name, sbom.content, facts.epoch) + wheel = Path(artifact_path) + embed_into(wheel, wheel_request(wheel, root, self._settings, scripts), content, facts) - def content_provider(self, settings: Settings) -> tuple[ContentProvider, tuple[PurePosixPath, ...]]: + def content_provider(self) -> ContentProvider: """ - Return what the project adds to its SBOM; override it in a custom build hook to supply a provider directly. - - :param settings: the project's options - :returns: the provider and the scripts that define it; by default the ``content-provider`` option's, or none + Return what the project adds to its SBOM. - """ - if settings.content_provider is None: - return ContentProvider(), () - provider, script = load_content_provider(settings.content_provider, Path(self.root)) - return provider, () if script is None else (script,) - - def backend(self, wheel: Path, settings: Settings, build_scripts: tuple[PurePosixPath, ...]) -> Backend: - """ - Return the backend that describes the built wheel. + Override it in a hatchling custom build hook to build the provider with what the hook knows; the + ``content-provider`` option is then ignored. - :param wheel: the wheel hatchling built - :param settings: the project's options - :param build_scripts: the scripts that shape the document - :returns: a :class:`~python_sbom.BuiltWheel` by default + :returns: the ``content-provider`` option's provider, or one that adds nothing """ - return BuiltWheel(wheel=wheel, root=Path(self.root), settings=settings, build_scripts=build_scripts) + return load_content_provider(self._settings.content_provider, Path(self.root)) + + @cached_property + def _settings(self) -> Settings: + # hatchling has parsed pyproject.toml already; the hook's own table holds only hatchling's options, so every + # entry point reads the same [tool.python-sbom] + if unknown := sorted(self.config.keys() - _HATCHLING_OPTIONS): + msg = ( + f"the {self.PLUGIN_NAME} hook's table holds {', '.join(unknown)}; move python-sbom's options to" + " [tool.python-sbom], where python-sbom embed and python-sbom archive read them too" + ) + raise SettingsError(msg) + return settings_from_table(self.metadata.config.get("tool", {}).get("python-sbom", {}), "[tool.python-sbom]") __all__ = [ diff --git a/tests/_cli/test_cli.py b/tests/_cli/test_commands.py similarity index 55% rename from tests/_cli/test_cli.py rename to tests/_cli/test_commands.py index 3970b16..813969e 100644 --- a/tests/_cli/test_cli.py +++ b/tests/_cli/test_commands.py @@ -10,12 +10,14 @@ import pytest -from python_sbom import dumps, main +from python_sbom._cli._main import main +from python_sbom._formats._json import dumps +from python_sbom.version import __version__ if TYPE_CHECKING: from conftest import Describe, Install, WriteWheel - from python_sbom import Document + from python_sbom._model._types import Document _SBOM: Final[str] = "native-1.0.dist-info/sboms/native.cdx.json" @@ -38,11 +40,12 @@ def wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: def test_embed( project: Path, wheel: Path, tmp_path: Path, install: Install, capsys: pytest.CaptureFixture[str] ) -> None: + commit = "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" install(site := tmp_path / "site", "Name: meson-python\nVersion: 0.18\n", {}) - main(["embed", str(wheel), "--root", str(project), "--commit", "c0ffee", "--build-env", str(site)]) + main(["embed", str(wheel), "--root", str(project), "--commit", commit, "--build-env", str(site)]) document = embedded(wheel) assert capsys.readouterr().out == f"wrote {_SBOM} into {wheel.name}\n" - assert {"name": "native:vcs-commit", "value": "c0ffee"} in document["metadata"]["component"]["properties"] + assert {"name": "native:vcs-commit", "value": commit} in document["metadata"]["component"]["properties"] assert [tool["name"] for tool in document["metadata"]["tools"]["components"][1:]] == ["meson-python"] @@ -52,9 +55,11 @@ def embedded(wheel: Path) -> Document: def test_embed_defaults_to_current_directory(project: Path, wheel: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(project) - monkeypatch.setenv("PYTHON_SBOM_COMMIT", "cafe") + monkeypatch.setenv("PYTHON_SBOM_COMMIT", "cafecafecafecafecafecafecafecafecafecafe") main(["embed", str(wheel), "--build-env", str(project)]) - assert {"name": "native:vcs-commit", "value": "cafe"} in embedded(wheel)["metadata"]["component"]["properties"] + assert {"name": "native:vcs-commit", "value": "cafecafecafecafecafecafecafecafecafecafe"} in embedded(wheel)[ + "metadata" + ]["component"]["properties"] @pytest.mark.parametrize( @@ -71,9 +76,9 @@ def test_embed_with_content_provider(tmp_path: Path, project: Path, wheel: Path, class Native(ContentProvider): - def contributions(self, request): + def contributions(self, context): return Contributions( - components=[], dependencies=[], root_properties=[{"name": "native:by", "value": "cli"}], tools=[] + root_properties=[{"name": "extra:by", "value": "cli"}] ) """ (project / f"{tmp_path.name}.py").write_text(provider, encoding="utf-8") @@ -88,7 +93,7 @@ def contributions(self, request): *(argument.replace("MODULE", tmp_path.name) for argument in option), ]) document = embedded(wheel) - assert {"name": "native:by", "value": "cli"} in document["metadata"]["component"]["properties"] + assert {"name": "extra:by", "value": "cli"} in document["metadata"]["component"]["properties"] assert document["metadata"]["tools"]["components"][0]["name"] == f"{tmp_path.name}.py" @@ -115,7 +120,7 @@ def test_validate_directory(project: Path, wheel: Path, capsys: pytest.CaptureFi def test_validate_reports_problems(wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: with pytest.raises(SystemExit): main(["validate", str(wheel)]) - assert capsys.readouterr().err == f"SBOM invalid: {wheel.name}: no .dist-info/sboms/*.cdx.json found in the wheel\n" + assert capsys.readouterr().err == f"SBOM invalid: {wheel.name}: no SBOM found in {wheel.name}\n" def test_validate_without_wheels(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: @@ -124,19 +129,6 @@ def test_validate_without_wheels(tmp_path: Path, capsys: pytest.CaptureFixture[s assert capsys.readouterr().err == f"no wheel found in {tmp_path}\n" -def test_validate_without_extra( - wheel: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - monkeypatch.setitem(sys.modules, "cyclonedx.schema", None) - monkeypatch.delitem(sys.modules, "python_sbom._validate._schemas", raising=False) - zipfile.ZipFile(wheel, "a").writestr(_SBOM, b"{}") - with pytest.raises(SystemExit): - main(["validate", str(wheel)]) - assert capsys.readouterr().err == ( - "python-sbom: validation needs the validate extra: pip install 'python-sbom[validate]'\n" - ) - - def test_module_entry_point(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: monkeypatch.setattr(sys, "argv", ["python-sbom", "--help"]) with pytest.raises(SystemExit): @@ -147,3 +139,78 @@ def test_module_entry_point(monkeypatch: pytest.MonkeyPatch, capsys: pytest.Capt def test_console_script() -> None: script = Path(sys.executable).parent / "python-sbom" assert subprocess.check_output([script, "--help"], text=True).startswith("usage: python-sbom ") + + +@pytest.fixture +def zipapp(tmp_path: Path) -> Path: + with zipfile.ZipFile(target := tmp_path / "app.pyz", "w") as handle: + handle.writestr("__main__.py", "import app\n") + handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\nLicense-Expression: MIT\n") + handle.writestr("app/__init__.py", "") + return target + + +def test_archive_writes_sbom_next_to_archive(project: Path, zipapp: Path, capsys: pytest.CaptureFixture[str]) -> None: + main(["archive", str(zipapp), "--root", str(project), "--build-env", str(project)]) + assert ( + capsys.readouterr().out, + json.loads(zipapp.with_name("app.pyz.cdx.json").read_text())["metadata"]["component"]["name"], + ) == ( + f"wrote {zipapp.with_name('app.pyz.cdx.json')}\n", + "app.pyz", + ) + + +def test_archive_embeds_sbom(project: Path, zipapp: Path, tmp_path: Path) -> None: + (project / "build_app.py").write_text("", encoding="utf-8") + main([ + "archive", + str(zipapp), + "--root", + str(project), + "--build-env", + str(project), + "--build-script", + "build_app.py", + "--output", + str(tmp_path / "out.json"), + "--embed", + ]) + entry = zipfile.ZipFile(zipapp).getinfo("app.pyz.cdx.json") + assert (entry.create_system, entry.external_attr, zipfile.ZipFile(zipapp).read(entry)) == ( + 3, + 0o644 << 16, + (tmp_path / "out.json").read_bytes(), + ) + + +def test_archive_refuses_second_embed(project: Path, zipapp: Path, capsys: pytest.CaptureFixture[str]) -> None: + arguments = ["archive", str(zipapp), "--root", str(project), "--build-env", str(project), "--embed"] + main(arguments) + capsys.readouterr() + with pytest.raises(SystemExit): + main(arguments) + assert capsys.readouterr().err == "python-sbom: app.pyz already holds app.pyz.cdx.json\n" + + +def test_version(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["--version"]) + assert capsys.readouterr().out == f"python-sbom {__version__}\n" + + +def test_spdx_refuses_non_document(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + (source := tmp_path / "other.json").write_text("[]", encoding="utf-8") + with pytest.raises(SystemExit): + main(["spdx", str(source), str(tmp_path / "out.json")]) + assert capsys.readouterr().err == f"python-sbom: {source} is not a CycloneDX document python-sbom can render\n" + + +def test_spdx_refuses_malformed_document(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + document = {"metadata": {"component": {"bom-ref": "app", "name": "app"}}, "components": [{"name": "x"}]} + (source := tmp_path / "broken.json").write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(SystemExit): + main(["spdx", str(source), str(tmp_path / "out.json")]) + assert capsys.readouterr().err.startswith( + f"python-sbom: {source} is not a CycloneDX document python-sbom can render: " + ) diff --git a/tests/_collect/test_collect_facts.py b/tests/_collect/test_collect_facts.py deleted file mode 100644 index 27bcce5..0000000 --- a/tests/_collect/test_collect_facts.py +++ /dev/null @@ -1,221 +0,0 @@ -from __future__ import annotations - -import hashlib -import platform -import sys -import time -from typing import TYPE_CHECKING - -import pytest - -from python_sbom import ( - Interpreter, - RecordRow, - Revision, - SettingsError, - SourceRevisionError, - collect_facts, - default_revision, - git_revision, -) - -if TYPE_CHECKING: - from pathlib import Path - - from conftest import Git, Install - - from python_sbom import BuildFacts - - -def test_epoch_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") - facts = collect_facts(tmp_path, no_revision, [], 0) - assert facts.epoch == 1_700_000_000 - assert facts.epoch_from_environment - - -def no_revision(_root: Path, /) -> None: - return None - - -def test_epoch_default(tmp_path: Path) -> None: - facts = collect_facts(tmp_path, no_revision, [], 42) - assert (facts.epoch, facts.epoch_from_environment) == (42, False) - - -def test_empty_epoch_is_unset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("SOURCE_DATE_EPOCH", "") - facts = collect_facts(tmp_path, no_revision, [], 42) - assert (facts.epoch, facts.epoch_from_environment) == (42, False) - - -def test_epoch_without_default_is_now(tmp_path: Path) -> None: - before = int(time.time()) - assert before <= collect_facts(tmp_path, no_revision, [], None).epoch <= time.time() - - -def test_invalid_epoch(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("SOURCE_DATE_EPOCH", "yesterday") - with pytest.raises(SettingsError, match="SOURCE_DATE_EPOCH must be a Unix timestamp, got 'yesterday'"): - collect_facts(tmp_path, no_revision, [], None) - - -def test_interpreter(tmp_path: Path) -> None: - assert collect_facts(tmp_path, no_revision, [], 0).interpreter == Interpreter( - name=sys.implementation.name, - version=platform.python_version(), - implementation=platform.python_implementation(), - ) - - -def test_revision_reads_root(tmp_path: Path) -> None: - def reader(root: Path, /) -> Revision: - return Revision(commit=root.name, dirty=False) - - assert collect_facts(tmp_path, reader, [], 0).revision == Revision(commit=tmp_path.name, dirty=False) - - -def test_default_path_is_this_interpreter(tmp_path: Path) -> None: - names = { - distribution.metadata["Name"] for distribution in collect_facts(tmp_path, no_revision, None, 0).distributions - } - assert "pytest" in names - - -def test_pure_distribution_files(tmp_path: Path, install: Install) -> None: - # nested, so the console script, two levels up as installers write it, stays inside the test's directory - install( - site := tmp_path / "lib" / "site-packages", - "Name: alpha\nVersion: 1.0\n", - { - "alpha/__init__.py": b"x = 1\n", - "../../bin/alpha": b"#!/usr/bin/python\n", - "alpha-1.0.dist-info/INSTALLER": b"uv\n", - "alpha-1.0.dist-info/REQUESTED": b"", - "alpha-1.0.dist-info/direct_url.json": b'{"url": "file:///tmp/alpha.whl", "archive_info": {}}', - "alpha-1.0.dist-info/uv_cache.json": b'{"timestamp": 1}', - }, - ) - (distribution,) = facts_from(site).distributions - assert distribution.files == [ - RecordRow( - path="alpha/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"x = 1\n").hexdigest(), size="6" - ), - RecordRow( - path="alpha-1.0.dist-info/METADATA", - algorithm="SHA-256", - digest=hashlib.sha256(b"Name: alpha\nVersion: 1.0\n").hexdigest(), - size="25", - ), - RecordRow( - path="alpha-1.0.dist-info/WHEEL", - algorithm="SHA-256", - digest=hashlib.sha256(b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\n").hexdigest(), - size="41", - ), - ] - - -def facts_from(site: Path) -> BuildFacts: - return collect_facts(site, no_revision, [str(site)], 0) - - -def test_platform_distribution_lists_no_files(tmp_path: Path, install: Install) -> None: - install(tmp_path, "Name: native\nVersion: 1.0\n", {"native.so": b"\x7fELF"}, purelib=False) - assert facts_from(tmp_path).distributions[0].files == [] - - -def test_editable_distribution_lists_no_files(tmp_path: Path, install: Install) -> None: - editable = '{"url": "file:///checkout", "dir_info": {"editable": true}}' - install(tmp_path, "Name: local\nVersion: 1.0\n", {"local.pth": b"/checkout/src\n"}, direct_url=editable) - assert facts_from(tmp_path).distributions[0].files == [] - - -def test_first_distribution_on_path_wins(tmp_path: Path, install: Install) -> None: - install(first := tmp_path / "first", "Name: alpha\nVersion: 1.0\n", {}) - install(second := tmp_path / "second", "Name: Alpha\nVersion: 2.0\n", {}) - (distribution,) = collect_facts(tmp_path, no_revision, [str(first), str(second)], 0).distributions - assert distribution.metadata["Version"] == "1.0" - - -def test_depends_on_installed_requirements_that_apply(tmp_path: Path, install: Install) -> None: - install( - tmp_path, - "Name: alpha\nVersion: 1.0\nRequires-Dist: Beta>=1\nRequires-Dist: gamma; python_version < '3'\n" - "Requires-Dist: delta; extra == 'cli'\nRequires-Dist: missing\n", - {}, - ) - for name in ("beta", "gamma", "delta"): - install(tmp_path, f"Name: {name}\nVersion: 1.0\n", {}) - by_name = {distribution.metadata["Name"]: distribution for distribution in facts_from(tmp_path).distributions} - assert by_name["alpha"].depends_on == ["beta"] - - -def test_commit_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("PYTHON_SBOM_COMMIT", "c0ffee") - assert default_revision(tmp_path) == Revision(commit="c0ffee", dirty=False) - - -@pytest.fixture -def checkout(tmp_path: Path, git: Git) -> Path: - (tmp_path / "checkout" / "tracked.txt").write_text("one\n", encoding="utf-8") - git("add", "tracked.txt") - git("commit", "--quiet", "--message", "first") - return tmp_path / "checkout" - - -def test_git_revision(checkout: Path, git: Git) -> None: - assert default_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=False) - - -def test_git_revision_dirty(checkout: Path, git: Git) -> None: - (checkout / "tracked.txt").write_text("two\n", encoding="utf-8") - assert git_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=True) - - -def test_untracked_files_leave_revision_clean(checkout: Path) -> None: - (checkout / "untracked.txt").write_text("new\n", encoding="utf-8") - revision = git_revision(checkout) - assert revision is not None - assert not revision.dirty - - -def test_git_revision_from_subdirectory(checkout: Path, git: Git) -> None: - (subdirectory := checkout / "packages" / "one").mkdir(parents=True) - assert git_revision(subdirectory) == Revision(commit=git("rev-parse", "HEAD"), dirty=False) - - -def test_git_revision_before_first_commit(tmp_path: Path, git: Git) -> None: - git("status") - assert git_revision(tmp_path / "checkout") is None - - -def test_git_revision_outside_checkout(tmp_path: Path) -> None: - assert git_revision(tmp_path) is None - - -def test_git_revision_without_git(checkout: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("PATH", "") - assert git_revision(checkout) is None - - -def test_git_revision_unreadable_checkout(tmp_path: Path) -> None: - (tmp_path / ".git").write_text(f"gitdir: {tmp_path / 'missing'}\n", encoding="utf-8") - with pytest.raises(SourceRevisionError, match="git rev-parse HEAD failed in"): - git_revision(tmp_path) - - -@pytest.mark.parametrize( - ("archival", "expected"), - [ - pytest.param(f"node: {'a' * 40}\n", Revision(commit="a" * 40, dirty=False), id="expanded"), - pytest.param("node: $Format:%H$\n", None, id="unexpanded"), - ], -) -def test_git_archival(tmp_path: Path, archival: str, expected: Revision | None) -> None: - (tmp_path / ".git_archival.txt").write_text(archival, encoding="utf-8") - assert default_revision(tmp_path) == expected - - -def test_no_revision_source(tmp_path: Path) -> None: - assert default_revision(tmp_path) is None diff --git a/tests/_collect/test_collect_source.py b/tests/_collect/test_collect_source.py deleted file mode 100644 index aec3235..0000000 --- a/tests/_collect/test_collect_source.py +++ /dev/null @@ -1,185 +0,0 @@ -from __future__ import annotations - -import dataclasses -import hashlib -from email import message_from_string -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Final - -import pytest -from conftest import record_digest - -from python_sbom import BuildScript, RecordRow, SbomRequest, WheelError, read_bundled_wheel, read_source_tree - -if TYPE_CHECKING: - from collections.abc import Mapping - - from conftest import WriteWheel - - from python_sbom import Settings - -_PIP: Final[dict[str, bytes]] = { - "pip/__init__.py": b"", - "pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n", -} - - -@pytest.mark.parametrize( - ("files", "expected"), - [ - pytest.param({}, None, id="no-license-file"), - pytest.param({"LICENSE": "MIT terms\n"}, None, id="no-copyright-line"), - pytest.param( - {"LICENSE": "MIT\n\nCopyright (c) 2026 Example\nCopyright (c) 2025 Other\n"}, - "Copyright (c) 2026 Example", - id="first-copyright-line", - ), - ], -) -def test_copyright(tmp_path: Path, settings: Settings, files: Mapping[str, str], expected: str | None) -> None: - for name, text in files.items(): - (tmp_path / name).write_text(text, encoding="utf-8") - source = read_source_tree(request(tmp_path, settings, "License-File: LICENSE\nLicense-File: NOTICE\n")) - assert source.copyright == expected - - -def request(root: Path, settings: Settings, metadata: str = "", scripts: tuple[PurePosixPath, ...] = ()) -> SbomRequest: - return SbomRequest( - root=root, - metadata=message_from_string(f"Name: example\nVersion: 1.0\n{metadata}"), - wheel_name="example-1.0-py3-none-any.whl", - settings=settings, - build_scripts=scripts, - ) - - -def test_bundled_wheels_sorted_once(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: - for directory in ("b", "a"): - write_wheel(tmp_path / "src" / directory / "pip-1.0-py3-none-any.whl", _PIP) - settings = dataclasses.replace(settings, bundled_wheels=("src/*/*.whl", "src/a/*.whl")) - paths = [wheel.path for wheel in read_source_tree(request(tmp_path, settings)).bundled] - assert paths == ["src/a/pip-1.0-py3-none-any.whl", "src/b/pip-1.0-py3-none-any.whl"] - - -def test_build_scripts_hashed(tmp_path: Path, settings: Settings) -> None: - (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") - script = PurePosixPath("hatch_build.py") - source = read_source_tree(request(tmp_path, settings, scripts=(script,))) - assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] - - -def test_build_script_outside_project(tmp_path: Path, settings: Settings) -> None: - # a custom hook path may be absolute, which on Windows carries a drive the join with the root must keep - (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") - script = PurePosixPath((tmp_path / "hatch_build.py").as_posix()) - source = read_source_tree(request(tmp_path / "project", settings, scripts=(script,))) - assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] - - -def test_bundled_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: - wheel = write_wheel( - tmp_path / "src" / "pip-1.0-py3-none-any.whl", - { - **_PIP, - "pip/_vendor/vendor.txt": b"# pinned\n\nurllib3==1.26.4 # http\n", - "pip/_vendor/rich-13.0.dist-info/METADATA": b"Name: rich\nVersion: 13.0\n", - }, - ) - bundled = read_bundled_wheel(tmp_path, wheel) - assert (bundled.path, bundled.sha256, bundled.metadata_name, bundled.metadata["Name"], bundled.files[0]) == ( - "src/pip-1.0-py3-none-any.whl", - hashlib.sha256(wheel.read_bytes()).hexdigest(), - "pip-1.0.dist-info/METADATA", - "pip", - RecordRow(path="pip/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"").hexdigest(), size="0"), - ) - assert [(source, str(metadata)) for source, metadata in bundled.vendored] == [ - ("pip/_vendor/vendor.txt", "Name: urllib3\nVersion: 1.26.4\n\n"), - ("pip/_vendor/rich-13.0.dist-info/METADATA", "Name: rich\nVersion: 13.0\n\n"), - ] - - -_EMPTY: Final[str] = "47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU" -_METADATA: Final[str] = f"pip-1.0.dist-info/METADATA,sha256={_EMPTY},0\n" - - -@pytest.mark.parametrize( - ("files", "message"), - [ - pytest.param({"pip/__init__.py": b""}, "has no .dist-info/METADATA", id="no-metadata"), - pytest.param({"pip-1.0.dist-info/METADATA": b"Name: pip\n"}, "METADATA lacks Name or Version", id="no-version"), - pytest.param( - {"pip-1.0.dist-info/METADATA": _PIP["pip-1.0.dist-info/METADATA"]}, - "has no pip-1.0.dist-info/RECORD", - id="no-record", - ), - pytest.param( - {"pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n" + b" " * 16 * 1024 * 1024}, - "METADATA is 16777239 bytes, over the 16777216 byte limit", - id="oversized-metadata", - ), - ], -) -def test_malformed_bundled_wheel( - tmp_path: Path, write_wheel: WriteWheel, files: Mapping[str, bytes], message: str -) -> None: - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - with pytest.raises(WheelError, match=message): - read_bundled_wheel(tmp_path, wheel) - - -@pytest.mark.parametrize( - ("record", "message"), - [ - pytest.param("pip/__init__.py,sha256=x\n", "RECORD row 1 has 2 columns, not 3", id="columns"), - pytest.param("pip/__init__.py,blake2b=x,0\n", "RECORD row 1: unknown hash algorithm 'blake2b'", id="algorithm"), - pytest.param("pip/__init__.py,sha256=abcde,0\n", "digest 'abcde' is not urlsafe base64", id="base64"), - pytest.param(_METADATA, "RECORD lacks a SHA-256 for pip/__init__.py", id="member-not-recorded"), - pytest.param( - f"{_METADATA}pip/__init__.py,sha256={_EMPTY},0\npip/gone.py,sha256={_EMPTY},0\n", - "RECORD lists pip/gone.py, which the wheel does not hold", - id="recorded-not-member", - ), - pytest.param( - f"pip/__init__.py,sha256={_EMPTY},0\npip-1.0.dist-info/METADATA,sha256={_EMPTY},0\n", - "the SHA-256 in RECORD differs from the content of pip-1.0.dist-info/METADATA", - id="digest-mismatch", - ), - ], -) -def test_record_disagrees_with_wheel(tmp_path: Path, write_wheel: WriteWheel, record: str, message: str) -> None: - files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - with pytest.raises(WheelError, match=message): - read_bundled_wheel(tmp_path, wheel) - - -@pytest.mark.parametrize( - ("manifest", "message"), - [ - pytest.param(b"urllib3 ==\n", "does not parse: urllib3 ==", id="unparsable"), - pytest.param(b"urllib3>=1\n", "pins no one exact version: urllib3>=1", id="range"), - pytest.param(b"urllib3==1.*\n", "pins no one exact version: urllib3==1.*", id="wildcard"), - pytest.param(b"urllib3==1,!=1.1\n", "pins no one exact version: urllib3==1,!=1.1", id="two-specifiers"), - ], -) -def test_vendor_manifest_needs_exact_pins( - tmp_path: Path, write_wheel: WriteWheel, manifest: bytes, message: str -) -> None: - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", {**_PIP, "pip/_vendor/vendor.txt": manifest}) - with pytest.raises(WheelError, match=message): - read_bundled_wheel(tmp_path, wheel) - - -def test_record_blank_lines_are_skipped(tmp_path: Path, write_wheel: WriteWheel) -> None: - metadata = _PIP["pip-1.0.dist-info/METADATA"] - record = ( - f"pip/__init__.py,sha256={_EMPTY},0\n\n" - f"pip-1.0.dist-info/METADATA,{record_digest(metadata)},{len(metadata)}\n" - "pip-1.0.dist-info/RECORD,,\n" - ) - wheel = write_wheel(tmp_path / "pip.whl", {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()}, record=False) - assert [row.path for row in read_bundled_wheel(tmp_path, wheel).files] == [ - "pip/__init__.py", - "pip-1.0.dist-info/METADATA", - ] diff --git a/tests/_generate/test_generate_content.py b/tests/_collect/test_content.py similarity index 53% rename from tests/_generate/test_generate_content.py rename to tests/_collect/test_content.py index f8a4b7e..3cb92ea 100644 --- a/tests/_generate/test_generate_content.py +++ b/tests/_collect/test_content.py @@ -1,36 +1,12 @@ from __future__ import annotations -from email import message_from_string from pathlib import Path, PurePosixPath import pytest -from python_sbom import ( - BundledWheel, - BundledWheelInfo, - ContentProvider, - Contributions, - SbomRequest, - Settings, - SettingsError, - load_content_provider, -) - - -def test_default_provider_adds_nothing_to_bundled_wheels() -> None: - wheel = BundledWheel( - path="a.whl", sha256="", metadata_name="", metadata=message_from_string(""), files=[], vendored=[] - ) - assert ContentProvider().bundled_wheel(wheel) == BundledWheelInfo(properties=[], supplier=None) - - -def test_default_provider_contributes_nothing(settings: Settings) -> None: - request = SbomRequest( - root=Path(), metadata=message_from_string(""), wheel_name="", settings=settings, build_scripts=() - ) - assert ContentProvider().contributions(request) == Contributions( - components=[], dependencies=[], root_properties=[], tools=[] - ) +from python_sbom import ContentProvider +from python_sbom._collect._content import load_content_provider, provider_script +from python_sbom._errors import SettingsError @pytest.fixture @@ -54,15 +30,23 @@ class Content(ContentProvider): @pytest.mark.parametrize("attribute", [pytest.param("Content", id="class"), pytest.param("instance", id="instance")]) def test_load_from_project(tmp_path: Path, module: str, attribute: str) -> None: - provider, script = load_content_provider(f"{module}:{attribute}", tmp_path) - assert type(provider).__name__ == "Content" - assert script == PurePosixPath(f"tools/{tmp_path.name}.py") + provider = load_content_provider(f"{module}:{attribute}", tmp_path) + assert (type(provider).__name__, provider_script(provider, tmp_path)) == ( + "Content", + PurePosixPath(f"tools/{tmp_path.name}.py"), + ) + + +def test_provider_outside_project_is_no_tool(tmp_path: Path) -> None: + provider = load_content_provider("python_sbom:ContentProvider", tmp_path) + assert (type(provider), provider_script(provider, tmp_path)) == (ContentProvider, None) -def test_load_from_outside_project(tmp_path: Path) -> None: - provider, script = load_content_provider("python_sbom:ContentProvider", tmp_path) - assert type(provider) is ContentProvider - assert script is None +def test_provider_class_outside_project_is_no_tool(tmp_path: Path) -> None: + class Local(ContentProvider): + pass + + assert provider_script(Local(), tmp_path) is None @pytest.mark.parametrize( @@ -78,3 +62,8 @@ def test_load_from_outside_project(tmp_path: Path) -> None: def test_load_refuses(tmp_path: Path, module: str, spec: str, message: str) -> None: with pytest.raises(SettingsError, match=message): load_content_provider(spec.replace("MODULE", module), tmp_path) + + +def test_provider_without_source_is_no_tool(tmp_path: Path) -> None: + provider = type("Generated", (ContentProvider,), {"__module__": "not_a_loaded_module"})() + assert provider_script(provider, tmp_path) is None diff --git a/tests/_collect/test_facts.py b/tests/_collect/test_facts.py new file mode 100644 index 0000000..4340c36 --- /dev/null +++ b/tests/_collect/test_facts.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +import hashlib +import platform +import sys +import time +from typing import TYPE_CHECKING + +import pytest + +from python_sbom._collect._facts import REVISION_FILE, collect_facts, default_revision, git_revision +from python_sbom._errors import SettingsError +from python_sbom._model._inputs import Interpreter, RecordRow, Revision + +if TYPE_CHECKING: + from pathlib import Path + + from conftest import Git, Install + + from python_sbom._model._inputs import BuildFacts + + +def test_epoch_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + facts = collect_facts(tmp_path, no_revision, path=[], default_epoch=0) + assert facts.epoch == 1_700_000_000 + assert facts.epoch_from_environment + + +def no_revision(_root: Path, /) -> None: + return None + + +def test_epoch_default(tmp_path: Path) -> None: + facts = collect_facts(tmp_path, no_revision, path=[], default_epoch=42) + assert (facts.epoch, facts.epoch_from_environment) == (42, False) + + +def test_empty_epoch_is_unset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "") + facts = collect_facts(tmp_path, no_revision, path=[], default_epoch=42) + assert (facts.epoch, facts.epoch_from_environment) == (42, False) + + +def test_epoch_without_default_is_now(tmp_path: Path) -> None: + before = int(time.time()) + assert before <= collect_facts(tmp_path, no_revision, path=[]).epoch <= time.time() + + +def test_invalid_epoch(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "yesterday") + with pytest.raises(SettingsError, match="SOURCE_DATE_EPOCH must be a Unix timestamp, got 'yesterday'"): + collect_facts(tmp_path, no_revision, path=[]) + + +def test_interpreter(tmp_path: Path) -> None: + assert collect_facts(tmp_path, no_revision, path=[], default_epoch=0).interpreter == Interpreter( + name=sys.implementation.name, + version=platform.python_version(), + implementation=platform.python_implementation(), + ) + + +def test_revision_reads_root(tmp_path: Path) -> None: + def reader(root: Path, /) -> Revision: + return Revision(commit=root.name, dirty=False) + + assert collect_facts(tmp_path, reader, path=[], default_epoch=0).revision == Revision( + commit=tmp_path.name, dirty=False + ) + + +def test_default_path_is_this_interpreter(tmp_path: Path) -> None: + names = { + distribution.metadata["Name"] + for distribution in collect_facts(tmp_path, no_revision, default_epoch=0).distributions + } + assert "pytest" in names + + +def test_pure_distribution_files(tmp_path: Path, install: Install) -> None: + # nested, so the console script, two levels up as installers write it, stays inside the test's directory + install( + site := tmp_path / "lib" / "site-packages", + "Name: alpha\nVersion: 1.0\n", + { + "alpha/__init__.py": b"x = 1\n", + "../../bin/alpha": b"#!/usr/bin/python\n", + "alpha-1.0.dist-info/INSTALLER": b"uv\n", + "alpha-1.0.dist-info/REQUESTED": b"", + "alpha-1.0.dist-info/direct_url.json": b'{"url": "file:///tmp/alpha.whl", "archive_info": {}}', + "alpha-1.0.dist-info/uv_cache.json": b'{"timestamp": 1}', + }, + ) + (distribution,) = facts_from(site).distributions + assert distribution.files == [ + RecordRow( + path="alpha/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"x = 1\n").hexdigest(), size="6" + ), + RecordRow( + path="alpha-1.0.dist-info/METADATA", + algorithm="SHA-256", + digest=hashlib.sha256(b"Name: alpha\nVersion: 1.0\n").hexdigest(), + size="25", + ), + RecordRow( + path="alpha-1.0.dist-info/WHEEL", + algorithm="SHA-256", + digest=hashlib.sha256(b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\n").hexdigest(), + size="41", + ), + ] + + +def facts_from(site: Path) -> BuildFacts: + return collect_facts(site, no_revision, path=[str(site)], default_epoch=0) + + +def test_platform_distribution_lists_no_files(tmp_path: Path, install: Install) -> None: + install(tmp_path, "Name: native\nVersion: 1.0\n", {"native.so": b"\x7fELF"}, purelib=False) + assert facts_from(tmp_path).distributions[0].files == [] + + +def test_editable_distribution_lists_no_files(tmp_path: Path, install: Install) -> None: + editable = '{"url": "file:///checkout", "dir_info": {"editable": true}}' + install(tmp_path, "Name: local\nVersion: 1.0\n", {"local.pth": b"/checkout/src\n"}, direct_url=editable) + assert facts_from(tmp_path).distributions[0].files == [] + + +def test_first_distribution_on_path_wins(tmp_path: Path, install: Install) -> None: + install(first := tmp_path / "first", "Name: alpha\nVersion: 1.0\n", {}) + install(second := tmp_path / "second", "Name: Alpha\nVersion: 2.0\n", {}) + (distribution,) = collect_facts( + tmp_path, no_revision, path=[str(first), str(second)], default_epoch=0 + ).distributions + assert distribution.metadata["Version"] == "1.0" + + +def test_depends_on_installed_requirements_that_apply(tmp_path: Path, install: Install) -> None: + install( + tmp_path, + "Name: alpha\nVersion: 1.0\nRequires-Dist: Beta>=1\nRequires-Dist: gamma; python_version < '3'\n" + "Requires-Dist: delta; extra == 'cli'\nRequires-Dist: missing\n", + {}, + ) + for name in ("beta", "gamma", "delta"): + install(tmp_path, f"Name: {name}\nVersion: 1.0\n", {}) + by_name = {distribution.metadata["Name"]: distribution for distribution in facts_from(tmp_path).distributions} + assert by_name["alpha"].depends_on == ["beta"] + + +def test_egg_info_distribution(tmp_path: Path) -> None: + (egg_info := tmp_path / "legacy-1.0-py3.14.egg-info").mkdir() + (egg_info / "PKG-INFO").write_text("Metadata-Version: 2.1\nName: Legacy\nVersion: 1.0\n", encoding="utf-8") + (distribution,) = facts_from(tmp_path).distributions + assert (distribution.name, distribution.metadata["Version"], distribution.files) == ("legacy", "1.0", []) + + +@pytest.mark.parametrize( + "files", + [ + pytest.param({}, id="without-metadata"), + pytest.param({"METADATA": "Name: broken\n"}, id="without-version"), + ], +) +def test_distribution_without_name_and_version_left_out( + tmp_path: Path, install: Install, files: dict[str, str] +) -> None: + install(tmp_path, "Name: ok\nVersion: 1.0\n", {}) + (broken := tmp_path / "broken-1.0.dist-info").mkdir() + for name, text in files.items(): + (broken / name).write_text(text, encoding="utf-8") + assert [distribution.name for distribution in facts_from(tmp_path).distributions] == ["ok"] + + +def test_invalid_requirement_left_out(tmp_path: Path, install: Install) -> None: + install(tmp_path, "Name: old\nVersion: 1.0\nRequires-Dist: pytz (>dev)\nRequires-Dist: pytz\n", {}) + install(tmp_path, "Name: pytz\nVersion: 1.0\n", {}) + with pytest.warns(UserWarning, match=r"left out the requirement 'pytz \(>dev\)' of old: "): + by_name = {distribution.name: distribution for distribution in facts_from(tmp_path).distributions} + assert by_name["old"].depends_on == ["pytz"] + + +@pytest.mark.parametrize("commit", [pytest.param("a" * 40, id="sha1"), pytest.param("b" * 64, id="sha256")]) +def test_commit_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, commit: str) -> None: + monkeypatch.setenv("PYTHON_SBOM_COMMIT", commit) + assert default_revision(tmp_path) == Revision(commit=commit, dirty=False) + + +@pytest.mark.parametrize( + "commit", + [ + pytest.param("c0ffee", id="abbreviated"), + pytest.param("A" * 40, id="uppercase"), + pytest.param("a" * 41, id="between-lengths"), + pytest.param(f"{'a' * 40}/../../evil", id="path"), + ], +) +def test_commit_from_environment_must_be_full_hash( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, commit: str +) -> None: + monkeypatch.setenv("PYTHON_SBOM_COMMIT", commit) + with pytest.raises(SettingsError, match="PYTHON_SBOM_COMMIT must be a full lowercase hex commit hash"): + default_revision(tmp_path) + + +def test_commit_from_environment_wins_over_git(checkout: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PYTHON_SBOM_COMMIT", "a" * 40) + assert default_revision(checkout) == Revision(commit="a" * 40, dirty=False) + + +@pytest.fixture +def checkout(tmp_path: Path, git: Git) -> Path: + (root := tmp_path / "checkout").joinpath("pyproject.toml").write_text("[project]\n", encoding="utf-8") + (root / ".gitignore").write_text("build/\n", encoding="utf-8") + git("add", ".") + git("commit", "--quiet", "--message", "first") + return root + + +def test_revision_file(tmp_path: Path) -> None: + (tmp_path / REVISION_FILE).write_text(f"node: {'d' * 40}\n", encoding="utf-8") + assert default_revision(tmp_path) == Revision(commit="d" * 40, dirty=False) + + +def test_invalid_revision_file(tmp_path: Path) -> None: + (tmp_path / REVISION_FILE).write_text("node: d\n", encoding="utf-8") + with pytest.raises(SettingsError, match=r"\.python-sbom-revision must be a full lowercase hex commit hash"): + default_revision(tmp_path) + + +def test_git_revision(checkout: Path, git: Git) -> None: + assert default_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=False) + + +def test_git_revision_dirty(checkout: Path, git: Git) -> None: + (checkout / "pyproject.toml").write_text("[project]\nname = 'changed'\n", encoding="utf-8") + assert git_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=True) + + +def test_untracked_file_marks_revision_dirty(checkout: Path) -> None: + # the build packages it, so the commit alone no longer describes the source + (checkout / "backdoor.py").write_text("import os\n", encoding="utf-8") + assert (revision := git_revision(checkout)) is not None + assert revision.dirty + + +def test_ignored_file_leaves_revision_clean(checkout: Path) -> None: + (checkout / "build").mkdir() + (checkout / "build" / "output.txt").write_text("new\n", encoding="utf-8") + assert (revision := git_revision(checkout)) is not None + assert not revision.dirty + + +@pytest.fixture +def monorepo(tmp_path: Path, git: Git) -> Path: + for project in ("one", "two"): + (root := tmp_path / "checkout" / project).mkdir() + (root / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + git("add", ".") + git("commit", "--quiet", "--message", "first") + return tmp_path / "checkout" + + +def test_git_revision_of_project_in_subdirectory(monorepo: Path, git: Git) -> None: + assert git_revision(monorepo / "one") == Revision(commit=git("rev-parse", "HEAD"), dirty=False) + + +def test_sibling_changes_leave_revision_clean(monorepo: Path) -> None: + (monorepo / "two" / "pyproject.toml").write_text("[project]\nname = 'changed'\n", encoding="utf-8") + (monorepo / "two" / "new.py").write_text("", encoding="utf-8") + assert (revision := git_revision(monorepo / "one")) is not None + assert not revision.dirty + + +def test_project_untracked_by_enclosing_checkout_uses_archival(checkout: Path) -> None: + # an sdist unpacked under another project's build directory is not that project's commit + (project := checkout / "build" / "project").mkdir(parents=True) + (project / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + (project / ".git_archival.txt").write_text(f"node: {'e' * 40}\n", encoding="utf-8") + assert default_revision(project) == Revision(commit="e" * 40, dirty=False) + + +@pytest.mark.parametrize( + "project", + [pytest.param(".", id="own-checkout"), pytest.param("build/project", id="enclosing-checkout")], +) +def test_git_runs_no_fsmonitor_command(tmp_path: Path, checkout: Path, git: Git, project: str) -> None: + git("config", "core.fsmonitor", f"touch {(marker := tmp_path / 'ran').as_posix()}; false") + (root := checkout / project).mkdir(parents=True, exist_ok=True) + (root / "pyproject.toml").touch() + git_revision(root) + assert not marker.exists() + + +def test_git_revision_before_first_commit(tmp_path: Path, git: Git) -> None: + (tmp_path / "checkout" / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + git("add", "pyproject.toml") + assert git_revision(tmp_path / "checkout") is None + + +def test_git_revision_outside_checkout(tmp_path: Path) -> None: + assert git_revision(tmp_path) is None + + +def test_git_revision_without_git(checkout: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PATH", "") + assert git_revision(checkout) is None + + +@pytest.mark.parametrize( + "git_file", + [ + # a worktree copied without the main checkout, as into a container + pytest.param("gitdir: {root}/missing\n", id="missing-gitdir"), + # the marker uv leaves in an unpacked sdist it builds from + pytest.param("", id="empty"), + ], +) +def test_git_failure_warns_and_falls_back(tmp_path: Path, git_file: str) -> None: + (tmp_path / ".git").write_text(git_file.format(root=tmp_path.as_posix()), encoding="utf-8") + (tmp_path / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + (tmp_path / ".git_archival.txt").write_text(f"node: {'f' * 40}\n", encoding="utf-8") + with pytest.warns(UserWarning, match="git cannot read the source revision in .*: fatal: "): + assert default_revision(tmp_path) == Revision(commit="f" * 40, dirty=False) + + +def test_git_status_failure_leaves_revision_unknown(checkout: Path, git: Git) -> None: + # a tree object lost from the store, so git knows HEAD but cannot compare the checkout with it + tree = git("rev-parse", "HEAD^{tree}") + (stored := checkout / ".git" / "objects" / tree[:2] / tree[2:]).chmod(0o644) + stored.unlink() + with pytest.warns(UserWarning, match="git cannot read the source revision in .*: error: bad tree object HEAD"): + assert git_revision(checkout) is None + + +@pytest.mark.parametrize( + ("archival", "expected"), + [ + pytest.param(f"node: {'a' * 40}\n", Revision(commit="a" * 40, dirty=False), id="expanded"), + pytest.param("node: $Format:%H$\n", None, id="unexpanded"), + pytest.param(f"node: {'a' * 41}\n", None, id="not-a-commit"), + ], +) +def test_git_archival(tmp_path: Path, archival: str, expected: Revision | None) -> None: + (tmp_path / ".git_archival.txt").write_text(archival, encoding="utf-8") + assert default_revision(tmp_path) == expected + + +def test_no_revision_source(tmp_path: Path) -> None: + assert default_revision(tmp_path) is None diff --git a/tests/_collect/test_collect_settings.py b/tests/_collect/test_settings.py similarity index 93% rename from tests/_collect/test_collect_settings.py rename to tests/_collect/test_settings.py index 17020c0..fa085b1 100644 --- a/tests/_collect/test_collect_settings.py +++ b/tests/_collect/test_settings.py @@ -4,12 +4,14 @@ import pytest -from python_sbom import Settings, SettingsError, settings_from_pyproject, settings_from_table +from python_sbom._collect._settings import settings_from_pyproject, settings_from_table +from python_sbom._errors import SettingsError +from python_sbom._model._inputs import Settings if TYPE_CHECKING: from pathlib import Path - from python_sbom import Toml + from python_sbom._collect._settings import Toml def test_missing_pyproject(tmp_path: Path) -> None: diff --git a/tests/_collect/test_source.py b/tests/_collect/test_source.py new file mode 100644 index 0000000..137460b --- /dev/null +++ b/tests/_collect/test_source.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import base64 +import dataclasses +import hashlib +import zipfile +from email import message_from_string +from itertools import starmap +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom._collect._source import read_bundled_wheel, read_source_tree, wheel_request +from python_sbom._errors import WheelError +from python_sbom._model._inputs import BuildScript, RecordRow, SbomRequest + +if TYPE_CHECKING: + from collections.abc import Mapping + + from conftest import WriteWheel + + from python_sbom._model._inputs import Settings + +_INIT: Final[str] = "pip/__init__.py" +_METADATA: Final[str] = "pip-1.0.dist-info/METADATA" +_PIP: Final[dict[str, bytes]] = {_INIT: b"", _METADATA: b"Name: pip\nVersion: 1.0\n"} + + +# defined before the parametrized cases below, which build their RECORD rows with it at import time +def row(path: str, content: bytes, algorithm: str = "sha256", *, encoded: str = "", size: int | None = None) -> str: + encoded = encoded or base64.urlsafe_b64encode(hashlib.new(algorithm, content).digest()).rstrip(b"=").decode() + return f"{path},{algorithm}={encoded},{len(content) if size is None else size}\n" + + +_METADATA_ROW: Final[str] = row(_METADATA, _PIP[_METADATA]) + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + pytest.param({}, None, id="no-license-file"), + pytest.param({"LICENSE": "MIT terms\n"}, None, id="no-copyright-line"), + pytest.param( + {"LICENSE": "MIT\n\nCopyright (c) 2026 Example\nCopyright (c) 2025 Other\n"}, + "Copyright (c) 2026 Example", + id="first-copyright-line", + ), + ], +) +def test_copyright(tmp_path: Path, settings: Settings, files: Mapping[str, str], expected: str | None) -> None: + for name, text in files.items(): + (tmp_path / name).write_text(text, encoding="utf-8") + source = read_source_tree(request(tmp_path, settings, "License-File: LICENSE\nLicense-File: NOTICE\n")) + assert source.copyright == expected + + +def request(root: Path, settings: Settings, metadata: str = "", scripts: tuple[PurePosixPath, ...] = ()) -> SbomRequest: + return SbomRequest( + root=root, + metadata=message_from_string(f"Name: example\nVersion: 1.0\n{metadata}"), + wheel_name="example-1.0-py3-none-any.whl", + settings=settings, + build_scripts=scripts, + ) + + +def test_bundled_wheels_sorted_once(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: + for directory in ("b", "a"): + write_wheel(tmp_path / "src" / directory / "pip-1.0-py3-none-any.whl", _PIP) + settings = dataclasses.replace(settings, bundled_wheels=("src/*/*.whl", "src/a/*.whl")) + paths = [wheel.path for wheel in read_source_tree(request(tmp_path, settings)).bundled] + assert paths == ["src/a/pip-1.0-py3-none-any.whl", "src/b/pip-1.0-py3-none-any.whl"] + + +def test_build_scripts_hashed(tmp_path: Path, settings: Settings) -> None: + (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") + script = PurePosixPath("hatch_build.py") + source = read_source_tree(request(tmp_path, settings, scripts=(script,))) + assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] + + +def test_build_script_outside_project(tmp_path: Path, settings: Settings) -> None: + # a custom hook path may be absolute, which on Windows carries a drive the join with the root must keep + (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") + script = PurePosixPath((tmp_path / "hatch_build.py").as_posix()) + source = read_source_tree(request(tmp_path / "project", settings, scripts=(script,))) + assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] + + +def test_bundled_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: + wheel = write_wheel( + tmp_path / "src" / "pip-1.0-py3-none-any.whl", + { + **_PIP, + "pip/_vendor/vendor.txt": b"# pinned\n\nurllib3==1.26.4 # http\n", + "pip/_vendor/rich-13.0.dist-info/METADATA": b"Name: rich\nVersion: 13.0\n", + }, + ) + bundled = read_bundled_wheel(wheel.read_bytes(), "src/pip-1.0-py3-none-any.whl") + assert (bundled.path, bundled.sha256, bundled.metadata_name, bundled.metadata["Name"], bundled.files[0]) == ( + "src/pip-1.0-py3-none-any.whl", + hashlib.sha256(wheel.read_bytes()).hexdigest(), + "pip-1.0.dist-info/METADATA", + "pip", + RecordRow(path="pip/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"").hexdigest(), size="0"), + ) + assert [(source, str(metadata)) for source, metadata in bundled.vendored] == [ + ("pip/_vendor/vendor.txt", "Name: urllib3\nVersion: 1.26.4\n\n"), + ("pip/_vendor/rich-13.0.dist-info/METADATA", "Name: rich\nVersion: 13.0\n\n"), + ] + + +@pytest.mark.parametrize( + ("files", "message"), + [ + pytest.param({"pip/__init__.py": b""}, "has no .dist-info/METADATA", id="no-metadata"), + pytest.param({"pip-1.0.dist-info/METADATA": b"Name: pip\n"}, "METADATA lacks Name or Version", id="no-version"), + pytest.param( + {"pip-1.0.dist-info/METADATA": _PIP["pip-1.0.dist-info/METADATA"]}, + "has no pip-1.0.dist-info/RECORD", + id="no-record", + ), + pytest.param( + {"pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n" + b" " * 16 * 1024 * 1024}, + "METADATA is over the 16777216 byte limit", + id="oversized-metadata", + ), + ], +) +def test_malformed_bundled_wheel( + tmp_path: Path, write_wheel: WriteWheel, files: Mapping[str, bytes], message: str +) -> None: + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + with pytest.raises(WheelError, match=message): + read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +def test_corrupt_bundled_wheel() -> None: + with pytest.raises(WheelError, match=r"bundled wheel pip\.whl is not a valid zip archive"): + read_bundled_wheel(b"not a zip", "pip.whl") + + +def test_bundled_wheel_with_duplicate_member(tmp_path: Path) -> None: + # installers extract the first copy, so a RECORD that hashes the second one would hide the installed code + with zipfile.ZipFile(wheel := tmp_path / "pip-1.0-py3-none-any.whl", "w") as archive: + archive.writestr(_INIT, b"import os\n") + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr(_INIT, b"") + archive.writestr(_METADATA, _PIP[_METADATA]) + archive.writestr("pip-1.0.dist-info/RECORD", f"{row(_INIT, b'')}{_METADATA_ROW}") + with pytest.raises(WheelError, match=f"holds more than one member named {_INIT}, ignoring case"): + read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +@pytest.mark.parametrize( + ("algorithm", "expected"), + [ + pytest.param("sha256", "SHA-256", id="sha256"), + pytest.param("sha384", "SHA-384", id="sha384"), + pytest.param("sha512", "SHA-512", id="sha512"), + ], +) +def test_bundled_wheel_record_algorithm(tmp_path: Path, write_wheel: WriteWheel, algorithm: str, expected: str) -> None: + record = "".join(row(path, content, algorithm) for path, content in _PIP.items()) + files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + assert [ + (file.path, file.algorithm, file.digest) + for file in read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files + ] == [(path, expected, hashlib.new(algorithm, content).hexdigest()) for path, content in _PIP.items()] + + +@pytest.mark.parametrize( + ("record", "message"), + [ + pytest.param(f"{_INIT},sha256=x\n", "RECORD row 1 has 2 columns, not 3", id="columns"), + pytest.param(f"{_INIT},,0\n", f"RECORD row 1 gives no digest for {_INIT}", id="no-digest"), + pytest.param(_METADATA_ROW, f"RECORD lacks a digest for {_INIT}", id="member-not-recorded"), + pytest.param( + f"{row(_INIT, b'', encoded='A' * 43)}{row(_INIT, b'')}{_METADATA_ROW}", + f"RECORD lists {_INIT} more than once", + id="listed-twice", + ), + pytest.param( + f"{row(_INIT, b'')}{row(_INIT, b'', 'sha512', encoded='A' * 86)}{_METADATA_ROW}", + f"RECORD lists {_INIT} more than once", + id="listed-twice-by-other-algorithm", + ), + pytest.param( + f"{row(_INIT, b'')}{row('pip/ghost.py', b'', 'md5', size=99)}{_METADATA_ROW}", + "RECORD lists pip/ghost.py, which the wheel does not hold", + id="recorded-not-member", + ), + pytest.param( + f"{row(_INIT, b'', size=999999)}{_METADATA_ROW}", + f"the size in RECORD differs from the size of {_INIT}", + id="size-mismatch", + ), + pytest.param( + f"{row(_INIT, b'', 'sha512', encoded='A' * 86)}{_METADATA_ROW}", + f"the digest in RECORD differs from the content of {_INIT}", + id="digest-mismatch", + ), + ], +) +def test_record_disagrees_with_wheel(tmp_path: Path, write_wheel: WriteWheel, record: str, message: str) -> None: + files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + with pytest.raises(WheelError, match=message): + read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +def test_record_signatures_need_no_row(tmp_path: Path, write_wheel: WriteWheel) -> None: + record = "".join(starmap(row, _PIP.items())) + files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), "pip-1.0.dist-info/RECORD.jws": b"{}"} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + assert [file.path for file in read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files] == list( + _PIP + ) + + +def test_vendored_metadata_needs_name_and_version(tmp_path: Path, write_wheel: WriteWheel) -> None: + files = {**_PIP, "pip/_vendor/rich-13.0.dist-info/METADATA": b"Metadata-Version: 2.1\n"} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files) + with pytest.raises(WheelError, match=r"pip/_vendor/rich-13\.0\.dist-info/METADATA lacks Name or Version"): + read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +@pytest.mark.parametrize( + ("manifest", "message"), + [ + pytest.param(b"urllib3 ==\n", "does not parse: urllib3 ==", id="unparsable"), + pytest.param(b"urllib3>=1\n", "pins no one exact version: urllib3>=1", id="range"), + pytest.param(b"urllib3==1.*\n", "pins no one exact version: urllib3==1.*", id="wildcard"), + pytest.param(b"urllib3==1,!=1.1\n", "pins no one exact version: urllib3==1,!=1.1", id="two-specifiers"), + ], +) +def test_vendor_manifest_needs_exact_pins( + tmp_path: Path, write_wheel: WriteWheel, manifest: bytes, message: str +) -> None: + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", {**_PIP, "pip/_vendor/vendor.txt": manifest}) + with pytest.raises(WheelError, match=message): + read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +def test_wheel_request(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: + wheel = write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"Name: native\n"}) + request = wheel_request(wheel, tmp_path, settings, (PurePosixPath("build.py"),)) + assert (request.root, request.metadata["Name"], request.wheel_name, request.settings, request.build_scripts) == ( + tmp_path, + "native", + "native-1.0-py3-none-any.whl", + settings, + (PurePosixPath("build.py"),), + ) + + +def test_wheel_request_needs_a_wheel(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: + archive = write_wheel(tmp_path / "native.whl", {"native/__init__.py": b""}, record=False) + with pytest.raises(WheelError, match=r"has no \.dist-info/METADATA, so it is not a wheel"): + wheel_request(archive, Path(), settings, ()) diff --git a/tests/_formats/test_formats_spdx.py b/tests/_formats/test_spdx.py similarity index 58% rename from tests/_formats/test_formats_spdx.py rename to tests/_formats/test_spdx.py index c20574f..4078f57 100644 --- a/tests/_formats/test_formats_spdx.py +++ b/tests/_formats/test_spdx.py @@ -1,22 +1,44 @@ from __future__ import annotations import dataclasses +import json import re -from typing import TYPE_CHECKING +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Final, TypeAlias import pytest -from python_sbom import to_spdx +from python_sbom._formats._json import dumps +from python_sbom._formats._spdx import render_spdx, to_spdx +from python_sbom._validate._schemas import spdx_problems if TYPE_CHECKING: from conftest import Describe - from python_sbom import Component, Settings + from python_sbom import Component + from python_sbom._formats._spdx import _Package + from python_sbom._model._inputs import Settings + +# a JSON value, as json.loads hands it over +_Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" + +# what auditwheel writes: CycloneDX 1.4 conventions, a legacy tools array, no timestamp and no serial number +_FOREIGN: Final[str] = json.dumps({ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "version": 1, + "metadata": { + "component": {"type": "library", "bom-ref": "pkg:pypi/plain@1.0", "name": "plain", "version": "1.0"}, + "tools": [{"name": "auditwheel", "version": "6.4.0"}, {"name": "repair"}, {"vendor": "PyPA"}], + }, + "components": [{"type": "library", "bom-ref": "pkg:rpm/almalinux/libfoo@1", "name": "libfoo", "version": "1"}], + "dependencies": [{"ref": "pkg:pypi/plain@1.0", "dependsOn": ["pkg:rpm/almalinux/libfoo@1"]}], +}) def test_minimal(describe: Describe) -> None: document = describe() - spdx = to_spdx(document, "1.2.3") + spdx = render_spdx(document, "1.2.3") ids = {package["name"]: package["SPDXID"] for package in spdx["packages"]} serial = document["serialNumber"] assert spdx == { @@ -81,14 +103,14 @@ def test_minimal(describe: Describe) -> None: def test_spdx_ids_are_valid_and_tell_similar_refs_apart(describe: Describe) -> None: document = describe("Requires-Dist: click; os_name == 'nt'\nRequires-Dist: click; os_name != 'nt'\n") - ids = [package["SPDXID"] for package in to_spdx(document, "1")["packages"]] + ids = [package["SPDXID"] for package in render_spdx(document, "1")["packages"]] assert all(re.fullmatch(r"SPDXRef-[A-Za-z0-9.-]+", spdx_id) for spdx_id in ids) assert len(set(ids)) == len(ids) def test_repository_names_the_namespace(describe: Describe, settings: Settings) -> None: document = describe(settings=dataclasses.replace(settings, repository="https://github.com/example/example")) - namespace = to_spdx(document, "1")["documentNamespace"] + namespace = render_spdx(document, "1")["documentNamespace"] assert namespace.startswith("https://github.com/example/example/sboms/Example-1.0-") @@ -98,7 +120,7 @@ def test_supplier_and_application_tools_are_creators(describe: Describe, setting {"type": "application", "bom-ref": "tool:hatchling", "name": "hatchling", "version": "1.28"}, {"type": "application", "bom-ref": "tool:script", "name": "script"}, ]) - assert to_spdx(document, "1")["creationInfo"]["creators"] == [ + assert render_spdx(document, "1")["creationInfo"]["creators"] == [ "Organization: Example Org", "Tool: hatchling-1.28", "Tool: script", @@ -109,7 +131,7 @@ def test_supplier_and_application_tools_are_creators(describe: Describe, setting def test_versionless_root(describe: Describe) -> None: document = describe() del document["metadata"]["component"]["version"] - spdx = to_spdx(document, "1") + spdx = render_spdx(document, "1") assert spdx["name"] == "Example" assert "versionInfo" not in spdx["packages"][0] @@ -133,7 +155,7 @@ def test_package_fields(describe: Describe) -> None: "components": [{"type": "file", "bom-ref": "bundled:a.whl#a.py", "name": "a.py"}], } document["components"].append(component) - (package,) = (package for package in to_spdx(document, "1")["packages"] if package["name"] == "a") + (package,) = (package for package in render_spdx(document, "1")["packages"] if package["name"] == "a") assert {key: value for key, value in package.items() if key != "SPDXID"} == { "name": "a", "downloadLocation": "NOASSERTION", @@ -153,7 +175,7 @@ def test_package_fields(describe: Describe) -> None: def test_license_refs_carry_extracted_text(describe: Describe) -> None: document = describe("License-Expression: MIT AND LicenseRef-Vendor\n") - assert to_spdx(document, "1").get("hasExtractedLicensingInfos") == [ + assert render_spdx(document, "1").get("hasExtractedLicensingInfos") == [ { "licenseId": "LicenseRef-Vendor", "extractedText": "LicenseRef-Vendor, as named in package metadata", @@ -170,7 +192,7 @@ def test_nested_libraries_are_contained(describe: Describe) -> None: "name": "outer", "components": [{"type": "library", "bom-ref": "inner", "name": "inner"}], }) - spdx = to_spdx(document, "1") + spdx = render_spdx(document, "1") ids = {package["name"]: package["SPDXID"] for package in spdx["packages"]} assert {"spdxElementId": ids["outer"], "relationshipType": "CONTAINS", "relatedSpdxElement": ids["inner"]} in spdx[ "relationships" @@ -188,5 +210,87 @@ def test_nested_libraries_are_contained(describe: Describe) -> None: def test_package_purpose(describe: Describe, kind: str, purpose: str) -> None: document = describe() document["components"].append({"type": kind, "bom-ref": "extra", "name": "extra"}) - (package,) = (package for package in to_spdx(document, "1")["packages"] if package["name"] == "extra") + (package,) = (package for package in render_spdx(document, "1")["packages"] if package["name"] == "extra") assert package["primaryPackagePurpose"] == purpose + + +def test_to_spdx_text(describe: Describe) -> None: + text = to_spdx(describe()) + assert (text.endswith("}\n"), json.loads(text)["spdxVersion"]) == (True, "SPDX-2.3") + + +def test_other_tools_document_renders_valid_spdx() -> None: + assert spdx_problems(render_spdx(json.loads(_FOREIGN), "1")) == [] + + +def test_legacy_tools_become_creators() -> None: + assert render_spdx(json.loads(_FOREIGN), "1")["creationInfo"]["creators"] == [ + "Tool: auditwheel-6.4.0", + "Tool: repair", + "Tool: python-sbom-1", + ] + + +def test_document_without_tools() -> None: + foreign = json.loads(_FOREIGN) + del foreign["metadata"]["tools"] + assert render_spdx(foreign, "1")["creationInfo"]["creators"] == ["Tool: python-sbom-1"] + + +def test_namespace_without_serial_names_the_content() -> None: + changed = json.loads(_FOREIGN) + changed["version"] = 2 + namespaces = [render_spdx(document, "1")["documentNamespace"] for document in (json.loads(_FOREIGN),) * 2] + assert (namespaces[0] == namespaces[1], namespaces[0] == render_spdx(changed, "1")["documentNamespace"]) == ( + True, + False, + ) + + +def test_creation_time_without_timestamp_is_the_rendering_time() -> None: + before = datetime.now(UTC).replace(microsecond=0) + created = datetime.fromisoformat(render_spdx(json.loads(_FOREIGN), "1")["creationInfo"]["created"]) + assert before <= created <= datetime.now(UTC) + + +def test_supplier_without_name_is_no_creator(describe: Describe) -> None: + # the schema makes the name optional, so the loaded JSON, not the typed document, loses it + loaded = json.loads(dumps(describe())) + loaded["metadata"]["supplier"] = {"url": ["https://example.org"]} + assert render_spdx(loaded, "1")["creationInfo"]["creators"] == ["Tool: python-sbom-1"] + + +@pytest.mark.parametrize( + ("algorithm", "expected"), + [ + pytest.param("SHA-1", "SHA1", id="sha1"), + pytest.param("SHA-512", "SHA512", id="sha512"), + pytest.param("SHA3-256", "SHA3-256", id="sha3"), + pytest.param("BLAKE2b-256", "BLAKE2b-256", id="blake2b"), + pytest.param("BLAKE3", "BLAKE3", id="blake3"), + ], +) +def test_hash_algorithms_take_spdx_names(algorithm: str, expected: str) -> None: + (package,) = packages_of({"hashes": [{"alg": algorithm, "content": "ab" * 32}]}) + assert package.get("checksums") == [{"algorithm": expected, "checksumValue": "ab" * 32}] + + +def packages_of(fields: dict[str, _Json]) -> list[_Package]: + document = json.loads(_FOREIGN) + document["components"] = [{"type": "library", "bom-ref": "lib", "name": "lib", **fields}] + return [package for package in render_spdx(document, "1")["packages"] if package["name"] == "lib"] + + +def test_hash_algorithm_spdx_lacks_is_dropped() -> None: + (package,) = packages_of({"hashes": [{"alg": "SHA-224", "content": "ab" * 28}]}) + assert "checksums" not in package + + +def test_component_supplier_without_name_is_dropped() -> None: + (package,) = packages_of({"supplier": {"url": ["https://example.org"]}}) + assert "supplier" not in package + + +def test_property_without_value() -> None: + (package,) = packages_of({"properties": [{"name": "plain:extra"}]}) + assert package["comment"] == "plain:extra: " diff --git a/tests/_generate/test_archive.py b/tests/_generate/test_archive.py new file mode 100644 index 0000000..aa7f755 --- /dev/null +++ b/tests/_generate/test_archive.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +import io +import json +import zipfile +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom import BundledWheelInfo, ContentProvider, Contributions, ProviderContext +from python_sbom._collect._facts import collect_facts +from python_sbom._errors import WheelError +from python_sbom._generate._archive import describe_archive +from python_sbom._generate._embed import revision_reader +from python_sbom._validate._artifact import validate + +if TYPE_CHECKING: + from conftest import WriteWheel + + from python_sbom import Component + from python_sbom._model._types import Document + +_COMMIT: Final[str] = "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" +_LIBRARY: Final[str] = "__bundled__/distlib-0.4-py3-none-any" +_WHEEL: Final[str] = "app/wheels/pip-1.0-py3-none-any.whl" + + +@pytest.fixture +def project(tmp_path: Path) -> Path: + (root := tmp_path / "project").mkdir() + (root / "pyproject.toml").write_text( + "[tool.python-sbom]\nsupplier = { name = 'Example Org' }\nrepository = 'https://github.com/example/app'\n", + encoding="utf-8", + ) + (root / "build_app.py").write_text("print('build')\n", encoding="utf-8") + return root + + +@pytest.fixture +def archive(tmp_path: Path, write_wheel: WriteWheel) -> Path: + wheel = write_wheel( + tmp_path / "pip-1.0-py3-none-any.whl", + {"pip/__init__.py": b"", "pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\nLicense-Expression: MIT\n"}, + ) + with zipfile.ZipFile(target := tmp_path / "app.pyz", "w") as handle: + handle.writestr("__main__.py", "import app\n") + handle.writestr("loaded.json", json.dumps({_LIBRARY: ["3.14"]})) + handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\nLicense-Expression: MIT\n") + handle.writestr("app/__init__.py", "") + handle.writestr("app/", "") + handle.writestr(_WHEEL, wheel.read_bytes()) + handle.writestr( + f"{_LIBRARY}/distlib-0.4.dist-info/METADATA", "Name: distlib\nVersion: 0.4\nLicense-Expression: MIT\n" + ) + handle.writestr(f"{_LIBRARY}/distlib/__init__.py", "") + handle.writestr("app.pyz.cdx.json", "{}") + return target + + +@pytest.fixture +def document(project: Path, archive: Path) -> Document: + facts = collect_facts(project, revision_reader(_COMMIT)) + return describe_archive( + project, archive, facts=facts, content=Bundles("loaded.json"), build_scripts=(PurePosixPath("build_app.py"),) + ) + + +class Bundles(ContentProvider): + def __init__(self, table: str) -> None: + self._table = table + + def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: + # reads the archive, as a real provider reads its tables + loaded = json.loads(context.read(PurePosixPath(self._table))) + return BundledWheelInfo( + properties=[{"name": "app:loaded-for", "value": python} for python in loaded.get(str(wheel), [])], + supplier=context.settings.supplier, + ) + + def contributions(self, context: ProviderContext) -> Contributions: + zlib: Component = {"type": "library", "bom-ref": "pkg:generic/zlib@1.3", "name": "zlib", "version": "1.3"} + return Contributions( + components=[zlib], + dependencies=[{"ref": "pkg:generic/zlib@1.3", "dependsOn": []}], + root_properties=[ + {"name": "extra:project", "value": context.root.name}, + {"name": "extra:table", "value": self._table}, + ], + ) + + +def test_archive_document_validates(document: Document, tmp_path: Path) -> None: + (sbom := tmp_path / "app.pyz.cdx.json").write_text(json.dumps(document), encoding="utf-8") + assert validate(sbom, frozenset({"MIT"})) == [] + + +def test_archive_root(document: Document) -> None: + root = document["metadata"]["component"] + assert {key: value for key, value in root.items() if key != "components"} == { + "type": "application", + "bom-ref": "pkg:generic/app.pyz@1.0", + "name": "app.pyz", + "version": "1.0", + "purl": "pkg:generic/app.pyz@1.0", + "supplier": {"name": "Example Org", "url": []}, + "properties": [ + {"name": "extra:project", "value": "project"}, + {"name": "extra:table", "value": "loaded.json"}, + {"name": "app:vcs-commit", "value": _COMMIT}, + ], + } + + +def test_archive_loose_files_belong_to_the_archive(document: Document) -> None: + files = document["metadata"]["component"].get("components", []) + assert [file["name"] for file in files] == ["__main__.py", "loaded.json"] + + +def test_archive_project_distribution(document: Document) -> None: + app = document["components"][0] + assert [(child["type"], child["name"]) for child in app.get("components", [])] == [ + ("file", "app-1.0.dist-info/METADATA"), + ("file", "app/__init__.py"), + ("library", "pip"), + ] + + +def test_archive_bundled_wheel(document: Document) -> None: + pip = document["components"][0].get("components", [])[2] + assert (pip["bom-ref"], pip.get("supplier"), pip.get("properties", [])[0]) == ( + f"bundled:{_WHEEL}", + {"name": "Example Org", "url": []}, + {"name": "app:bundled-wheel", "value": _WHEEL}, + ) + + +def test_archive_unpacked_wheel(document: Document) -> None: + distlib = document["components"][1] + assert ( + distlib["bom-ref"], + "supplier" in distlib, + distlib.get("properties", []), + [child["name"] for child in distlib.get("components", [])], + ) == ( + f"bundled:{_LIBRARY}", + True, + [{"name": "app:bundled-wheel", "value": _LIBRARY}, {"name": "app:loaded-for", "value": "3.14"}], + [f"{_LIBRARY}/distlib-0.4.dist-info/METADATA", f"{_LIBRARY}/distlib/__init__.py"], + ) + + +def test_archive_tools_and_metadata(document: Document) -> None: + metadata = document["metadata"] + script = metadata["tools"]["components"][0] + assert (script["name"], script.get("externalReferences"), metadata.get("properties"), metadata.get("supplier")) == ( + "build_app.py", + [{"type": "vcs", "url": "https://github.com/example/app/blob/main/build_app.py"}], + [{"name": "app:sbom:file", "value": "app.pyz.cdx.json"}], + {"name": "Example Org", "url": []}, + ) + + +def test_archive_contributions(document: Document) -> None: + assert (document["components"][-1]["name"], document["dependencies"][-1]) == ( + "zlib", + {"ref": "pkg:generic/zlib@1.3", "dependsOn": []}, + ) + + +def test_archive_defaults(tmp_path: Path, archive: Path) -> None: + (tmp_path / "pyproject.toml").write_text("[project]\nname = 'app'\n", encoding="utf-8") + document = describe_archive(tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None))) + assert ("supplier" in document["metadata"], document["metadata"]["component"].get("properties")) == (False, []) + + +def test_archive_content_provider_option(tmp_path: Path, archive: Path) -> None: + (tmp_path / "pyproject.toml").write_text( + f"[tool.python-sbom]\ncontent-provider = '{tmp_path.name}:Provider'\n", encoding="utf-8" + ) + (tmp_path / f"{tmp_path.name}.py").write_text( + "from python_sbom import ContentProvider\n\n\nclass Provider(ContentProvider):\n pass\n", encoding="utf-8" + ) + tools = describe_archive(tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None)))["metadata"][ + "tools" + ]["components"] + assert tools[0]["name"] == f"{tmp_path.name}.py" + + +def test_archive_without_project_distribution(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as handle: + handle.writestr("__main__.py", "") + (archive := tmp_path / "bare.pyz").write_bytes(buffer.getvalue()) + with pytest.raises( + WheelError, match=r"bare\.pyz holds 0 top-level \.dist-info/METADATA, not the one naming the project" + ): + describe_archive(tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None))) diff --git a/tests/_generate/test_embed.py b/tests/_generate/test_embed.py new file mode 100644 index 0000000..8eb2282 --- /dev/null +++ b/tests/_generate/test_embed.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import json +import zipfile +from typing import TYPE_CHECKING + +from python_sbom import BundledWheelInfo, ContentProvider, ProviderContext +from python_sbom._generate._embed import embed_sbom + +if TYPE_CHECKING: + from pathlib import Path, PurePosixPath + + from conftest import WriteWheel + + +def test_bundled_wheel_named_by_its_path_inside_the_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: + (root := tmp_path / "project").mkdir() + (root / "pyproject.toml").write_text("[tool.python-sbom]\nbundled-wheels = ['src/pkg/wheels/*.whl']\n") + bundled = write_wheel( + root / "src" / "pkg" / "wheels" / "pip-1.0-py3-none-any.whl", + {"pip/__init__.py": b"", "pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\nLicense-Expression: MIT\n"}, + ) + wheel = write_wheel( + tmp_path / "dist" / "pkg-1.0-py3-none-any.whl", + { + "pkg/wheels/pip-1.0-py3-none-any.whl": bundled.read_bytes(), + "pkg/wheels/table.txt": b"3.15\n", + "pkg-1.0.dist-info/METADATA": b"Name: pkg\nVersion: 1.0\n", + }, + ) + written = embed_sbom(wheel, root, content=Table("table.txt"), build_env=[str(root)]) + (pip,) = json.loads(zipfile.ZipFile(wheel).read(written))["components"] + assert (pip["bom-ref"], pip["properties"][:2]) == ( + "bundled:pkg/wheels/pip-1.0-py3-none-any.whl", + [ + {"name": "pkg:bundled-wheel", "value": "pkg/wheels/pip-1.0-py3-none-any.whl"}, + {"name": "pkg:table", "value": "3.15"}, + ], + ) + + +class Table(ContentProvider): + def __init__(self, name: str) -> None: + self._name = name + + def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: + # the table sits next to the wheel inside the built wheel + value = context.read(wheel.parent / self._name).decode().strip() + return BundledWheelInfo(properties=[{"name": f"{context.identity.property_prefix}:table", "value": value}]) diff --git a/tests/_generate/test_generate_backend.py b/tests/_generate/test_generate_backend.py deleted file mode 100644 index 2bd64e0..0000000 --- a/tests/_generate/test_generate_backend.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING - -import pytest - -from python_sbom import BuiltWheel, MetadataBackend, WheelError - -if TYPE_CHECKING: - from conftest import WriteWheel - - from python_sbom import Settings - - -def test_metadata_backend(tmp_path: Path, settings: Settings) -> None: - backend = MetadataBackend( - root=tmp_path, - metadata="Name: example\nVersion: 1.0\n", - wheel_name="example-1.0-py3-none-any.whl", - settings=settings, - build_scripts=(PurePosixPath("hatch_build.py"),), - ) - request = backend.request() - assert (request.root, request.metadata["Name"], request.wheel_name, request.settings, request.build_scripts) == ( - tmp_path, - "example", - "example-1.0-py3-none-any.whl", - settings, - (PurePosixPath("hatch_build.py"),), - ) - - -def test_built_wheel(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: - wheel = write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"Name: native\n"}) - request = BuiltWheel(wheel=wheel, root=tmp_path, settings=settings, build_scripts=()).request() - assert (request.metadata["Name"], request.wheel_name) == ("native", "native-1.0-py3-none-any.whl") - - -def test_built_wheel_needs_a_wheel(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: - archive = write_wheel(tmp_path / "native.whl", {"native/__init__.py": b""}, record=False) - with pytest.raises(WheelError, match=r"has no \.dist-info/METADATA, so it is not a wheel"): - BuiltWheel(wheel=archive, root=Path(), settings=settings, build_scripts=()).request() diff --git a/tests/_generate/test_pipeline.py b/tests/_generate/test_pipeline.py new file mode 100644 index 0000000..dc746c1 --- /dev/null +++ b/tests/_generate/test_pipeline.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from email import message_from_string +from pathlib import Path +from typing import TYPE_CHECKING + +from python_sbom import ContentProvider +from python_sbom._formats._json import dumps +from python_sbom._generate._pipeline import build_sbom +from python_sbom._model._inputs import SbomRequest, SourceTree + +if TYPE_CHECKING: + from conftest import ContextFor + + from python_sbom._model._inputs import BuildFacts, Settings + + +def test_sbom_file_name_and_content(settings: Settings, facts: BuildFacts, context_for: ContextFor) -> None: + request = SbomRequest( + root=Path(), + metadata=message_from_string("Name: My.Project\nVersion: 1.0\n"), + wheel_name="my_project-1.0-py3-none-any.whl", + settings=settings, + build_scripts=(), + ) + context = context_for(Path(), Path()) + sbom = build_sbom( + request, SourceTree(copyright=None, bundled=[], build_scripts=[]), facts, ContentProvider(), context + ) + assert (sbom.file_name, sbom.content) == ("my-project.cdx.json", dumps(sbom.document)) diff --git a/tests/_model/test_model_bom.py b/tests/_model/test_bom.py similarity index 85% rename from tests/_model/test_model_bom.py rename to tests/_model/test_bom.py index b2eab19..ef64c99 100644 --- a/tests/_model/test_model_bom.py +++ b/tests/_model/test_bom.py @@ -4,28 +4,20 @@ import json import uuid from email import message_from_string -from pathlib import Path, PurePosixPath +from pathlib import PurePosixPath from typing import TYPE_CHECKING, Final import pytest -from python_sbom import ( - BuildScript, - BundledWheel, - BundledWheelInfo, - ContentProvider, - Contributions, - Revision, - SbomRequest, - SourceTree, - build_sbom, - dumps, -) +from python_sbom import BundledWheelInfo, ContentProvider, Contributions, ProviderContext +from python_sbom._model._inputs import BuildScript, BundledWheel, Revision, SourceTree if TYPE_CHECKING: from conftest import Describe - from python_sbom import BuildFacts, Component, OrganizationalEntity, Settings + from python_sbom import Component + from python_sbom._model._inputs import BuildFacts, Settings + from python_sbom._model._types import OrganizationalEntity _INTERPRETER: Final[str] = "tool:pkg:generic/cpython@3.15.0" @@ -118,18 +110,6 @@ def test_serial_follows_content(describe: Describe) -> None: assert describe()["serialNumber"] != describe("Summary: other\n")["serialNumber"] -def test_sbom_file_name_and_content(settings: Settings, facts: BuildFacts) -> None: - request = SbomRequest( - root=Path(), - metadata=message_from_string("Name: My.Project\nVersion: 1.0\n"), - wheel_name="my_project-1.0-py3-none-any.whl", - settings=settings, - build_scripts=(), - ) - sbom = build_sbom(request, SourceTree(copyright=None, bundled=[], build_scripts=[]), facts, ContentProvider()) - assert (sbom.file_name, sbom.content) == ("my-project.cdx.json", dumps(sbom.document)) - - @pytest.mark.parametrize( ("dirty", "trail"), [ @@ -250,25 +230,6 @@ def test_repeated_requirement_is_one_component(describe: Describe) -> None: assert [component["bom-ref"] for component in document["components"]] == ["requires-dist:packaging>=24"] -@pytest.mark.parametrize( - ("marker", "applies"), - [ - pytest.param('extra == "cli"', False, id="extra"), - pytest.param('"cli" == extra', False, id="extra-on-the-right"), - pytest.param('extra != "cli"', True, id="not-extra"), - pytest.param('python_version < "3.11"', True, id="environment"), - pytest.param('extra == "cli" or python_version < "3.11"', True, id="extra-or-environment"), - pytest.param('extra == "cli" and python_version < "3.11"', False, id="extra-and-environment"), - pytest.param('(extra == "a" or extra == "b") and os_name == "nt"', False, id="grouped-extras"), - pytest.param('python_version < "3.11" and (extra == "a" or os_name == "nt")', True, id="grouped-unknown"), - pytest.param('python_version >= "3" or extra == "a"', True, id="environment-or-extra"), - ], -) -def test_requirement_marker(describe: Describe, marker: str, applies: bool) -> None: - document = describe(f"Requires-Dist: click; {marker}\n") - assert bool(document["components"]) is applies - - def test_scope_names_bundled_directories(describe: Describe, settings: Settings) -> None: document = describe(settings=dataclasses.replace(settings, bundled_wheels=("src/b/*.whl", "src/a/x/*.whl"))) assert document["metadata"]["properties"][0]["value"] == ( @@ -291,12 +252,12 @@ class Seeded(ContentProvider): def __init__(self, name: str) -> None: self._name = name - def bundled_wheel(self, wheel: BundledWheel) -> BundledWheelInfo: - return BundledWheelInfo(properties=[{"name": self._name, "value": wheel.path}], supplier=None) + def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: + return BundledWheelInfo( + properties=[{"name": f"{context.identity.property_prefix}:{self._name}", "value": str(wheel)}] + ) - document = describe( - source=SourceTree(copyright=None, bundled=[wheel], build_scripts=[]), content=Seeded("example:seeded") - ) + document = describe(source=SourceTree(copyright=None, bundled=[wheel], build_scripts=[]), content=Seeded("seeded")) assert document["components"][0]["properties"][1] == { "name": "example:seeded", "value": "src/pip-1.0-py3-none-any.whl", @@ -351,11 +312,11 @@ class Native(ContentProvider): def __init__(self, components: list[Component]) -> None: self._components = components - def contributions(self, request: SbomRequest) -> Contributions: + def contributions(self, context: ProviderContext) -> Contributions: return Contributions( components=self._components, dependencies=[{"ref": component["bom-ref"], "dependsOn": []} for component in self._components], - root_properties=[{"name": "example:native", "value": request.wheel_name}], + root_properties=[{"name": "native:prefix", "value": context.identity.property_prefix}], tools=[tool], ) @@ -372,8 +333,42 @@ def contributions(self, request: SbomRequest) -> Contributions: {"ref": "pkg:pypi/example@1.0", "dependsOn": ["pkg:generic/zlib@1.3"]}, {"ref": "pkg:generic/zlib@1.3", "dependsOn": []}, ], - [{"name": "example:native", "value": "example-1.0-py3-none-any.whl"}], + [{"name": "native:prefix", "value": "example"}], tool, [{"ref": _INTERPRETER}, {"ref": "tool:pkg:generic/cmake@4"}], ) assert native["serialNumber"] != describe()["serialNumber"] + + +_ORG: Final[OrganizationalEntity] = {"name": "Example Org", "url": []} + + +@pytest.mark.parametrize( + ("supplied", "expected"), + [pytest.param(True, _ORG, id="by-project"), pytest.param(False, None, id="unnamed")], +) +def test_bundled_wheel_supplier_is_the_projects_choice( + describe: Describe, settings: Settings, supplied: bool, expected: OrganizationalEntity | None +) -> None: + wheel = BundledWheel( + path="src/pip-1.0-py3-none-any.whl", + sha256="ab" * 32, + metadata_name="pip-1.0.dist-info/METADATA", + metadata=message_from_string("Name: pip\nVersion: 1.0\nProject-URL: Source, https://github.com/example/pip\n"), + files=[], + vendored=[], + ) + + class Supplied(ContentProvider): + def __init__(self, supplied: bool) -> None: + self._supplied = supplied + + def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: # ruff: ignore[unused-method-argument] # the base class's signature + return BundledWheelInfo(supplier=context.settings.supplier if self._supplied else None) + + document = describe( + settings=dataclasses.replace(settings, supplier=_ORG), + source=SourceTree(copyright=None, bundled=[wheel], build_scripts=[]), + content=Supplied(supplied), + ) + assert document["components"][0].get("supplier") == expected diff --git a/tests/_model/test_model_components.py b/tests/_model/test_components.py similarity index 85% rename from tests/_model/test_model_components.py rename to tests/_model/test_components.py index 8455a69..bed0b85 100644 --- a/tests/_model/test_model_components.py +++ b/tests/_model/test_components.py @@ -7,18 +7,17 @@ import pytest -from python_sbom import ( - BundledWheel, - BundledWheelInfo, - RecordRow, +from python_sbom._model._components import ( bundled_wheel_component, component_from_metadata, file_component_from_bytes, script_tool, ) +from python_sbom._model._inputs import BundledWheel, RecordRow if TYPE_CHECKING: - from python_sbom import Component, License, OrganizationalEntity + from python_sbom import Component, Property + from python_sbom._model._types import License, OrganizationalEntity _WHEEL: Final[str] = "src/example/wheels/pip-1.0-py3-none-any.whl" @@ -149,6 +148,36 @@ def test_component_ignores_license_text() -> None: [{"type": "issue-tracker", "url": "https://example.com/issues", "comment": "Project-URL: Bug Tracker"}], id="normalized-label", ), + pytest.param( + "Project-URL: Source Code, https://example.com/x\n", + [{"type": "vcs", "url": "https://example.com/x", "comment": "Project-URL: Source Code"}], + id="pep-753-source-code", + ), + pytest.param( + "Project-URL: Bugs, https://example.com/x\n", + [{"type": "issue-tracker", "url": "https://example.com/x", "comment": "Project-URL: Bugs"}], + id="pep-753-bugs", + ), + pytest.param( + "Project-URL: Issue, https://example.com/x\n", + [{"type": "issue-tracker", "url": "https://example.com/x", "comment": "Project-URL: Issue"}], + id="pep-753-issue", + ), + pytest.param( + "Project-URL: Release Notes, https://example.com/x\n", + [{"type": "release-notes", "url": "https://example.com/x", "comment": "Project-URL: Release Notes"}], + id="pep-753-release-notes", + ), + pytest.param( + "Project-URL: What's New, https://example.com/x\n", + [{"type": "release-notes", "url": "https://example.com/x", "comment": "Project-URL: What's New"}], + id="pep-753-whats-new", + ), + pytest.param( + "Project-URL: History, https://example.com/x\n", + [{"type": "release-notes", "url": "https://example.com/x", "comment": "Project-URL: History"}], + id="pep-753-history", + ), pytest.param( "Project-URL: Funding, https://example.com/fund\n", [{"type": "other", "url": "https://example.com/fund", "comment": "Project-URL: Funding"}], @@ -171,8 +200,8 @@ def test_bundled_wheel_component() -> None: vendored=[], ) supplier: OrganizationalEntity = {"name": "PyPA", "url": ["https://www.pypa.io"]} - info = BundledWheelInfo(properties=[{"name": "example:seeded-for-python", "value": "3.15"}], supplier=supplier) - assert bundled_wheel_component(wheel, "example", info) == { + properties: list[Property] = [{"name": "example:seeded-for-python", "value": "3.15"}] + assert bundled_wheel_component(wheel, "example", properties, supplier) == { "type": "library", "bom-ref": f"bundled:{_WHEEL}", "name": "pip", @@ -227,7 +256,7 @@ def test_bundled_wheel_vendored_metadata_wins_over_manifest() -> None: (dist_info, message_from_string("Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n")), ], ) - component = bundled_wheel_component(wheel, "example", BundledWheelInfo(properties=[], supplier=None)) + component = bundled_wheel_component(wheel, "example", [], None) assert component["components"] == [ { "type": "library", @@ -261,7 +290,7 @@ def component(path: str) -> Component: files=[], vendored=[], ) - return bundled_wheel_component(wheel, "example", BundledWheelInfo(properties=[], supplier=None)) + return bundled_wheel_component(wheel, "example", [], None) refs = {component(f"src/{directory}/pip-1.0-py3-none-any.whl")["bom-ref"] for directory in ("a", "b")} assert refs == {"bundled:src/a/pip-1.0-py3-none-any.whl", "bundled:src/b/pip-1.0-py3-none-any.whl"} @@ -276,7 +305,7 @@ def test_bundled_wheel_supplier_never_comes_from_the_wheel() -> None: files=[], vendored=[], ) - component = bundled_wheel_component(wheel, "example", BundledWheelInfo(properties=[], supplier=None)) + component = bundled_wheel_component(wheel, "example", [], None) assert "supplier" not in component diff --git a/tests/_model/test_content.py b/tests/_model/test_content.py new file mode 100644 index 0000000..0a969a0 --- /dev/null +++ b/tests/_model/test_content.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import zipfile +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +import pytest + +from python_sbom import BundledWheelInfo, ContentProvider, Contributions + +if TYPE_CHECKING: + from conftest import ContextFor + + +def test_default_provider_adds_nothing_to_bundled_wheels(context_for: ContextFor) -> None: + info = ContentProvider().bundled_wheel(context_for(Path(), Path()), PurePosixPath("a.whl")) + assert info == BundledWheelInfo() + + +def test_default_provider_contributes_nothing(context_for: ContextFor) -> None: + assert ContentProvider().contributions(context_for(Path(), Path())) == Contributions() + + +def test_context_carries_project(context_for: ContextFor, tmp_path: Path) -> None: + context = context_for(tmp_path, tmp_path) + assert (context.root, context.identity.property_prefix, context.settings.default_branch) == ( + tmp_path, + "example", + "main", + ) + + +def test_context_reads_a_directory(tmp_path: Path, context_for: ContextFor) -> None: + (tmp_path / "embed").mkdir() + (tmp_path / "embed" / "table.py").write_bytes(b"x = 1\n") + assert context_for(tmp_path, tmp_path).read(PurePosixPath("embed/table.py")) == b"x = 1\n" + + +def test_context_reads_a_zip(tmp_path: Path, context_for: ContextFor) -> None: + with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: + archive.writestr("embed/table.py", "x = 1\n") + assert context_for(tmp_path, wheel).read(PurePosixPath("embed/table.py")) == b"x = 1\n" + + +@pytest.mark.parametrize("artifact", [pytest.param("dir", id="directory"), pytest.param("a.whl", id="zip")]) +def test_context_misses(tmp_path: Path, context_for: ContextFor, artifact: str) -> None: + (tmp_path / "dir").mkdir() + with zipfile.ZipFile(tmp_path / "a.whl", "w") as archive: + archive.writestr("other.py", "") + with pytest.raises(KeyError): + context_for(tmp_path, tmp_path / artifact).read(PurePosixPath("missing.py")) diff --git a/tests/_model/test_document.py b/tests/_model/test_document.py new file mode 100644 index 0000000..6cf9626 --- /dev/null +++ b/tests/_model/test_document.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import json +import uuid +from typing import TYPE_CHECKING + +import pytest + +from python_sbom._errors import SbomError +from python_sbom._model._document import contributed_properties, with_header + +if TYPE_CHECKING: + from python_sbom import Property + from python_sbom._model._types import Body + + +def test_document_serial_derives_from_body() -> None: + body: Body = { + "metadata": { + "timestamp": "2023-11-14T22:13:20+00:00", + "lifecycles": [{"phase": "build"}], + "tools": {"components": []}, + "component": {"type": "application", "bom-ref": "app", "name": "app"}, + }, + "components": [], + "dependencies": [], + } + namespace = uuid.uuid5(uuid.NAMESPACE_URL, "https://example.org/sboms") + assert with_header(body, namespace) == { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": f"urn:uuid:{uuid.uuid5(namespace, json.dumps(body, sort_keys=True))}", + "version": 1, + **body, + } + + +def test_header_refuses_repeated_refs() -> None: + body: Body = { + "metadata": { + "timestamp": "2023-11-14T22:13:20+00:00", + "lifecycles": [{"phase": "build"}], + "tools": {"components": [{"type": "application", "bom-ref": "app", "name": "tool"}]}, + "component": {"type": "application", "bom-ref": "app", "name": "app"}, + }, + "components": [], + "dependencies": [], + } + with pytest.raises(SbomError, match="bom-ref values repeat: app"): + with_header(body, uuid.uuid4()) + + +def test_header_refuses_unknown_dependency() -> None: + body: Body = { + "metadata": { + "timestamp": "2023-11-14T22:13:20+00:00", + "lifecycles": [{"phase": "build"}], + "tools": {"components": []}, + "component": {"type": "application", "bom-ref": "app", "name": "app"}, + }, + "components": [], + "dependencies": [{"ref": "app", "dependsOn": ["ghost"]}], + } + with pytest.raises(SbomError, match="dependencies name bom-refs no component has: ghost"): + with_header(body, uuid.uuid4()) + + +def test_contributed_properties_keep_their_own_prefix() -> None: + properties: list[Property] = [{"name": "extra:key", "value": "1"}] + assert contributed_properties(properties, "example") == properties + + +def test_contributed_properties_refuse_the_project_prefix() -> None: + with pytest.raises(SbomError, match="example:vcs-commit use the example: prefix python-sbom writes"): + contributed_properties([{"name": "example:vcs-commit", "value": "0"}], "example") diff --git a/tests/_model/test_model_document.py b/tests/_model/test_identity.py similarity index 57% rename from tests/_model/test_model_document.py rename to tests/_model/test_identity.py index cc38683..6aa3616 100644 --- a/tests/_model/test_model_document.py +++ b/tests/_model/test_identity.py @@ -1,17 +1,16 @@ from __future__ import annotations import dataclasses -import json import uuid from email import message_from_string from typing import TYPE_CHECKING import pytest -from python_sbom import ProjectIdentity, document, project_identity +from python_sbom._model._identity import ProjectIdentity, project_identity if TYPE_CHECKING: - from python_sbom import Body, Settings + from python_sbom._model._inputs import Settings @pytest.mark.parametrize( @@ -38,29 +37,7 @@ def test_project_identity_repository( metadata = message_from_string(f"Name: My_Project\nVersion: 1.0\n{urls}") identity = project_identity(metadata, dataclasses.replace(settings, repository=repository)) assert identity == ProjectIdentity( - namespace="my-project", + property_prefix="my-project", repository=expected, serial_namespace=uuid.uuid5(uuid.NAMESPACE_URL, f"{expected or 'https://pypi.org/project/my-project'}/sboms"), ) - - -def test_document_serial_derives_from_body() -> None: - body: Body = { - "metadata": { - "timestamp": "2023-11-14T22:13:20+00:00", - "lifecycles": [{"phase": "build"}], - "tools": {"components": []}, - "component": {"type": "application", "bom-ref": "app", "name": "app"}, - }, - "components": [], - "dependencies": [], - } - namespace = uuid.uuid5(uuid.NAMESPACE_URL, "https://example.org/sboms") - assert document(body, namespace) == { - "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", - "bomFormat": "CycloneDX", - "specVersion": "1.6", - "serialNumber": f"urn:uuid:{uuid.uuid5(namespace, json.dumps(body, sort_keys=True))}", - "version": 1, - **body, - } diff --git a/tests/_model/test_licenses.py b/tests/_model/test_licenses.py new file mode 100644 index 0000000..494896a --- /dev/null +++ b/tests/_model/test_licenses.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from email import message_from_string +from typing import TYPE_CHECKING + +import pytest + +from python_sbom._model._licenses import licenses + +if TYPE_CHECKING: + from python_sbom import License + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + pytest.param( + "mit or apache-2.0", [{"expression": "MIT OR Apache-2.0", "acknowledgement": "declared"}], id="canonical" + ), + pytest.param( + "MIT WITH GPL-3.0-only", + [{"license": {"name": "MIT WITH GPL-3.0-only", "acknowledgement": "declared"}}], + id="invalid-kept-as-name", + ), + ], +) +def test_license_expression(expression: str, expected: list[License]) -> None: + assert licenses(message_from_string(f"License-Expression: {expression}\n")) == expected diff --git a/tests/_model/test_markers.py b/tests/_model/test_markers.py new file mode 100644 index 0000000..92eeff3 --- /dev/null +++ b/tests/_model/test_markers.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import pytest +from packaging.requirements import Requirement + +from python_sbom._model._markers import applies_without_extras + + +@pytest.mark.parametrize( + ("requirement", "applies"), + [ + pytest.param('click; extra == "cli"', False, id="extra"), + pytest.param('click; "cli" == extra', False, id="extra-on-the-right"), + pytest.param('click; extra != "cli"', True, id="not-extra"), + pytest.param('click; python_version < "3.11"', True, id="environment"), + pytest.param('click; extra == "cli" or python_version < "3.11"', True, id="extra-or-environment"), + pytest.param('click; extra == "cli" and python_version < "3.11"', False, id="extra-and-environment"), + pytest.param('click; (extra == "a" or extra == "b") and os_name == "nt"', False, id="grouped-extras"), + pytest.param( + 'click; python_version < "3.11" and (extra == "a" or os_name == "nt")', True, id="grouped-unknown" + ), + pytest.param('click; python_version >= "3" or extra == "a"', True, id="environment-or-extra"), + pytest.param("click", True, id="no-marker"), + ], +) +def test_applies_without_extras(requirement: str, applies: bool) -> None: + assert applies_without_extras(Requirement(requirement)) is applies diff --git a/tests/_model/test_model_tools.py b/tests/_model/test_tools.py similarity index 76% rename from tests/_model/test_model_tools.py rename to tests/_model/test_tools.py index 4019b03..d2fff7f 100644 --- a/tests/_model/test_model_tools.py +++ b/tests/_model/test_tools.py @@ -4,17 +4,22 @@ from email import message_from_string from typing import TYPE_CHECKING -from python_sbom import InstalledDistribution, RecordRow, tool_components +from python_sbom._model._inputs import InstalledDistribution, RecordRow +from python_sbom._model._tools import tool_components if TYPE_CHECKING: - from python_sbom import BuildFacts + from python_sbom._model._inputs import BuildFacts def test_tools_list_interpreter_then_distributions_by_name(facts: BuildFacts) -> None: row = RecordRow(path="zeta/__init__.py", algorithm="SHA-256", digest="ab" * 32, size="0") distributions = [ - InstalledDistribution(metadata=message_from_string("Name: Zeta\nVersion: 2\n"), files=[row], depends_on=[]), - InstalledDistribution(metadata=message_from_string("Name: alpha\nVersion: 1\n"), files=[], depends_on=["zeta"]), + InstalledDistribution( + name="zeta", metadata=message_from_string("Name: Zeta\nVersion: 2\n"), files=[row], depends_on=[] + ), + InstalledDistribution( + name="alpha", metadata=message_from_string("Name: alpha\nVersion: 1\n"), files=[], depends_on=["zeta"] + ), ] tools, dependencies = tool_components(dataclasses.replace(facts, distributions=distributions)) assert tools[1:] == [ diff --git a/tests/_seeder/hatch_build.py b/tests/_seeder/hatch_build.py index c2877e1..4475a61 100644 --- a/tests/_seeder/hatch_build.py +++ b/tests/_seeder/hatch_build.py @@ -1,14 +1,15 @@ from __future__ import annotations import ast -from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING from python_sbom import BundledWheelInfo, ContentProvider from python_sbom.hatch import SbomBuildHook if TYPE_CHECKING: - from python_sbom import BundledWheel, Settings + from pathlib import PurePosixPath + + from python_sbom import ProviderContext def get_build_hook() -> type[SeedSbomHook]: @@ -17,37 +18,32 @@ def get_build_hook() -> type[SeedSbomHook]: class SeedSbomHook(SbomBuildHook): - def content_provider( - self, - settings: Settings, # ruff: ignore[unused-method-argument] # the base class's signature - ) -> tuple[ContentProvider, tuple[PurePosixPath, ...]]: - # no script to add: the SBOM already lists this one, the hook, among the build tools - return BundleSupport(Path(self.root), self.metadata.name), () + def content_provider(self) -> ContentProvider: # ruff: ignore[no-self-use] # overrides the hook's factory + return BundleSupport("BUNDLE_SUPPORT") class BundleSupport(ContentProvider): - def __init__(self, root: Path, project: str) -> None: - self._root = root - self._project = project - - def bundled_wheel(self, wheel: BundledWheel) -> BundledWheelInfo: - # the BUNDLE_SUPPORT table next to the wheels decides which Python versions receive each one; it is read by - # name, without importing the package - path = PurePosixPath(wheel.path) - module = ast.parse((self._root / str(path.parent) / "__init__.py").read_text(encoding="utf-8")) + def __init__(self, table: str) -> None: + self._table = table + + def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: + # the table next to the wheels decides which Python versions receive each one; it is read by name, without + # importing the package + module = ast.parse(context.read(wheel.parent / "__init__.py").decode("utf-8")) support: dict[str, dict[str, str]] = ast.literal_eval( next( node.value for node in module.body if isinstance(node, ast.Assign) - and any(isinstance(target, ast.Name) and target.id == "BUNDLE_SUPPORT" for target in node.targets) + and any(isinstance(target, ast.Name) and target.id == self._table for target in node.targets) ) ) return BundledWheelInfo( properties=[ - {"name": f"{self._project}:seeded-for-python", "value": python} + {"name": f"{context.identity.property_prefix}:seeded-for-python", "value": python} for python, wheels in support.items() - if path.name in wheels.values() + if wheel.name in wheels.values() ], - supplier=None, + # the seeder bundles only its own supplier's wheels + supplier=context.settings.supplier, ) diff --git a/tests/_seeder/pyproject.toml b/tests/_seeder/pyproject.toml index 338f2af..7289986 100644 --- a/tests/_seeder/pyproject.toml +++ b/tests/_seeder/pyproject.toml @@ -21,6 +21,9 @@ classifiers = [ urls.Source = "https://github.com/example/seeder" [tool.hatch] -build.hooks.custom.supplier = { name = "Example Org", url = [ "https://example.org" ] } -build.hooks.custom.bundled-wheels = [ "src/seeder/embed/*.whl" ] +build.hooks.custom = {} build.targets.wheel.packages = [ "src/seeder" ] + +[tool.python-sbom] +supplier = { name = "Example Org", url = [ "https://example.org" ] } +bundled-wheels = [ "src/seeder/embed/*.whl" ] diff --git a/tests/_validate/test_artifact.py b/tests/_validate/test_artifact.py new file mode 100644 index 0000000..1336d55 --- /dev/null +++ b/tests/_validate/test_artifact.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import json +import zipfile +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom._errors import WheelError +from python_sbom._formats._json import dumps +from python_sbom._validate._artifact import validate + +if TYPE_CHECKING: + from pathlib import Path + + from conftest import Describe, WriteWheel + +_METADATA: Final[dict[str, bytes]] = {"example-1.0.dist-info/METADATA": b"Name: Example\nVersion: 1.0\n"} + + +def test_archive_without_sbom(tmp_path: Path, write_wheel: WriteWheel) -> None: + wheel = write_wheel(tmp_path / "example-1.0-py3-none-any.whl", _METADATA) + assert validate(wheel, None) == ["no SBOM found in example-1.0-py3-none-any.whl"] + + +def test_every_sbom_is_checked(tmp_path: Path, write_wheel: WriteWheel, describe: Describe) -> None: + # the schema makes the serial optional, so the loaded JSON, not the typed document, loses it + broken = json.loads(dumps(describe())) + del broken["serialNumber"] + foreign = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1}).encode() + wheel = write_wheel( + tmp_path / "example-1.0-py3-none-any.whl", + { + **_METADATA, + "example-1.0.dist-info/sboms/example.cdx.json": json.dumps(broken).encode(), + "example-1.0.dist-info/sboms/other.cdx.json": foreign, + "example-1.0.dist-info/sboms/notes.txt": b"not an SBOM", + "app.pyz.cdx.json": foreign, + "example/nested.cdx.json": json.dumps(broken).encode(), + "example/sboms/nested.cdx.json": json.dumps(broken).encode(), + }, + ) + assert validate(wheel, None) == [ + "example-1.0.dist-info/sboms/example.cdx.json: serialNumber must be a urn:uuid: RFC 4122 UUID, got ''" + ] + + +def test_sbom_of_every_dist_info_is_checked(tmp_path: Path, write_wheel: WriteWheel, describe: Describe) -> None: + # a decoy .dist-info sorting first must not hide the SBOM of the wheel's own one + broken = json.loads(dumps(describe())) + del broken["serialNumber"] + wheel = write_wheel( + tmp_path / "example-1.0-py3-none-any.whl", + { + "aaa-0.dist-info/METADATA": b"Name: aaa\nVersion: 0\n", + "aaa-0.dist-info/sboms/aaa.cdx.json": dumps(describe()), + **_METADATA, + "example-1.0.dist-info/sboms/example.cdx.json": json.dumps(broken).encode(), + }, + record=False, + ) + assert validate(wheel, None) == [ + "example-1.0.dist-info/sboms/example.cdx.json: serialNumber must be a urn:uuid: RFC 4122 UUID, got ''" + ] + + +def test_oversized_sbom(tmp_path: Path, write_wheel: WriteWheel) -> None: + sbom = b" " * (64 * 1024 * 1024 + 1) + wheel = write_wheel(tmp_path / "example-1.0-py3-none-any.whl", {**_METADATA, "example.cdx.json": sbom}) + with pytest.raises(WheelError, match=r"example\.cdx\.json is over the 67108864 byte limit"): + validate(wheel, None) + + +def test_archive_with_duplicate_member(tmp_path: Path) -> None: + with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: + archive.writestr("example.cdx.json", b"{}") + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr("example.cdx.json", b"{}") + with pytest.raises(WheelError, match=r"holds more than one member named example\.cdx\.json"): + validate(wheel, None) + + +def test_corrupt_archive(tmp_path: Path) -> None: + with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("example.cdx.json", b"{}" * 1000) + data = bytearray(wheel.read_bytes()) + data[40:50] = b"\xff" * 10 + wheel.write_bytes(bytes(data)) + with pytest.raises(WheelError, match=r"example-1\.0-py3-none-any\.whl is not a valid zip archive"): + validate(wheel, None) + + +def test_json_file(tmp_path: Path, describe: Describe) -> None: + (sbom := tmp_path / "example.cdx.json").write_bytes(dumps(describe())) + assert validate(sbom, frozenset({"MIT"})) == [] diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py new file mode 100644 index 0000000..d3f0d8f --- /dev/null +++ b/tests/_validate/test_document.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Final, TypeAlias + +import pytest + +from python_sbom._formats._json import dumps +from python_sbom._validate._document import is_document, validate_document + +if TYPE_CHECKING: + from collections.abc import Callable + + from conftest import Describe + from pytest_mock import MockerFixture + + from python_sbom import Component, License + from python_sbom._model._types import Document + +# a JSON value, as json.loads hands it over +_Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" +# what auditwheel writes: CycloneDX 1.4 conventions, a legacy tools array, no timestamp +_FOREIGN: Final[str] = json.dumps({ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "version": 1, + "metadata": { + "component": {"type": "library", "bom-ref": "pkg:pypi/plain@1.0", "name": "plain", "version": "1.0"}, + "tools": [{"name": "auditwheel", "version": "6.4.0"}], + }, + "components": [{"type": "library", "bom-ref": "pkg:rpm/almalinux/libfoo@1", "name": "libfoo", "version": "1"}], + "dependencies": [{"ref": "pkg:pypi/plain@1.0", "dependsOn": ["pkg:rpm/almalinux/libfoo@1"]}], +}) + + +def test_valid(describe: Describe) -> None: + assert validate_document(dumps(describe("Requires-Dist: click\nLicense-Expression: MIT\n")).decode(), None) == [] + + +def test_not_json() -> None: + assert validate_document("this is not json", None) == ["is not JSON: Expecting value: line 1 column 1 (char 0)"] + + +def test_schema_violation() -> None: + (problem,) = validate_document(json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"}), None) + assert problem == "does not conform to the CycloneDX 1.6 schema: 'one' is not of type 'integer' at $.version" + + +def test_other_tools_documents_get_the_schema_check_alone() -> None: + assert validate_document(_FOREIGN, None) == [] + + +def test_oversized_document_is_not_parsed() -> None: + assert validate_document(" " * (16 * 1024 * 1024 + 1), None) == [ + "is 16777217 characters long, over the 16777216 character limit" + ] + + +def test_long_components_array_skips_the_schema() -> None: + # the schema compares every pair of array items, which takes minutes at this length + components = [{"type": "library", "bom-ref": f"ref-{index}", "name": "many"} for index in range(10_001)] + raw = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components}) + assert validate_document(raw, None) == [ + "lists 10001 components in one array, over the 10000 the schema check can take" + ] + + +def test_missing_fields(describe: Describe) -> None: + # the schema makes the timestamp optional, so the loaded JSON, not the typed document, loses it + loaded = json.loads(dumps(describe())) + del loaded["metadata"]["timestamp"] + assert validate_document(json.dumps(loaded), None) == [ + "lacks metadata.timestamp, which every document python-sbom writes carries" + ] + + +def test_serial_is_required(describe: Describe) -> None: + # the schema makes it optional, so the loaded JSON, not the typed document, loses it + loaded = json.loads(dumps(describe())) + del loaded["serialNumber"] + assert validate_document(json.dumps(loaded), None) == ["serialNumber must be a urn:uuid: RFC 4122 UUID, got ''"] + + +def test_duplicate_refs(describe: Describe) -> None: + library: Component = {"type": "library", "bom-ref": "pkg:pypi/example@1.0", "name": "copy"} + assert validate_document(with_components(describe(), library), None) == [ + "bom-ref values are not unique: ['pkg:pypi/example@1.0']" + ] + + +def with_components(document: Document, *components: Component) -> str: + document["components"].extend(components) + document["dependencies"][0].setdefault("dependsOn", []).extend(component["bom-ref"] for component in components) + return dumps(document).decode() + + +def test_duplicate_refs_in_other_tools_documents() -> None: + foreign = json.loads(_FOREIGN) + foreign["components"].append({"type": "library", "bom-ref": "pkg:pypi/plain@1.0", "name": "copy"}) + assert validate_document(json.dumps(foreign), None) == ["bom-ref values are not unique: ['pkg:pypi/plain@1.0']"] + + +def test_component_without_bom_ref(describe: Describe) -> None: + # the schema makes the bom-ref optional, so the loaded JSON, not the typed document, loses it + loaded = json.loads(dumps(describe())) + loaded["components"].append({"type": "library", "name": "nameless"}) + assert validate_document(json.dumps(loaded), None) == ["components lack a bom-ref: ['nameless']"] + + +@pytest.mark.parametrize( + "mutate", + [ + pytest.param( + lambda loaded: loaded["formulation"][0]["workflows"][0].pop("resourceReferences"), + id="workflow-without-resources", + ), + pytest.param( + lambda loaded: loaded["formulation"][0]["workflows"][0]["resourceReferences"].append({ + "externalReference": {"type": "other", "url": "https://example.org"} + }), + id="workflow-external-resource", + ), + pytest.param( + lambda loaded: loaded["metadata"]["properties"].append({"name": "example:extra"}), + id="property-without-value", + ), + pytest.param( + lambda loaded: loaded["metadata"]["component"]["properties"].append({"name": "example:extra"}), + id="root-property-without-value", + ), + pytest.param( + lambda loaded: loaded["metadata"].update(supplier={"url": ["https://example.org"]}), + id="supplier-without-name", + ), + ], +) +def test_schema_valid_variations_pass(describe: Describe, mutate: Callable[[dict[str, _Json]], None]) -> None: + loaded = json.loads(dumps(describe())) + mutate(loaded) + assert validate_document(json.dumps(loaded), None) == [] + + +def test_unknown_dependency_refs(describe: Describe) -> None: + document = describe() + document["dependencies"].append({"ref": "ghost", "dependsOn": ["phantom"]}) + assert validate_document(dumps(document).decode(), None) == [ + "dependencies reference bom-refs that do not exist: ['ghost', 'phantom']" + ] + + +def test_root_must_depend_on_components(describe: Describe) -> None: + document = describe("Requires-Dist: click\n") + document["dependencies"][0]["dependsOn"] = [] + assert validate_document(dumps(document).decode(), None) == [ + "root dependsOn [] != components ['requires-dist:click']" + ] + + +def test_workflow_must_reference_tools(describe: Describe) -> None: + document = describe() + document.get("formulation", [])[0]["workflows"][0]["resourceReferences"].append({"ref": "pkg:pypi/example@1.0"}) + assert validate_document(dumps(document).decode(), None) == [ + "workflow wheel-build references unknown tools: ['pkg:pypi/example@1.0']" + ] + + +def test_formula_without_workflows(describe: Describe) -> None: + loaded = json.loads(dumps(describe())) + loaded["formulation"] = [{"bom-ref": "formula:other"}] + assert validate_document(json.dumps(loaded), None) == [] + + +def test_spdx_rendering_must_validate(describe: Describe) -> None: + document = describe() + properties = document["metadata"].get("properties", []) + properties.append({"name": "example:sbom:repository", "value": "https://example.org/a#fragment"}) + (problem,) = validate_document(dumps(document).decode(), None) + assert problem.startswith("SPDX rendering invalid: ") + + +def test_spdx_rendering_must_parse(describe: Describe) -> None: + raw = with_components(describe(), shipped([{"expression": "MIT AND", "acknowledgement": "declared"}])) + assert any(problem.startswith("SPDX rendering does not parse: ") for problem in validate_document(raw, None)) + + +def shipped(licenses: list[License]) -> Component: + return { + "type": "library", + "bom-ref": "bundled:lib.whl", + "name": "lib", + "hashes": [{"alg": "SHA-256", "content": "ab" * 32}], + "licenses": licenses, + } + + +def test_deep_nesting_is_reported(describe: Describe) -> None: + nested = "".join( + f'{{"type": "library", "bom-ref": "deep-{level}", "name": "deep", "components": [' for level in range(500) + ) + nested += "]}" * 500 + raw = json.dumps(describe()).replace('"components": []', f'"components": [{nested}]', 1) + assert validate_document(raw, None) == [ + "does not conform to the CycloneDX 1.6 schema: it nests too deeply to check" + ] + + +def test_license_policy(describe: Describe) -> None: + raw = with_components(describe(), shipped([{"expression": "MIT AND GPL-3.0-only"}])) + assert validate_document(raw, frozenset({"MIT"})) == [ + "bundled:lib.whl: license MIT AND GPL-3.0-only is outside the allowed licenses ['MIT']" + ] + + +def test_unexpected_error_is_reported(describe: Describe, mocker: MockerFixture) -> None: + mocker.patch("python_sbom._validate._document.render_spdx", autospec=True, side_effect=RuntimeError("x" * 300)) + assert validate_document(dumps(describe()).decode(), None) == [f"could not be checked, RuntimeError: {'x' * 200}"] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + pytest.param(json.loads(_FOREIGN), True, id="foreign"), + pytest.param({"metadata": {"component": {"name": "plain"}}}, False, id="root-without-bom-ref"), + pytest.param({"metadata": {"component": {"bom-ref": "plain"}}}, False, id="root-without-name"), + pytest.param({"metadata": {}}, False, id="no-root"), + pytest.param([], False, id="not-an-object"), + ], +) +def test_is_document(value: _Json, expected: bool) -> None: + assert is_document(value) is expected diff --git a/tests/_validate/test_licenses.py b/tests/_validate/test_licenses.py new file mode 100644 index 0000000..b29fc80 --- /dev/null +++ b/tests/_validate/test_licenses.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom._validate._licenses import license_problems + +if TYPE_CHECKING: + from python_sbom import Component, License + +_ALLOWED: Final[frozenset[str]] = frozenset({"MIT", "BSD-3-Clause", "GPL-2.0-only"}) + + +@pytest.mark.parametrize( + "licenses", + [ + pytest.param([{"expression": "MIT"}], id="expression"), + pytest.param([{"license": {"id": "mit", "acknowledgement": "concluded"}}], id="id-case-insensitive"), + pytest.param([{"expression": "MIT OR GPL-3.0-only"}], id="any-of"), + pytest.param([{"expression": "(GPL-3.0-only OR MIT) AND BSD-3-Clause"}], id="grouped"), + pytest.param([{"expression": "GPL-2.0-only WITH Classpath-exception-2.0"}], id="exception"), + ], +) +def test_allows(licenses: list[License]) -> None: + assert license_problems([shipped(licenses)], _ALLOWED) == [] + + +def shipped(licenses: list[License]) -> Component: + return { + "type": "library", + "bom-ref": "bundled:lib.whl", + "name": "lib", + "hashes": [{"alg": "SHA-256", "content": "ab" * 32}], + "licenses": licenses, + } + + +@pytest.mark.parametrize( + ("licenses", "problem"), + [ + pytest.param( + [{"expression": "MIT AND GPL-3.0-only"}], + "license MIT AND GPL-3.0-only is outside the allowed licenses ['BSD-3-Clause', 'GPL-2.0-only', 'MIT']", + id="denied", + ), + pytest.param( + [{"license": {"name": "BSD License", "acknowledgement": "declared"}}], + "declares license names ['BSD License'] with no SPDX id", + id="names", + ), + pytest.param([], "declares no license", id="none"), + pytest.param([{"expression": "MIT AND"}], "license expression 'MIT AND' is malformed: it ends early", id="end"), + pytest.param([{"expression": "(MIT"}], "license expression '(MIT' is malformed: it ends early", id="open"), + pytest.param( + [{"expression": "(MIT ISC)"}], + "license expression '(MIT ISC)' is malformed: expected ')', got 'ISC'", + id="pair", + ), + pytest.param([{"expression": "MIT)"}], "license expression 'MIT)' is malformed: unexpected ')'", id="close"), + pytest.param( + [{"expression": "AND MIT"}], "license expression 'AND MIT' is malformed: unexpected 'AND'", id="op" + ), + pytest.param( + [{"expression": "MIT WITH OR"}], "license expression 'MIT WITH OR' is malformed: unexpected 'OR'", id="with" + ), + ], +) +def test_reports(licenses: list[License], problem: str) -> None: + assert license_problems([shipped(licenses)], _ALLOWED) == [f"bundled:lib.whl: {problem}"] + + +def test_deep_nesting_is_malformed() -> None: + # each level costs the parser stack frames, so this deep a nesting would exhaust the stack + expression = f"{'(' * 5000}MIT{')' * 5000}" + assert license_problems([shipped([{"expression": expression}])], _ALLOWED) == [ + ( + f"bundled:lib.whl: license expression {expression!r} is malformed: it nests 5000 levels deep, over the 32" + " level limit" + ) + ] + + +def test_nesting_up_to_the_limit_parses() -> None: + expression = f"{'(' * 32}MIT{')' * 32}" + assert license_problems([shipped([{"expression": expression}])], _ALLOWED) == [] + + +def test_judges_only_shipped_components() -> None: + declared: Component = {"type": "library", "bom-ref": "requires-dist:click", "name": "click"} + vendored: Component = {"type": "library", "bom-ref": "bundled:outer.whl#vendored", "name": "vendored"} + outer = shipped([{"expression": "MIT"}]) + outer["components"] = [vendored, {"type": "file", "bom-ref": "outer#a.py", "name": "a.py"}] + assert license_problems([declared, outer], frozenset({"MIT"})) == [] + + +def test_component_without_bom_ref_is_named() -> None: + # the schema makes the bom-ref optional, so the loaded JSON, not the typed document, loses it + component = json.loads(json.dumps(shipped([]))) + del component["bom-ref"] + assert license_problems([component], _ALLOWED) == ["lib: declares no license"] diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py new file mode 100644 index 0000000..9221ff9 --- /dev/null +++ b/tests/_validate/test_schemas.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import json +import sys + +import pytest + +from python_sbom._errors import MissingExtraError +from python_sbom._validate._document import validate_document +from python_sbom._validate._schemas import schema_error + + +def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch) -> None: + # None in sys.modules makes the import fail, as when the extra is not installed + monkeypatch.setitem(sys.modules, "cyclonedx.schema", None) + monkeypatch.delitem(sys.modules, "python_sbom._validate._schemas", raising=False) + with pytest.raises(MissingExtraError, match=r"pip install 'python-sbom\[validate\]'"): + validate_document("{}", None) + + +def test_schema_error_names_the_path() -> None: + raw = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"}) + assert schema_error(raw) == "'one' is not of type 'integer' at $.version" + + +def test_schema_error_clips_the_offending_value() -> None: + raw = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "x" * 1000}) + assert schema_error(raw) == f"'{'x' * 199} at $.version" diff --git a/tests/_validate/test_validate_document.py b/tests/_validate/test_validate_document.py deleted file mode 100644 index 8be907e..0000000 --- a/tests/_validate/test_validate_document.py +++ /dev/null @@ -1,184 +0,0 @@ -from __future__ import annotations - -import json -from typing import TYPE_CHECKING, Final - -import pytest - -from python_sbom import dumps, validate - -if TYPE_CHECKING: - from conftest import Describe - - from python_sbom import Component, Document, License - -# what auditwheel writes: CycloneDX 1.4 conventions, a legacy tools array, no timestamp -_FOREIGN: Final[str] = json.dumps({ - "bomFormat": "CycloneDX", - "specVersion": "1.4", - "version": 1, - "metadata": { - "component": {"type": "library", "bom-ref": "pkg:pypi/plain@1.0", "name": "plain", "version": "1.0"}, - "tools": [{"name": "auditwheel", "version": "6.4.0"}], - }, - "components": [{"type": "library", "bom-ref": "pkg:rpm/almalinux/libfoo@1", "name": "libfoo", "version": "1"}], - "dependencies": [{"ref": "pkg:pypi/plain@1.0", "dependsOn": ["pkg:rpm/almalinux/libfoo@1"]}], -}) -_ALLOWED: Final[frozenset[str]] = frozenset({"MIT", "BSD-3-Clause", "GPL-2.0-only"}) - - -def test_valid(describe: Describe) -> None: - assert validate(dumps(describe("Requires-Dist: click\nLicense-Expression: MIT\n")).decode(), None) == [] - - -def test_schema_violation() -> None: - (problem,) = validate(json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"}), None) - assert problem.startswith("does not conform to the CycloneDX 1.6 schema: ") - - -def test_other_tools_documents_get_the_schema_check_alone() -> None: - assert validate(_FOREIGN, None) == [] - - -def test_missing_fields(describe: Describe) -> None: - # the schema makes the timestamp optional, so the loaded JSON, not the typed document, loses it - loaded = json.loads(dumps(describe())) - del loaded["metadata"]["timestamp"] - assert validate(json.dumps(loaded), None) == [ - "lacks metadata.timestamp, which every document python-sbom writes carries" - ] - - -def test_serial_is_required(describe: Describe) -> None: - # the schema makes it optional, so the loaded JSON, not the typed document, loses it - loaded = json.loads(dumps(describe())) - del loaded["serialNumber"] - assert validate(json.dumps(loaded), None) == ["serialNumber must be a urn:uuid: RFC 4122 UUID, got ''"] - - -def test_duplicate_refs(describe: Describe) -> None: - library: Component = {"type": "library", "bom-ref": "pkg:pypi/example@1.0", "name": "copy"} - assert validate(with_components(describe(), library), None) == [ - "bom-ref values are not unique: ['pkg:pypi/example@1.0']" - ] - - -def with_components(document: Document, *components: Component) -> str: - document["components"].extend(components) - document["dependencies"][0].setdefault("dependsOn", []).extend(component["bom-ref"] for component in components) - return dumps(document).decode() - - -def test_unknown_dependency_refs(describe: Describe) -> None: - document = describe() - document["dependencies"].append({"ref": "ghost", "dependsOn": ["phantom"]}) - assert validate(dumps(document).decode(), None) == [ - "dependencies reference bom-refs that do not exist: ['ghost', 'phantom']" - ] - - -def test_root_must_depend_on_components(describe: Describe) -> None: - document = describe("Requires-Dist: click\n") - document["dependencies"][0]["dependsOn"] = [] - assert validate(dumps(document).decode(), None) == ["root dependsOn [] != components ['requires-dist:click']"] - - -def test_workflow_must_reference_tools(describe: Describe) -> None: - document = describe() - document.get("formulation", [])[0]["workflows"][0]["resourceReferences"].append({"ref": "pkg:pypi/example@1.0"}) - assert validate(dumps(document).decode(), None) == [ - "workflow wheel-build references unknown tools: ['pkg:pypi/example@1.0']" - ] - - -def test_formula_without_workflows(describe: Describe) -> None: - loaded = json.loads(dumps(describe())) - loaded["formulation"] = [{"bom-ref": "formula:other"}] - assert validate(json.dumps(loaded), None) == [] - - -def test_spdx_rendering_must_validate(describe: Describe) -> None: - document = describe() - properties = document["metadata"].get("properties", []) - properties.append({"name": "example:sbom:repository", "value": "https://example.org/a#fragment"}) - (problem,) = validate(dumps(document).decode(), None) - assert problem.startswith("SPDX rendering invalid: ") - - -def test_spdx_rendering_must_parse(describe: Describe) -> None: - raw = with_components(describe(), shipped("broken", [{"expression": "MIT AND", "acknowledgement": "declared"}])) - assert any(problem.startswith("SPDX rendering does not parse: ") for problem in validate(raw, None)) - - -def shipped(name: str, licenses: list[License]) -> Component: - return { - "type": "library", - "bom-ref": f"bundled:{name}.whl", - "name": name, - "hashes": [{"alg": "SHA-256", "content": "ab" * 32}], - "licenses": licenses, - } - - -def test_deep_nesting_is_reported(describe: Describe) -> None: - nested = '{"type": "library", "bom-ref": "deep", "name": "deep", "components": [' * 500 + "]}" * 500 - raw = json.dumps(describe()).replace('"components": []', f'"components": [{nested}]', 1) - assert validate(raw, None) == ["does not conform to the CycloneDX 1.6 schema: it nests too deeply to check"] - - -@pytest.mark.parametrize( - "licenses", - [ - pytest.param([{"expression": "MIT"}], id="expression"), - pytest.param([{"license": {"id": "mit", "acknowledgement": "concluded"}}], id="id-case-insensitive"), - pytest.param([{"expression": "MIT OR GPL-3.0-only"}], id="any-of"), - pytest.param([{"expression": "(GPL-3.0-only OR MIT) AND BSD-3-Clause"}], id="grouped"), - pytest.param([{"expression": "GPL-2.0-only WITH Classpath-exception-2.0"}], id="exception"), - ], -) -def test_license_policy_allows(describe: Describe, licenses: list[License]) -> None: - problems = validate(with_components(describe(), shipped("lib", licenses)), _ALLOWED) - assert [problem for problem in problems if problem.startswith("bundled:lib.whl")] == [] - - -@pytest.mark.parametrize( - ("licenses", "problem"), - [ - pytest.param( - [{"expression": "MIT AND GPL-3.0-only"}], - "license MIT AND GPL-3.0-only is outside the allowed licenses ['BSD-3-Clause', 'GPL-2.0-only', 'MIT']", - id="denied", - ), - pytest.param( - [{"license": {"name": "BSD License", "acknowledgement": "declared"}}], - "declares license names ['BSD License'] with no SPDX id", - id="names", - ), - pytest.param([], "declares no license", id="none"), - pytest.param([{"expression": "MIT AND"}], "license expression 'MIT AND' is malformed: it ends early", id="end"), - pytest.param([{"expression": "(MIT"}], "license expression '(MIT' is malformed: it ends early", id="open"), - pytest.param( - [{"expression": "(MIT ISC)"}], - "license expression '(MIT ISC)' is malformed: expected ')', got 'ISC'", - id="pair", - ), - pytest.param([{"expression": "MIT)"}], "license expression 'MIT)' is malformed: unexpected ')'", id="close"), - pytest.param( - [{"expression": "AND MIT"}], "license expression 'AND MIT' is malformed: unexpected 'AND'", id="op" - ), - pytest.param( - [{"expression": "MIT WITH OR"}], "license expression 'MIT WITH OR' is malformed: unexpected 'OR'", id="with" - ), - ], -) -def test_license_policy_reports(describe: Describe, licenses: list[License], problem: str) -> None: - problems = validate(with_components(describe(), shipped("lib", licenses)), _ALLOWED) - assert f"bundled:lib.whl: {problem}" in problems - - -def test_license_policy_judges_only_shipped_components(describe: Describe) -> None: - declared: Component = {"type": "library", "bom-ref": "requires-dist:click", "name": "click"} - vendored: Component = {"type": "library", "bom-ref": "bundled:outer.whl#vendored", "name": "vendored"} - outer = shipped("outer", [{"expression": "MIT"}]) - outer["components"] = [vendored, {"type": "file", "bom-ref": "outer#a.py", "name": "a.py"}] - assert validate(with_components(describe(), declared, outer), frozenset({"MIT"})) == [] diff --git a/tests/_validate/test_validate_extra.py b/tests/_validate/test_validate_extra.py deleted file mode 100644 index e5029b8..0000000 --- a/tests/_validate/test_validate_extra.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -import sys - -import pytest - -from python_sbom import MissingExtraError, validate - - -def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch) -> None: - # None in sys.modules makes the import fail, as when the extra is not installed - monkeypatch.setitem(sys.modules, "cyclonedx.schema", None) - monkeypatch.delitem(sys.modules, "python_sbom._validate._schemas", raising=False) - with pytest.raises(MissingExtraError, match=r"pip install 'python-sbom\[validate\]'"): - validate("{}", None) diff --git a/tests/_validate/test_validate_wheel.py b/tests/_validate/test_validate_wheel.py deleted file mode 100644 index 4dbb381..0000000 --- a/tests/_validate/test_validate_wheel.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -import json -from typing import TYPE_CHECKING, Final - -from python_sbom import dumps, validate_wheel - -if TYPE_CHECKING: - from pathlib import Path - - from conftest import Describe, WriteWheel - -_METADATA: Final[dict[str, bytes]] = {"example-1.0.dist-info/METADATA": b"Name: Example\nVersion: 1.0\n"} - - -def test_wheel_without_sbom(tmp_path: Path, write_wheel: WriteWheel) -> None: - wheel = write_wheel(tmp_path / "example-1.0-py3-none-any.whl", _METADATA) - assert validate_wheel(wheel, None) == ["no .dist-info/sboms/*.cdx.json found in the wheel"] - - -def test_every_sbom_is_checked(tmp_path: Path, write_wheel: WriteWheel, describe: Describe) -> None: - # the schema makes the serial optional, so the loaded JSON, not the typed document, loses it - broken = json.loads(dumps(describe())) - del broken["serialNumber"] - wheel = write_wheel( - tmp_path / "example-1.0-py3-none-any.whl", - { - **_METADATA, - "example-1.0.dist-info/sboms/example.cdx.json": json.dumps(broken).encode(), - "example-1.0.dist-info/sboms/other.cdx.json": json.dumps({ - "bomFormat": "CycloneDX", - "specVersion": "1.6", - "version": 1, - }).encode(), - "example-1.0.dist-info/sboms/notes.txt": b"not an SBOM", - }, - ) - assert validate_wheel(wheel, None) == [ - "example-1.0.dist-info/sboms/example.cdx.json: serialNumber must be a urn:uuid: RFC 4122 UUID, got ''" - ] diff --git a/tests/_wheel/test_archive.py b/tests/_wheel/test_archive.py new file mode 100644 index 0000000..63425d2 --- /dev/null +++ b/tests/_wheel/test_archive.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import stat +import struct +import sys +import tracemalloc +import zipfile +from contextlib import ExitStack +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom._errors import WheelError +from python_sbom._wheel._archive import open_archive, read_member, write_sbom + +if TYPE_CHECKING: + from collections.abc import Iterator + + from conftest import WriteWheel + from pytest_mock import MockerFixture + +_SBOM: Final[str] = "native-1.0.dist-info/sboms/native.cdx.json" +# the SHA-256 of b"{}\n", urlsafe base64 without padding +_SBOM_ROW: Final[str] = f"{_SBOM},sha256=yj0WO6sFU4GCciYUBWjzvvfqrBh869doeOC2Pp5EI1Y,3" + + +@pytest.fixture +def wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: + return write_wheel( + tmp_path / "dist" / "native-1.0-py3-none-any.whl", + {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"}, + ) + + +def test_embed_returns_member_path(wheel: Path) -> None: + assert write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) == _SBOM + + +def test_embed_puts_sbom_before_record(wheel: Path) -> None: + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert zipfile.ZipFile(wheel).namelist() == [ + "native/__init__.py", + "native-1.0.dist-info/METADATA", + _SBOM, + "native-1.0.dist-info/RECORD", + ] + + +def test_embed_records_sbom_before_record_row(wheel: Path) -> None: + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + record = zipfile.ZipFile(wheel).read("native-1.0.dist-info/RECORD").decode().splitlines() + assert record[-2:] == [_SBOM_ROW, "native-1.0.dist-info/RECORD,,"] + + +def test_embed_replaces_existing_sbom(wheel: Path) -> None: + write_sbom(wheel, "native.cdx.json", b"[]\n", 1_700_000_000) + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + with zipfile.ZipFile(wheel) as archive: + assert archive.read(_SBOM) == b"{}\n" + assert [ + line for line in archive.read("native-1.0.dist-info/RECORD").decode().splitlines() if "sboms" in line + ] == [_SBOM_ROW] + + +def test_embed_keeps_other_entries(wheel: Path) -> None: + before = {info.filename: (info.date_time, info.external_attr) for info in zipfile.ZipFile(wheel).infolist()} + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + after = {info.filename: (info.date_time, info.external_attr) for info in zipfile.ZipFile(wheel).infolist()} + assert {name: after[name] for name in before} == before + + +@pytest.mark.parametrize( + ("epoch", "date_time"), + [ + pytest.param(1_700_000_000, (2023, 11, 14, 22, 13, 20), id="epoch"), + pytest.param(0, (1980, 1, 1, 0, 0, 0), id="before-zip-epoch"), + ], +) +def test_embed_entry_is_the_same_on_every_system(wheel: Path, epoch: int, date_time: tuple[int, ...]) -> None: + write_sbom(wheel, "native.cdx.json", b"{}\n", epoch) + info = zipfile.ZipFile(wheel).getinfo(_SBOM) + assert (info.date_time, info.create_system, info.external_attr) == ( + date_time, + 3, + (stat.S_IFREG | 0o644) << 16, + ) + + +def test_embed_is_deterministic(tmp_path: Path, write_wheel: WriteWheel) -> None: + files = {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"} + first, second = (write_wheel(tmp_path / name / "native-1.0-py3-none-any.whl", files) for name in ("a", "b")) + for wheel in (first, second): + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert first.read_bytes() == second.read_bytes() + + +def test_embed_keeps_record_line_endings(tmp_path: Path, write_wheel: WriteWheel) -> None: + record = b"native-1.0.dist-info/METADATA,,\r\nnative-1.0.dist-info/RECORD,,\r\n" + wheel = write_wheel( + tmp_path / "native-1.0-py3-none-any.whl", + {"native-1.0.dist-info/METADATA": b"Name: native\n", "native-1.0.dist-info/RECORD": record}, + record=False, + ) + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert zipfile.ZipFile(wheel).read("native-1.0.dist-info/RECORD") == ( + f"native-1.0.dist-info/METADATA,,\r\n{_SBOM_ROW}\r\nnative-1.0.dist-info/RECORD,,\r\n".encode() + ) + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows has no POSIX permission bits") +def test_embed_keeps_wheel_permissions(wheel: Path) -> None: # pragma: win32 no cover + wheel.chmod(0o640) + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert stat.S_IMODE(wheel.stat().st_mode) == 0o640 + + +@pytest.mark.parametrize( + ("files", "message"), + [ + pytest.param({"native/__init__.py": b""}, "has no .dist-info/METADATA, so it is not a wheel", id="no-metadata"), + pytest.param( + {"native-1.0.dist-info/METADATA": b"Name: native\n"}, + "has no native-1.0.dist-info/RECORD", + id="no-record", + ), + ], +) +def test_embed_refuses_non_wheel(tmp_path: Path, files: dict[str, bytes], message: str) -> None: + with zipfile.ZipFile(archive := tmp_path / "native.whl", "w") as handle: + for name, content in files.items(): + handle.writestr(name, content) + with pytest.raises(WheelError, match=message): + write_sbom(archive, "native.cdx.json", b"{}\n", 1_700_000_000) + + +def test_failed_embed_leaves_wheel_and_no_temporary_file(wheel: Path, mocker: MockerFixture) -> None: + original = wheel.read_bytes() + mocker.patch.object(Path, "replace", autospec=True, side_effect=OSError("disk full")) + with pytest.raises(OSError, match="disk full"): + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert [path.name for path in wheel.parent.iterdir()] == [wheel.name] + assert wheel.read_bytes() == original + + +def test_embed_refuses_duplicate_member(tmp_path: Path) -> None: + with zipfile.ZipFile(wheel := tmp_path / "native-1.0-py3-none-any.whl", "w") as handle: + handle.writestr("native-1.0.dist-info/METADATA", b"Name: native\n") + handle.writestr("native-1.0.dist-info/RECORD", b"") + with pytest.warns(UserWarning, match="Duplicate name"): + handle.writestr("native-1.0.dist-info/RECORD", b"") + with pytest.raises(WheelError, match=r"holds more than one member named native-1\.0\.dist-info/record"): + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + + +def test_embed_refuses_corrupt_archive(tmp_path: Path) -> None: + (wheel := tmp_path / "native-1.0-py3-none-any.whl").write_bytes(b"not a zip") + with pytest.raises(WheelError, match="is not a valid zip archive: File is not a zip file"): + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + + +@pytest.mark.parametrize( + "names", + [ + pytest.param(("a.py", "A.py"), id="case"), + pytest.param(("pkg/a.py", "PKG/a.py"), id="directory-case"), + ], +) +def test_open_archive_refuses_names_equal_ignoring_case(names: tuple[str, str]) -> None: + message = f"w holds more than one member named {names[0].lower()}, ignoring case" + with pytest.raises(WheelError, match=message), ExitStack() as stack: + stack.enter_context(open_archive(zip_of(*((name, b"") for name in names)), "w")) + + +def zip_of(*members: tuple[str, bytes]) -> BytesIO: + buffer = BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in members: + archive.writestr(name, content) + return buffer + + +def test_open_archive_refuses_corrupt_member() -> None: + data = bytearray(zip_of(("a", b"hello" * 1000)).getvalue()) + data[40:50] = b"\xff" * 10 + with pytest.raises(WheelError, match="w is not a valid zip"), open_archive(BytesIO(bytes(data)), "w") as archive: + archive.read("a") + + +def test_read_member() -> None: + with open_archive(zip_of(("a", b"12345")), "w") as archive: + assert read_member(archive, "a", "w", limit=5) == b"12345" + + +def test_read_member_over_limit() -> None: + with open_archive(zip_of(("a", b"12345")), "w") as archive, pytest.raises(WheelError, match="w: a is over the 4"): + read_member(archive, "a", "w", limit=4) + + +def test_read_member_missing() -> None: + with open_archive(zip_of(("a", b"")), "w") as archive, pytest.raises(WheelError, match="w has no b"): + read_member(archive, "b", "w") + + +@pytest.mark.usefixtures("traced") +def test_read_member_refuses_understated_size() -> None: + # the archive claims 10 bytes for 32 MiB; reading the member whole would decompress all of it before the CRC check + data = bytearray(zip_of(("a", b"A" * 32 * 1024 * 1024)).getvalue()) + data[22:26] = data[(central := data.rfind(b"PK\x01\x02")) + 24 : central + 28] = struct.pack(" Iterator[None]: + tracemalloc.start() + yield + tracemalloc.stop() + + +def test_wheel_with_two_dist_info_directories(tmp_path: Path, write_wheel: WriteWheel) -> None: + wheel = write_wheel( + tmp_path / "native-1.0-py3-none-any.whl", + { + "aaa-0.dist-info/METADATA": b"Name: aaa\nVersion: 0\n", + "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n", + }, + record=False, + ) + with pytest.raises(WheelError, match=r"holds 2 \.dist-info directories, aaa-0\.dist-info, native-1\.0\.dist-info"): + write_sbom(wheel, "native.cdx.json", b"{}\n", 0) diff --git a/tests/_wheel/test_record.py b/tests/_wheel/test_record.py new file mode 100644 index 0000000..46166bf --- /dev/null +++ b/tests/_wheel/test_record.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import hashlib +from typing import Final + +import pytest + +from python_sbom._errors import WheelError +from python_sbom._model._inputs import RecordRow +from python_sbom._wheel._record import decode_digest, parse_record, record_line + +# the SHA-256 of b"", urlsafe base64 without padding +_EMPTY: Final[str] = "47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU" + + +def test_parse_record_row() -> None: + assert parse_record(f"pkg/__init__.py,sha256={_EMPTY},0\n", "w") == [ + RecordRow(path="pkg/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"").hexdigest(), size="0") + ] + + +def test_parse_record_unquotes_csv() -> None: + assert [row.path for row in parse_record(f'"pkg/a,b.py",sha256={_EMPTY},0\n', "w")] == ["pkg/a,b.py"] + + +@pytest.mark.parametrize( + "text", + [ + pytest.param("\n", id="blank"), + pytest.param("pkg-1.0.dist-info/RECORD,,\n", id="record"), + pytest.param("pkg-1.0.dist-info/RECORD.jws,,\n", id="jws-signature"), + pytest.param("pkg-1.0.dist-info/RECORD.p7s,,\n", id="p7s-signature"), + ], +) +def test_parse_record_skips_rows_without_digest(text: str) -> None: + assert parse_record(text, "w") == [] + + +@pytest.mark.parametrize( + ("text", "message"), + [ + pytest.param("pkg/__init__.py,,0\n", "w: RECORD row 1 gives no digest for pkg/__init__.py", id="no-digest"), + pytest.param("pkg/,,\n", "w: RECORD row 1 gives no digest for pkg/", id="directory"), + pytest.param(f"pkg/__init__.py,sha256={_EMPTY}\n", "w: RECORD row 1 has 2 columns, not 3", id="two-columns"), + pytest.param(f"a,sha256={_EMPTY},0,extra\n", "w: RECORD row 1 has 4 columns, not 3", id="four-columns"), + pytest.param("\npkg/__init__.py\n", "w: RECORD row 2 has 1 columns, not 3", id="row-number"), + ], +) +def test_parse_record_rejects_row(text: str, message: str) -> None: + with pytest.raises(WheelError, match=message): + parse_record(text, "w") + + +@pytest.mark.parametrize( + ("digest", "expected"), + [ + pytest.param(f"sha256={_EMPTY}", ("SHA-256", hashlib.sha256(b"").hexdigest()), id="sha256"), + pytest.param( + "md5=1B2M2Y8AsgTpgAmY7PhCfg", ("MD5", hashlib.md5(b"", usedforsecurity=False).hexdigest()), id="md5" + ), + ], +) +def test_decode_digest(digest: str, expected: tuple[str, str]) -> None: + assert decode_digest(digest, "w") == expected + + +@pytest.mark.parametrize( + ("digest", "message"), + [ + pytest.param("blake2b=AAAA", "w: unknown hash algorithm 'blake2b'", id="algorithm"), + pytest.param("sha256=abcde", "w: digest 'abcde' is not urlsafe base64", id="base64"), + ], +) +def test_decode_digest_rejects(digest: str, message: str) -> None: + with pytest.raises(WheelError, match=message): + decode_digest(digest, "w") + + +def test_record_line() -> None: + assert record_line("pkg/__init__.py", b"") == f"pkg/__init__.py,sha256={_EMPTY},0" diff --git a/tests/_wheel/test_wheel_embed.py b/tests/_wheel/test_wheel_embed.py deleted file mode 100644 index 7bb084f..0000000 --- a/tests/_wheel/test_wheel_embed.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import stat -import sys -import zipfile -from pathlib import Path -from typing import TYPE_CHECKING, Final - -import pytest - -from python_sbom import WheelError, embed_sbom - -if TYPE_CHECKING: - from conftest import WriteWheel - from pytest_mock import MockerFixture - -_SBOM: Final[str] = "native-1.0.dist-info/sboms/native.cdx.json" -# the SHA-256 of b"{}\n", urlsafe base64 without padding -_SBOM_ROW: Final[str] = f"{_SBOM},sha256=yj0WO6sFU4GCciYUBWjzvvfqrBh869doeOC2Pp5EI1Y,3" - - -@pytest.fixture -def wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: - return write_wheel( - tmp_path / "dist" / "native-1.0-py3-none-any.whl", - {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"}, - ) - - -def test_embed_returns_member_path(wheel: Path) -> None: - assert embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) == _SBOM - - -def test_embed_puts_sbom_before_record(wheel: Path) -> None: - embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) - assert zipfile.ZipFile(wheel).namelist() == [ - "native/__init__.py", - "native-1.0.dist-info/METADATA", - _SBOM, - "native-1.0.dist-info/RECORD", - ] - - -def test_embed_records_sbom_before_record_row(wheel: Path) -> None: - embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) - record = zipfile.ZipFile(wheel).read("native-1.0.dist-info/RECORD").decode().splitlines() - assert record[-2:] == [_SBOM_ROW, "native-1.0.dist-info/RECORD,,"] - - -def test_embed_replaces_existing_sbom(wheel: Path) -> None: - embed_sbom(wheel, "native.cdx.json", b"[]\n", 1_700_000_000) - embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) - with zipfile.ZipFile(wheel) as archive: - assert archive.read(_SBOM) == b"{}\n" - assert [ - line for line in archive.read("native-1.0.dist-info/RECORD").decode().splitlines() if "sboms" in line - ] == [_SBOM_ROW] - - -def test_embed_keeps_other_entries(wheel: Path) -> None: - before = {info.filename: (info.date_time, info.external_attr) for info in zipfile.ZipFile(wheel).infolist()} - embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) - after = {info.filename: (info.date_time, info.external_attr) for info in zipfile.ZipFile(wheel).infolist()} - assert {name: after[name] for name in before} == before - - -@pytest.mark.parametrize( - ("epoch", "date_time"), - [ - pytest.param(1_700_000_000, (2023, 11, 14, 22, 13, 20), id="epoch"), - pytest.param(0, (1980, 1, 1, 0, 0, 0), id="before-zip-epoch"), - ], -) -def test_embed_entry_is_the_same_on_every_system(wheel: Path, epoch: int, date_time: tuple[int, ...]) -> None: - embed_sbom(wheel, "native.cdx.json", b"{}\n", epoch) - info = zipfile.ZipFile(wheel).getinfo(_SBOM) - assert (info.date_time, info.create_system, info.external_attr) == ( - date_time, - 3, - (stat.S_IFREG | 0o644) << 16, - ) - - -def test_embed_is_deterministic(tmp_path: Path, write_wheel: WriteWheel) -> None: - files = {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"} - first, second = (write_wheel(tmp_path / name / "native-1.0-py3-none-any.whl", files) for name in ("a", "b")) - for wheel in (first, second): - embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) - assert first.read_bytes() == second.read_bytes() - - -def test_embed_keeps_record_line_endings(tmp_path: Path, write_wheel: WriteWheel) -> None: - record = b"native-1.0.dist-info/METADATA,,\r\nnative-1.0.dist-info/RECORD,,\r\n" - wheel = write_wheel( - tmp_path / "native-1.0-py3-none-any.whl", - {"native-1.0.dist-info/METADATA": b"Name: native\n", "native-1.0.dist-info/RECORD": record}, - record=False, - ) - embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) - assert zipfile.ZipFile(wheel).read("native-1.0.dist-info/RECORD") == ( - f"native-1.0.dist-info/METADATA,,\r\n{_SBOM_ROW}\r\nnative-1.0.dist-info/RECORD,,\r\n".encode() - ) - - -@pytest.mark.skipif(sys.platform == "win32", reason="Windows has no POSIX permission bits") -def test_embed_keeps_wheel_permissions(wheel: Path) -> None: - wheel.chmod(0o640) - embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) - assert stat.S_IMODE(wheel.stat().st_mode) == 0o640 - - -@pytest.mark.parametrize( - ("files", "message"), - [ - pytest.param({"native/__init__.py": b""}, "has no .dist-info/METADATA, so it is not a wheel", id="no-metadata"), - pytest.param( - {"native-1.0.dist-info/METADATA": b"Name: native\n"}, - "has no native-1.0.dist-info/RECORD", - id="no-record", - ), - ], -) -def test_embed_refuses_non_wheel(tmp_path: Path, files: dict[str, bytes], message: str) -> None: - with zipfile.ZipFile(archive := tmp_path / "native.whl", "w") as handle: - for name, content in files.items(): - handle.writestr(name, content) - with pytest.raises(WheelError, match=message): - embed_sbom(archive, "native.cdx.json", b"{}\n", 1_700_000_000) - - -def test_failed_embed_leaves_wheel_and_no_temporary_file(wheel: Path, mocker: MockerFixture) -> None: - original = wheel.read_bytes() - mocker.patch.object(Path, "replace", autospec=True, side_effect=OSError("disk full")) - with pytest.raises(OSError, match="disk full"): - embed_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) - assert [path.name for path in wheel.parent.iterdir()] == [wheel.name] - assert wheel.read_bytes() == original diff --git a/tests/conftest.py b/tests/conftest.py index c12966a..dff5c64 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,12 +11,15 @@ import pytest -from python_sbom import BuildFacts, ContentProvider, Interpreter, SbomRequest, Settings, SourceTree, build_sbom +from python_sbom import ContentProvider, ProviderContext +from python_sbom._generate._pipeline import build_sbom +from python_sbom._model._identity import project_identity +from python_sbom._model._inputs import BuildFacts, Interpreter, SbomRequest, Settings, SourceTree if TYPE_CHECKING: from collections.abc import Mapping - from python_sbom import Document + from python_sbom._model._types import Document class WriteWheel(Protocol): @@ -33,6 +36,10 @@ class Git(Protocol): def __call__(self, *arguments: str) -> str: ... +class ContextFor(Protocol): + def __call__(self, root: Path, artifact: Path) -> ProviderContext: ... + + class Describe(Protocol): def __call__( self, @@ -141,11 +148,27 @@ def describe( settings=settings, build_scripts=(), ) + context = ProviderContext( + root=Path(), + settings=settings, + identity=project_identity(request.metadata, settings), + artifact=Path(), + ) return build_sbom( request, source or SourceTree(copyright=None, bundled=[], build_scripts=[]), facts, content or ContentProvider(), + context, ).document return describe + + +@pytest.fixture +def context_for(settings: Settings) -> ContextFor: + def context(root: Path, artifact: Path) -> ProviderContext: + identity = project_identity(message_from_string("Name: Example\nVersion: 1.0\n"), settings) + return ProviderContext(root=root, settings=settings, identity=identity, artifact=artifact) + + return context diff --git a/tests/hatch/conftest.py b/tests/hatch/conftest.py index 3bf15bc..cdba574 100644 --- a/tests/hatch/conftest.py +++ b/tests/hatch/conftest.py @@ -61,8 +61,8 @@ def seeder(tmp_path: Path, write_wheel: WriteWheel) -> Path: pyproject = target / "pyproject.toml" pyproject.write_text( pyproject.read_text(encoding="utf-8").replace( - "build.hooks.custom.supplier", - f'build.hooks.custom.path = "{(_SEEDER / "hatch_build.py").as_posix()}"\nbuild.hooks.custom.supplier', + "build.hooks.custom = {}", + f'build.hooks.custom.path = "{(_SEEDER / "hatch_build.py").as_posix()}"', ), encoding="utf-8", ) diff --git a/tests/hatch/test_hatch_hook.py b/tests/hatch/test_hook.py similarity index 66% rename from tests/hatch/test_hatch_hook.py rename to tests/hatch/test_hook.py index 97ba2cf..4a32557 100644 --- a/tests/hatch/test_hatch_hook.py +++ b/tests/hatch/test_hook.py @@ -1,19 +1,24 @@ from __future__ import annotations import json +import shutil +import tarfile import time import zipfile from typing import TYPE_CHECKING, Final -from python_sbom import validate_wheel +import pytest + +from python_sbom._errors import SettingsError +from python_sbom._validate._artifact import validate if TYPE_CHECKING: from pathlib import Path - import pytest from conftest import BuildWheel - from python_sbom import Document + from python_sbom._model._types import Document + from tests.conftest import Git _SBOM: Final[str] = "example-1.0.dist-info/sboms/example.cdx.json" # a later hook that changes what the wheel declares and how it is tagged, after the SBOM hook's turn @@ -33,7 +38,7 @@ def sbom_of(wheel: Path, name: str = _SBOM) -> Document: def test_wheel_carries_valid_sbom(project: Path, build_wheel: BuildWheel) -> None: - assert validate_wheel(build_wheel(project), frozenset({"MIT"})) == [] + assert validate(build_wheel(project), frozenset({"MIT"})) == [] def test_sbom_describes_the_built_wheel(project: Path, build_wheel: BuildWheel) -> None: @@ -53,17 +58,23 @@ def configure(root: Path, tables: str) -> None: pyproject.write_text(f"{pyproject.read_text(encoding='utf-8')}{tables}", encoding="utf-8") -def test_hook_table_overrides_project_table(project: Path, build_wheel: BuildWheel) -> None: +def test_project_table_options(project: Path, build_wheel: BuildWheel) -> None: configure( project, - "enable-by-default = true\nrequire-runtime-dependencies = false\ndefault-branch = 'trunk'\n" + "enable-by-default = true\nrequire-runtime-dependencies = false\n" "[tool.python-sbom]\ndefault-branch = 'dev'\nsupplier = { name = 'Example Org' }\n", ) root = sbom_of(build_wheel(project))["metadata"]["component"] - assert {"type": "license", "url": "https://github.com/example/example/blob/trunk/LICENSE"} in root[ - "externalReferences" - ] - assert root.get("supplier") == {"name": "Example Org", "url": []} + assert (root.get("supplier"), root["externalReferences"][-1]) == ( + {"name": "Example Org", "url": []}, + {"type": "license", "url": "https://github.com/example/example/blob/dev/LICENSE"}, + ) + + +def test_hook_table_refuses_python_sbom_options(project: Path, build_wheel: BuildWheel) -> None: + configure(project, "default-branch = 'trunk'\n") + with pytest.raises(SettingsError, match=r"hook's table holds default-branch; move python-sbom's options"): + build_wheel(project) def test_reproducible_build_uses_hatchling_time(project: Path, build_wheel: BuildWheel) -> None: @@ -111,21 +122,21 @@ def test_content_provider_option(project: Path, build_wheel: BuildWheel, tmp_pat (project / f"{tmp_path.name}.py").write_text( "from python_sbom import ContentProvider, Contributions\n\n\n" "class Content(ContentProvider):\n" - " def contributions(self, request):\n" - " return Contributions(components=[], dependencies=[], tools=[]," - " root_properties=[{'name': 'example:by', 'value': 'hook'}])\n", + " def contributions(self, context):\n" + " return Contributions(" + " root_properties=[{'name': 'extra:by', 'value': 'hook'}])\n", encoding="utf-8", ) - configure(project, f"content-provider = '{tmp_path.name}:Content'\n") + configure(project, f"[tool.python-sbom]\ncontent-provider = '{tmp_path.name}:Content'\n") document = sbom_of(build_wheel(project)) - assert {"name": "example:by", "value": "hook"} in document["metadata"]["component"]["properties"] + assert {"name": "extra:by", "value": "hook"} in document["metadata"]["component"]["properties"] assert document["metadata"]["tools"]["components"][0]["name"] == f"{tmp_path.name}.py" def test_seeder_bundles_described(seeder: Path, build_wheel: BuildWheel) -> None: (pip,) = sbom_of(build_wheel(seeder), "seeder-1.0.dist-info/sboms/seeder.cdx.json")["components"] assert pip.get("properties", [])[:3] == [ - {"name": "seeder:bundled-wheel", "value": "src/seeder/embed/pip-1.0-py3-none-any.whl"}, + {"name": "seeder:bundled-wheel", "value": "seeder/embed/pip-1.0-py3-none-any.whl"}, {"name": "seeder:seeded-for-python", "value": "3.14"}, {"name": "seeder:seeded-for-python", "value": "3.15"}, ] @@ -139,3 +150,35 @@ def test_seeder_hook_script_is_a_tool(seeder: Path, build_wheel: BuildWheel) -> def test_seeder_supplier(seeder: Path, build_wheel: BuildWheel) -> None: document = sbom_of(build_wheel(seeder), "seeder-1.0.dist-info/sboms/seeder.cdx.json") assert document["metadata"].get("supplier") == {"name": "Example Org", "url": ["https://example.org"]} + + +def test_seeder_bundled_wheel_supplier(seeder: Path, build_wheel: BuildWheel) -> None: + document = sbom_of(build_wheel(seeder), "seeder-1.0.dist-info/sboms/seeder.cdx.json") + assert document["components"][0].get("supplier") == {"name": "Example Org", "url": ["https://example.org"]} + + +@pytest.fixture +def checkout(project: Path, tmp_path: Path, git: Git) -> Path: + shutil.copytree(project, target := tmp_path / "checkout", dirs_exist_ok=True) + git("add", ".") + git("commit", "--quiet", "--message", "first") + return target + + +def test_sdist_records_clean_revision(checkout: Path, git: Git, build_wheel: BuildWheel) -> None: + files = sdist_files(build_wheel(checkout, target="sdist")) + assert files.get(".python-sbom-revision") == f"node: {git('rev-parse', 'HEAD')}\n".encode() + + +def sdist_files(sdist: Path) -> dict[str, bytes]: + with tarfile.open(sdist) as archive: + return { + member.name.split("/", 1)[1]: handle.read() + for member in archive.getmembers() + if (handle := archive.extractfile(member)) is not None + } + + +def test_sdist_of_dirty_checkout_records_no_revision(checkout: Path, build_wheel: BuildWheel) -> None: + (checkout / "LICENSE").write_text("changed\n", encoding="utf-8") + assert ".python-sbom-revision" not in sdist_files(build_wheel(checkout, target="sdist")) diff --git a/tests/test_examples.py b/tests/test_examples.py deleted file mode 100644 index 85b74ff..0000000 --- a/tests/test_examples.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import annotations - -import zipfile -from typing import TYPE_CHECKING - -from docs.examples.zipapp import describe_zipapp -from python_sbom import dumps, validate - -if TYPE_CHECKING: - from pathlib import Path - - -def test_zipapp_example(tmp_path: Path) -> None: - (tmp_path / "pyproject.toml").write_text("[project]\nname = 'app'\n", encoding="utf-8") - with zipfile.ZipFile(zipapp := tmp_path / "app.pyz", "w") as archive: - archive.writestr("__main__.py", "print('hi')\n") - archive.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") - archive.writestr("app/", "") - document = describe_zipapp(tmp_path, zipapp) - assert validate(dumps(document).decode(), None) == [] - assert [file["name"] for file in document["metadata"]["component"].get("components", [])] == [ - "__main__.py", - "app-1.0.dist-info/METADATA", - ] diff --git a/tests/test_public_api.py b/tests/test_public_api.py new file mode 100644 index 0000000..788bbcc --- /dev/null +++ b/tests/test_public_api.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import collections.abc +import email.message +import importlib +import importlib.metadata +import inspect +import pathlib +import pkgutil +import typing +from typing import Final + +import pytest + +import python_sbom +import python_sbom.hatch + +_PUBLIC: Final[set[str]] = { + "BundledWheelInfo", + "Component", + "Contact", + "ContentProvider", + "Contributions", + "Dependency", + "Evidence", + "ExternalReference", + "Hash", + "Identity", + "IdentityMethod", + "License", + "LicenseInfo", + "OrganizationalEntity", + "ProjectIdentity", + "Property", + "ProviderContext", + "SbomError", + "Settings", + "__version__", +} + + +def test_public_api() -> None: + assert set(python_sbom.__all__) == _PUBLIC + + +def test_hatch_api() -> None: + assert set(python_sbom.hatch.__all__) == {"SbomBuildHook"} + + +def reached(start: object) -> set[object]: # simplint: ignore[SMP602] # hints are arbitrary typing objects + # every python_sbom type a hint of a public name leads to, through fields, parameters and nested hints + found: set[object] = set() # simplint: ignore[SMP602] # see above + pending: list[object] = [start] # simplint: ignore[SMP602] # see above + while pending: + if (item := pending.pop()) in found: + continue + found.add(item) + if typing.is_typeddict(item) or inspect.isclass(item): + pending.extend(typing.get_type_hints(item, globalns=_names(), include_extras=True).values()) + for _, member in inspect.getmembers(item, inspect.isfunction): + if member.__module__.startswith("python_sbom"): + pending.extend(typing.get_type_hints(member, globalns=_names(), include_extras=True).values()) + pending.extend(typing.get_args(item)) + if (origin := typing.get_origin(item)) is not None: + pending.append(origin) + return {item for item in found if getattr(item, "__module__", "").startswith("python_sbom")} + + +@pytest.mark.parametrize("name", sorted(_PUBLIC - {"__version__"}), ids=str) +def test_public_types_only_reach_public_types(name: str) -> None: + assert reached(getattr(python_sbom, name)) - exported() == set() + + +def test_hook_only_reaches_public_types() -> None: + assert reached(python_sbom.hatch.SbomBuildHook) - exported() == set() + + +def exported() -> set[object]: # simplint: ignore[SMP602] # see above + return {getattr(python_sbom, name) for name in _PUBLIC} | {python_sbom.hatch.SbomBuildHook} + + +def _names() -> dict[str, object]: # simplint: ignore[SMP602] # see above + # the annotations name types the modules import only for type checking, so every module's names resolve them + modules = [ + pathlib, + collections.abc, + typing, + *(importlib.import_module(found.name) for found in pkgutil.walk_packages(python_sbom.__path__, "python_sbom.")), + ] + names = dict(member for module in modules for member in inspect.getmembers(module)) + return {**names, "Message": email.message.Message, "PackageMetadata": importlib.metadata.PackageMetadata} diff --git a/tox.toml b/tox.toml index 1d46288..b6e36da 100644 --- a/tox.toml +++ b/tox.toml @@ -58,9 +58,9 @@ commands = [ "--color", "-b", "html", - { replace = "posargs", default = [ - "-W", - ], extend = true }, + "-W", + "--keep-going", + { replace = "posargs", extend = true }, ], [ "python", @@ -114,10 +114,12 @@ commands = [ [ "python", "-c", - # the project's own wheel carries the SBOM its hook writes + # the project's own wheel carries the SBOM its hook writes, which lists that hook script among the build tools """\ - import glob, zipfile; (wheel,) = glob.glob(r'{env_tmp_dir}{/}*.whl'); assert any('/sboms/' in name for name in \ - zipfile.ZipFile(wheel).namelist()), f'no SBOM in {wheel}'\ + import glob, json, zipfile; (wheel,) = glob.glob(r'{env_tmp_dir}{/}*.whl'); archive = zipfile.ZipFile(wheel); \ + (sbom,) = (name for name in archive.namelist() if '/sboms/' in name); tools = \ + json.loads(archive.read(sbom))['metadata']['tools']['components']; assert any(tool['name'] == 'hatch_build.py' for \ + tool in tools), f'hatch_build.py missing from {sbom}'\ """, ], ] From ab7d9f0137e7400f63a555888cfdf66ad1db9e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 23:09:23 -0800 Subject: [PATCH 021/115] =?UTF-8?q?=F0=9F=93=9D=20docs:=20document=20provi?= =?UTF-8?q?ders=20and=20the=20archive=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/THREAT_MODEL.md | 24 ++++-- README.md | 17 ++-- docs/conf.py | 6 -- docs/contributing.rst | 16 ++++ docs/examples/__init__.py | 1 - docs/examples/zipapp.py | 60 ------------- docs/explanation.rst | 49 ++++++----- docs/extend.rst | 149 +++++++++++--------------------- docs/py_api.rst | 175 +++++++++----------------------------- docs/usage.rst | 41 +++++---- 10 files changed, 185 insertions(+), 353 deletions(-) delete mode 100644 docs/examples/__init__.py delete mode 100644 docs/examples/zipapp.py diff --git a/.github/THREAT_MODEL.md b/.github/THREAT_MODEL.md index 5f3b8ca..a50cc46 100644 --- a/.github/THREAT_MODEL.md +++ b/.github/THREAT_MODEL.md @@ -8,8 +8,9 @@ SECURITY.md wins and we fix this file. The build hook runs inside a hatchling wheel build. It reads the wheel's core metadata, the project's license files, the wheels the project bundles and the installed distributions of the build environment, and writes a CycloneDX document -that hatchling adds to the wheel. `python-sbom embed` does the same for a wheel another backend built and rewrites that -wheel. `python-sbom spdx` and `validate` read a CycloneDX document and write or check its SPDX form. +that it writes into the built wheel. `python-sbom embed` does the same for a wheel another backend built, +`python-sbom archive` describes another zip archive such as a zipapp, and `python-sbom spdx` and `validate` read +CycloneDX documents and write or check them. ```mermaid flowchart LR @@ -28,17 +29,22 @@ flowchart LR An attacker who controls any of these already has what python-sbom could give them: -- The project source, its `pyproject.toml` and the build hook script. +- The project source, its `pyproject.toml`, the build hook script, and the module the `content-provider` option or + `--content-provider` names: python-sbom imports that module from the project directory, so `embed` and `archive` run + code from `--root`. +- The git checkout that tracks the project, whose `HEAD` python-sbom reads. A repository that merely contains the + project directory is not trusted: git is used only when it tracks the project, and runs with `core.fsmonitor` off. - The build environment and the build backend. -- The wheel handed to `python-sbom embed`, which the project's own build produced. +- The wheel handed to `python-sbom embed` and the archive handed to `python-sbom archive`, which the project's own build + produced. ## Untrusted inputs -| Input | Comes from | What python-sbom guarantees | -| --------------------------------------- | ------------------------- | ----------------------------------------------------------------------------------------- | -| Metadata and `RECORD` of bundled wheels | Upstream projects | Read as text, copied into the document as JSON strings; nothing is extracted or executed. | -| `_vendor/vendor.txt` of bundled wheels | Upstream projects | Each line must pin one exact version, or the build fails. | -| SBOM files given to `spdx`, `validate` | Whoever produced the file | Parsed as JSON only; `spdx` writes the one file named on the command line. | +| Input | Comes from | What python-sbom guarantees | +| --------------------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Metadata and `RECORD` of bundled wheels | Upstream projects | Read as text with capped sizes, copied as JSON strings; every member is checked against its `RECORD` row, duplicate members are refused, nothing is executed. | +| `_vendor/vendor.txt` of bundled wheels | Upstream projects | Each line must pin one exact version, or the build fails. | +| SBOM files given to `spdx`, `validate` | Whoever produced the file | Parsed as JSON only, with capped sizes and nesting; problems are reported, not raised; `spdx` writes the one file named on the command line. | ## Threats and controls diff --git a/README.md b/README.md index f99da24..3d316ce 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,19 @@ reproduces it byte for byte. ## Install -The command line, to embed, convert and validate SBOMs: +A hatchling project names `python-sbom[hatch]` in its build requirements, as shown below. To embed an SBOM into a wheel +another backend built, install the command line into the environment that built the wheel, since the SBOM lists that +environment's packages as the build tools (or point `--build-env` at it): ```console -pipx install 'python-sbom[validate]' +python -m pip install python-sbom ``` -A hatchling project names `python-sbom[hatch]` in its build requirements instead, as shown below. +To convert and validate SBOMs in CI, a separate tool environment is fine: + +```console +pipx install 'python-sbom[validate]' +``` ## Use @@ -40,5 +46,6 @@ With another build backend, such as meson-python, embed the SBOM after the build python-sbom embed dist/*.whl ``` -The [documentation](https://python-sbom.readthedocs.io) covers the options, extending the SBOM from a custom build hook, -rendering it as SPDX and validating it. +Describe another zip archive the project publishes, such as a zipapp, with `python-sbom archive`. The +[documentation](https://python-sbom.readthedocs.io) covers the options, adding project knowledge through a content +provider, rendering an SBOM as SPDX and validating it. diff --git a/docs/conf.py b/docs/conf.py index 2ee4df6..afcceed 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,6 @@ def _inheritance_aliases() -> dict[str, str]: inheritance_graph_attrs: Final[dict[str, str]] = {"rankdir": "TB"} # each directive names the members it documents, so hatchling's lifecycle methods and inherited members stay out autodoc_default_options: Final[dict[str, bool | str]] = {"member-order": "bysource", "show-inheritance": True} -autodoc_type_aliases: Final[dict[str, str]] = {"Toml": "python_sbom.Toml"} typehints_fully_qualified: Final[bool] = False always_document_param_types: Final[bool] = True typehints_document_rtype: Final[bool] = True @@ -81,11 +80,6 @@ def _inheritance_aliases() -> dict[str, str]: # the Python documentation describes the protocol without a reference target ("py:class", "importlib.metadata._meta.PackageMetadata"), ] -# parts of the document types that the CycloneDX and SPDX references describe -nitpick_ignore_regex: Final[list[tuple[str, str]]] = [ - ("py:class", r"python_sbom\._model\._types\.(Composition|LicenseInfo|Metadata)"), - ("py:class", r"python_sbom\._formats\._spdx\._\w+"), -] def setup(app: Sphinx) -> None: diff --git a/docs/contributing.rst b/docs/contributing.rst index 0a1f58d..7530dac 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -1,6 +1,22 @@ Contributing ============ +How the code fits together +-------------------------- + +Every SBOM comes out of one pipeline. An adapter, the hatchling hook or a command of the command line, gathers what the +edge knows: the request (the wheel's core metadata and file name, the project directory, the options and the build +scripts), the source tree (license files and bundled wheels), and the build facts (time, source revision and toolchain). +It hands them, with a content provider, to the pipeline, which builds the document with pure functions in ``_model``. +:doc:`explanation` draws the packages and their imports. + +To record a new field, read it at the edge into ``_model/_inputs.py`` and write it in ``_model``; to add a subcommand, +add a registration function and a handler next to the others in ``_cli/_commands.py`` and list the registration in +``_cli/_parser.py``. Tests mirror the source: ``tests//test_.py``. + +Changelog fragments +------------------- + Add a changelog fragment under ``docs/changelog.d`` when a pull request changes what package users can use or observe. Name it ``+short-name..rst``, for example ``+vendored-licenses.feature.rst``. The name describes the change; you do not need a pull request number before opening the pull request. Write one change per fragment, and use several diff --git a/docs/examples/__init__.py b/docs/examples/__init__.py deleted file mode 100644 index 5b1d075..0000000 --- a/docs/examples/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Examples the documentation includes and the tests run.""" diff --git a/docs/examples/zipapp.py b/docs/examples/zipapp.py deleted file mode 100644 index 495dba2..0000000 --- a/docs/examples/zipapp.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -import zipfile -from datetime import UTC, datetime -from email.parser import Parser -from typing import TYPE_CHECKING - -from python_sbom import ( - collect_facts, - default_revision, - document, - file_component_from_bytes, - formulation, - project_identity, - settings_from_pyproject, - tool_components, -) - -if TYPE_CHECKING: - from pathlib import Path - - from python_sbom import Body, Component, Document - - -def describe_zipapp(root: Path, zipapp: Path) -> Document: # simplint: ignore[SMP408] # the docs include it - with zipfile.ZipFile(zipapp) as archive: - members = sorted(name for name in archive.namelist() if not name.endswith("/")) - metadata = Parser().parsestr( - archive.read(next(name for name in members if name.endswith(".dist-info/METADATA"))).decode() - ) - app: Component = { - "type": "application", - "bom-ref": f"pkg:generic/{zipapp.name}@{metadata['Version']}", - "name": zipapp.name, - "version": metadata["Version"], - } - app["components"] = [file_component_from_bytes(app["bom-ref"], name, archive.read(name)) for name in members] - # the build toolchain, time and serial namespace, taken the way the wheel SBOM takes them - facts = collect_facts(root, default_revision, None, None) - identity = project_identity(metadata, settings_from_pyproject(root, {})) - tools, tool_dependencies = tool_components(facts) - body: Body = { - "metadata": { - "timestamp": datetime.fromtimestamp(facts.epoch, UTC).isoformat(), - "lifecycles": [{"phase": "build"}], - "tools": {"components": tools}, - "component": app, - # naming the document's file marks it as built the python-sbom way, so validate checks it in full - "properties": [{"name": f"{identity.namespace}:sbom:file", "value": f"{zipapp.name}.cdx.json"}], - }, - "components": [], - "dependencies": [{"ref": app["bom-ref"], "dependsOn": []}, *tool_dependencies], - "formulation": [formulation("zipapp-build", "build the zipapp", tools, app["bom-ref"], facts)], - } - return document(body, identity.serial_namespace) - - -__all__ = [ - "describe_zipapp", -] diff --git a/docs/explanation.rst b/docs/explanation.rst index b33cd4a..4375ee1 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -5,32 +5,35 @@ Architecture ------------ Each concern has its own subpackage, and only the adapters know a build system or the command line. The hatchling hook -and the ``embed`` command gather what the edge knows, then call one pipeline function, :func:`~python_sbom.build_sbom`, -with explicit inputs: the :class:`~python_sbom.SbomRequest` a :class:`~python_sbom.Backend` returns, the -:class:`~python_sbom.SourceTree` and :class:`~python_sbom.BuildFacts` the collectors read, and a -:class:`~python_sbom.ContentProvider`. The model turns those inputs into the document with pure functions. The diagram -follows the imports between the subpackages; an arrow points at what a package imports. +and the ``embed`` and ``archive`` commands gather what the edge knows: the request (core metadata, file name, options, +build scripts), the source tree, and the build facts (time, revision, toolchain). They hand those, with a +:class:`~python_sbom.ContentProvider` and its :class:`~python_sbom.ProviderContext`, to one pipeline, and the model +turns them into the document with pure functions. The diagram follows the imports between the subpackages: an arrow +points at what a package imports, and a dashed arrow at what it imports only for type annotations. .. mermaid:: flowchart TB subgraph adapters[Adapters] hook[hatch
hatchling build hook] - cli[_cli
embed, spdx, validate] + cli[_cli
embed, archive, spdx, validate] end - generate[_generate
Backend, ContentProvider, build_sbom] - collect[_collect
settings, build facts, source tree] + api[python_sbom
provider API] + generate[_generate
wheel and archive pipelines] + collect[_collect
settings, facts, source tree, provider loading] validate[_validate
schema, references, licenses] formats[_formats
CycloneDX JSON, SPDX 2.3] - wheel[_wheel
RECORD rows, archive rewrite] - model[_model
inputs, types, pure document building] + wheel[_wheel
RECORD rows, archive reads and rewrite] + model[_model
inputs, types, identity, document building] hook --> generate hook --> collect - hook --> wheel + hook -.-> model cli --> generate cli --> collect cli --> validate - cli --> wheel + cli --> formats + api --> model + generate --> collect generate --> formats generate --> wheel generate --> model @@ -43,17 +46,16 @@ follows the imports between the subpackages; an arrow points at what a package i wheel --> model classDef edge fill:#cfe4ff,stroke:#1f6feb,color:#0b1f3a; classDef core fill:#d8f3dc,stroke:#2d6a4f,color:#081c15; - class hook,cli,collect edge; + class hook,cli,collect,api edge; class generate,validate,formats,wheel,model core; -The model imports nothing else of the package, and nothing below the adapters imports hatchling, so the library and -the command line work without it. Every package raises subclasses of :class:`~python_sbom.SbomError`, which the command +The model imports nothing else of the package but its errors, and only the hook imports hatchling, so the provider API +and the command line work without it. Every package raises subclasses of :class:`~python_sbom.SbomError`, which the command line reports without a traceback. -Extension points are explicit: a build system plugs in as a :class:`~python_sbom.Backend`, a project adds content as a -:class:`~python_sbom.ContentProvider`, and another version control system as a :class:`~python_sbom.RevisionReader`. -The hatch hook exposes two factories, :meth:`~python_sbom.hatch.SbomBuildHook.content_provider` and -:meth:`~python_sbom.hatch.SbomBuildHook.backend`, for a subclass to override. +The one extension point is the :class:`~python_sbom.ContentProvider`, which every entry point loads from the +``content-provider`` option. A hatchling custom hook can override :meth:`~python_sbom.hatch.SbomBuildHook.content_provider` to build the provider +itself. What the SBOM records --------------------- @@ -71,6 +73,7 @@ dependencies incomplete for that reason. An extra's dependencies are left out. The formulation records how the wheel was built: the interpreter's version and implementation, every distribution of the build environment with its dependencies, the files and hashes of each pure-Python one, and ``SOURCE_DATE_EPOCH`` when it set the build time. + Property names start with the project's normalized name, such as ``virtualenv:bundled-wheel``, or with ``python:`` for values copied from core metadata. @@ -102,10 +105,10 @@ Limitations - The document describes what the project declares and bundles, not the full tree of runtime dependencies, which depends on the environment the wheel is installed into. - The links to the PyPI release and attestation assume the project publishes to PyPI with attestations. -- A build from an sdist has no git checkout, so its SBOM names no commit unless ``PYTHON_SBOM_COMMIT`` or an expanded - ``.git_archival.txt`` supplies it. -- ``python-sbom embed`` records the environment ``--build-env`` names, by default the one it runs in. Unless that is - the environment the wheel was built in, the build tool record does not match the real build. +- A wheel built from an sdist of a dirty checkout names no commit, since no commit reproduces it. +- ``python-sbom embed`` and ``archive`` record the packages of the environment ``--build-env`` names, by default the + one they run in, and always the interpreter they run on. Unless that is the environment the artifact was built in, + the build tool record does not match the real build. - The SPDX rendering leaves out per-file hashes and the build record, which SPDX 2.3 cannot hold. - A tool that renames the wheel after the build, such as ``auditwheel repair``, leaves the attestation link pointing at the old file name until ``python-sbom embed`` runs on the renamed wheel. diff --git a/docs/extend.rst b/docs/extend.rst index 0adb8aa..9ec3d5e 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -2,23 +2,30 @@ Extend it ========= The options cover what every project shares. A project that knows more than its metadata says adds it through a -content provider. A build system other than hatchling plugs in as a backend. An artifact other than a wheel gets its -SBOM from the same public pieces the hook uses. +content provider, and the same provider serves the wheel's SBOM and the SBOM of any other archive the project +publishes. Add content with a provider --------------------------- -A :class:`~python_sbom.ContentProvider` has two methods, each with a default that adds nothing: - -- :meth:`~python_sbom.ContentProvider.bundled_wheel` returns the properties and supplier of one bundled wheel. The - properties follow ``:bundled-wheel`` and come before those taken from the wheel's metadata. -- :meth:`~python_sbom.ContentProvider.contributions` returns extra top-level components, dependency entries, root - properties and build tools, such as the C libraries a native wheel links. They join the document before the root's - dependencies and the serial number are derived from it, so the document stays consistent. - -Name the provider in the ``content-provider`` option, and the hook and ``python-sbom embed`` both load it. When it -lives in the project, the SBOM lists its module among the build tools, hashed, because it shapes the document a rebuild -has to reproduce: +A :class:`~python_sbom.ContentProvider` has two methods. Each receives a :class:`~python_sbom.ProviderContext`: the +project directory, its :class:`~python_sbom.Settings`, its :class:`~python_sbom.ProjectIdentity` with the prefix of its +properties, and :meth:`~python_sbom.ProviderContext.read`, which reads a file of the artifact being described, a member +of the built wheel or of the archive. Each returns a value whose fields default to empty, so a provider names only what +it adds. + +- :meth:`~python_sbom.ContentProvider.bundled_wheel` returns the properties and the supplier of one bundled wheel. The + properties follow ``:bundled-wheel`` and come before those from the wheel's metadata. A wheel's supplier comes + only from here, never from the wheel's own metadata. +- :meth:`~python_sbom.ContentProvider.contributions` returns extra top-level components, the dependencies between + them, root properties and build tools, such as the C libraries a native wheel links. They join the document before + the root's dependencies and the serial number derive from it. The build fails when a ``bom-ref`` repeats, a + dependency names one that no component has, or a root property uses the project's own prefix. + +Name the provider in the ``content-provider`` option, as ``module:attribute`` importable from the project root, and +the hook, ``python-sbom embed`` and ``python-sbom archive`` all load it. When the provider's class lives in the +project, the SBOM lists its module among the build tools, hashed, because it shapes the document a rebuild has to +reproduce: .. code-block:: python @@ -26,40 +33,41 @@ has to reproduce: class Content(ContentProvider): - def contributions(self, request): + def contributions(self, context): zlib = {'type': 'library', 'bom-ref': 'pkg:generic/zlib@1.3.1', 'name': 'zlib', 'version': '1.3.1'} - return Contributions(components=[zlib], dependencies=[], root_properties=[], tools=[]) + return Contributions(components=[zlib]) Worked example: virtualenv -~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------- -`virtualenv `_ ships the pip and setuptools wheels it installs into new environments, -under ``src/virtualenv/seed/wheels/embed``. A ``BUNDLE_SUPPORT`` table in that package's ``__init__.py``, after a -``BUNDLE_FOLDER`` assignment, decides which Python versions receive each wheel. The project already has a hatchling -`custom build hook `_, so it subclasses -:class:`~python_sbom.hatch.SbomBuildHook` there, and the options live in the custom hook's table: +`virtualenv `_ ships the pip and setuptools wheels it installs into new +environments, under ``src/virtualenv/seed/wheels/embed``. A ``BUNDLE_SUPPORT`` table in that package's ``__init__.py``, +after a ``BUNDLE_FOLDER`` assignment, decides which Python versions receive each wheel. Its options name the supplier +and the wheels: .. code-block:: toml [build-system] build-backend = 'hatchling.build' - requires = ['hatch-vcs>=0.5', 'hatchling>=1.28', 'python-sbom[hatch]>=0.1'] + requires = ['hatch-vcs>=0.5', 'hatchling>=1.28', "python-sbom[hatch]>=0.1; python_version >= '3.11'"] [tool.hatch.build.hooks.custom] path = 'hatch_build.py' + + [tool.python-sbom] supplier = { name = 'Python Packaging Authority', url = ['https://www.pypa.io'] } bundled-wheels = ['src/virtualenv/seed/wheels/embed/*.whl'] -Its ``hatch_build.py`` overrides :meth:`~python_sbom.hatch.SbomBuildHook.content_provider` to hand over a provider that -reads the table next to the wheels by name, without importing the package. The test suite builds a project with -virtualenv's layout from this script: +virtualenv still builds on Python versions python-sbom does not support, so its ``hatch_build.py`` loads the hook +only where the requirement's marker installs it, and hands over a provider that reads the table by name, without +importing the package. The test suite builds a project with virtualenv's layout from this script: .. literalinclude:: ../tests/_seeder/hatch_build.py :language: python The ``get_build_hook`` function tells hatchling which class to load, since the module also sees the base class it -imports. The SBOM lists the script among the build tools, hashed. The pip component then carries these properties, in -this order: +imports. The SBOM lists the script among the build tools, hashed. The pip component then carries these properties, +in this order: .. list-table:: :header-rows: 1 @@ -67,7 +75,7 @@ this order: * - Property - Value * - ``virtualenv:bundled-wheel`` - - ``src/virtualenv/seed/wheels/embed/pip-26.1-py3-none-any.whl`` + - ``virtualenv/seed/wheels/embed/pip-26.1-py3-none-any.whl`` * - ``virtualenv:seeded-for-python`` - ``3.14`` * - ``virtualenv:seeded-for-python`` @@ -75,78 +83,23 @@ this order: * - ``python:requires-python`` - ``>=3.10`` -A project that still builds on a Python older than python-sbom supports, as virtualenv does on 3.9 and 3.10, puts an -environment marker on the requirement, ``python-sbom[hatch]>=0.1; python_version >= '3.11'``. Its script then falls -back to hatchling's plain hook there, so those builds produce a wheel without an SBOM: - -.. code-block:: python - - import sys +A bundled wheel is named by its path inside the artifact, so the wheel's SBOM and the zipapp's give it the same +``bom-ref``. - if sys.version_info >= (3, 11): - from python_sbom.hatch import SbomBuildHook as BaseHook - else: # python-sbom needs Python 3.11; older builds get no SBOM - from hatchling.builders.hooks.plugin.interface import BuildHookInterface as BaseHook +Describe another archive +------------------------ -Add another build backend -------------------------- +virtualenv also publishes a zipapp, ``virtualenv.pyz``, that holds virtualenv's own ``.dist-info`` at its top level, +the wheels it seeds, and its dependencies unpacked under ``__virtualenv__/``. ``python-sbom archive`` describes such an +archive the way the hook describes a wheel: the archive's own files, the project's distribution, each wheel it +bundles as a ``.whl`` member, and each one unpacked into a directory with a ``.dist-info``. The content provider +hears of every bundled wheel, and its context reads the archive's files: -A :class:`~python_sbom.Backend` says what a build system knows about the wheel: :meth:`~python_sbom.Backend.request` -returns the project directory, the core metadata, the wheel's file name, the settings and the build scripts. Two -backends ship: +.. code-block:: console -- :class:`~python_sbom.MetadataBackend` takes the metadata as text, for a build hook of another backend, such as a PDM - or scikit-build-core plugin, that has it before the wheel exists. -- :class:`~python_sbom.BuiltWheel` reads it from a finished wheel, as ``python-sbom embed`` does. - -The pipeline is the same for every backend: - -.. code-block:: python - - from python_sbom import ( - ContentProvider, - MetadataBackend, - build_sbom, - collect_facts, - default_revision, - read_source_tree, - settings_from_pyproject, - ) - - settings = settings_from_pyproject(root, {}) - request = MetadataBackend( - root=root, metadata=core_metadata, wheel_name=wheel_name, settings=settings, build_scripts=() - ).request() - facts = collect_facts(root, default_revision, None, None) - sbom = build_sbom(request, read_source_tree(request), facts, ContentProvider()) - -``sbom.file_name`` and ``sbom.content`` go into the wheel's ``.dist-info/sboms/``; a backend that can only rewrite a -finished wheel passes them to :func:`~python_sbom.embed_sbom`. In the hatch hook, -:meth:`~python_sbom.hatch.SbomBuildHook.backend` returns the backend, so a subclass can swap it. - -Describe another artifact -------------------------- - -virtualenv also publishes a zipapp. Its SBOM describes what the archive holds and records the same build toolchain, -time and serial namespace as the wheel's. The pieces are public for that: - -- :func:`~python_sbom.collect_facts` reads the build time from ``SOURCE_DATE_EPOCH``, the source revision and the - interpreter and distributions of the build environment, once, at the edge. -- :func:`~python_sbom.tool_components` turns those facts into build tool components and their dependencies, and - :func:`~python_sbom.formulation` records the build. -- :func:`~python_sbom.file_component_from_bytes`, :func:`~python_sbom.bundled_wheel_component` and - :func:`~python_sbom.component_from_metadata` describe what the artifact holds. -- :func:`~python_sbom.project_identity` names the project the way its wheel SBOM does, and :func:`~python_sbom.document` - adds the CycloneDX header, with a serial number derived from the content. - -This function, which the test suite runs and validates, describes a zipapp and the files it holds: - -.. literalinclude:: examples/zipapp.py - :language: python - :pyobject: describe_zipapp + $ python-sbom archive virtualenv.pyz --build-script tasks/make_zipapp.py --embed + wrote virtualenv.pyz.cdx.json + wrote virtualenv.pyz.cdx.json into virtualenv.pyz -:func:`~python_sbom.validate` takes the JSON text and needs the ``validate`` extra: -``validate(dumps(document).decode(), None)`` checks the result the way ``python-sbom validate`` checks a wheel's SBOM, -and an allowed license set in place of ``None`` adds the license policy. The ``:sbom:file`` metadata property -marks the document as built from these pieces; without it, the validator treats the document as another tool's and -checks the schema alone. +The document takes its options, source revision, build time and toolchain the way the wheel's does, so both name the +project alike. ``--embed`` adds it to the archive's top level, where ``python-sbom validate virtualenv.pyz`` finds it. diff --git a/docs/py_api.rst b/docs/py_api.rst index 8ba4a85..bfdea4b 100644 --- a/docs/py_api.rst +++ b/docs/py_api.rst @@ -1,8 +1,9 @@ Python API ========== -The core is importable from ``python_sbom`` without hatchling; the hatchling build hook lives in ``python_sbom.hatch``. -:doc:`explanation` shows how the pieces fit together. +The Python API is what a project needs to add its own knowledge to the SBOM: the hatchling build hook and the content +provider it calls, with the types those take and return. Everything else, from embedding to validation, runs through +the :doc:`cli`. The core is importable from ``python_sbom`` without hatchling; the hook lives in ``python_sbom.hatch``. .. py:data:: python_sbom.__version__ :type: str @@ -13,172 +14,74 @@ Hatchling build hook -------------------- .. autoclass:: python_sbom.hatch.SbomBuildHook - :members: content_provider, backend + :members: content_provider -Pipeline --------- - -.. autofunction:: python_sbom.build_sbom - -.. autoclass:: python_sbom.Sbom - :members: content - -.. autoclass:: python_sbom.Backend - :members: request - -.. autoclass:: python_sbom.MetadataBackend - -.. autoclass:: python_sbom.BuiltWheel +Content provider +---------------- .. autoclass:: python_sbom.ContentProvider :members: bundled_wheel, contributions -.. autofunction:: python_sbom.load_content_provider - -.. autofunction:: python_sbom.embed_sbom - -Collecting inputs ------------------ - -.. autofunction:: python_sbom.settings_from_pyproject - -.. autofunction:: python_sbom.settings_from_table - -.. py:data:: python_sbom.Toml - - A value of a TOML table, as :mod:`tomllib` or hatchling hand it over: a string, number, boolean, date or time, or - an array or table of them. - -.. autofunction:: python_sbom.collect_facts - -.. autoclass:: python_sbom.RevisionReader - :special-members: __call__ - -.. autofunction:: python_sbom.default_revision - -.. autofunction:: python_sbom.git_revision - -.. autofunction:: python_sbom.read_source_tree - -.. autofunction:: python_sbom.read_bundled_wheel - -Inputs ------- - -.. autoclass:: python_sbom.SbomRequest - -.. autoclass:: python_sbom.Settings - -.. autoclass:: python_sbom.BuildFacts - -.. autoclass:: python_sbom.Interpreter - -.. autoclass:: python_sbom.InstalledDistribution - -.. autoclass:: python_sbom.Revision - -.. autoclass:: python_sbom.RecordRow - -.. autoclass:: python_sbom.SourceTree - -.. autoclass:: python_sbom.BundledWheel - -.. autoclass:: python_sbom.BuildScript +.. autoclass:: python_sbom.ProviderContext + :members: read .. autoclass:: python_sbom.BundledWheelInfo .. autoclass:: python_sbom.Contributions -Building blocks ---------------- - -.. autofunction:: python_sbom.component_from_metadata - -.. autofunction:: python_sbom.bundled_wheel_component - -.. autofunction:: python_sbom.file_component_from_bytes - -.. autofunction:: python_sbom.script_tool - -.. autofunction:: python_sbom.tool_components - -.. autofunction:: python_sbom.formulation - -.. autofunction:: python_sbom.project_identity +.. autoclass:: python_sbom.Settings .. autoclass:: python_sbom.ProjectIdentity -.. autofunction:: python_sbom.document - -.. autofunction:: python_sbom.dumps - -SPDX and validation -------------------- - -.. autofunction:: python_sbom.to_spdx - -.. autofunction:: python_sbom.validate - -.. autofunction:: python_sbom.validate_wheel - -Errors ------- - -.. autoexception:: python_sbom.SbomError - -.. autoexception:: python_sbom.SettingsError - -.. autoexception:: python_sbom.WheelError - -.. autoexception:: python_sbom.SourceRevisionError - -.. autoexception:: python_sbom.MissingExtraError - -Types ------ +Component types +--------------- -These are :class:`~typing.TypedDict` types of the JSON documents, for type checking code that builds or reads them. Each -follows the CycloneDX 1.6 JSON object of the same name; the `CycloneDX 1.6 reference -`_ describes every key. +These :class:`~typing.TypedDict` types describe the JSON a provider builds. Each follows the CycloneDX 1.6 object of the +same name; the `CycloneDX 1.6 reference `_ describes every key. -.. py:class:: python_sbom.Document +.. py:class:: python_sbom.Component - A complete CycloneDX 1.6 document: the ``$schema``, ``bomFormat``, ``specVersion``, ``serialNumber`` and ``version`` - header, plus the keys of :class:`~python_sbom.Body`. See the `BOM `_ - reference. + A CycloneDX 1.6 component, a distribution, an application, the interpreter or a file, as a dictionary with the keys + the CycloneDX component object defines. python-sbom uses ``type``, ``bom-ref``, ``name``, ``version``, ``purl``, + ``supplier`` (an :class:`~python_sbom.OrganizationalEntity`), ``authors`` (:class:`~python_sbom.Contact`), + ``description``, ``licenses`` (:class:`~python_sbom.License`), ``copyright``, ``hashes`` + (:class:`~python_sbom.Hash`), ``externalReferences`` (:class:`~python_sbom.ExternalReference`), ``properties`` + (:class:`~python_sbom.Property`), ``evidence`` (:class:`~python_sbom.Evidence`) and the nested ``components``. -.. autoclass:: python_sbom.Body +.. autoclass:: python_sbom.Property :no-members: -.. py:class:: python_sbom.Component +.. autoclass:: python_sbom.Dependency + :no-members: - A distribution, application, platform or file, with ``type``, ``bom-ref`` and ``name`` and the optional - ``version``, ``purl``, ``supplier``, ``authors``, ``description``, ``licenses``, ``copyright``, ``hashes``, - ``externalReferences``, ``properties``, ``evidence`` and nested ``components``. See the `component - `_ reference. +.. autoclass:: python_sbom.OrganizationalEntity + :no-members: -.. autoclass:: python_sbom.Dependency +.. autoclass:: python_sbom.Contact :no-members: -.. py:class:: python_sbom.Formula +.. autoclass:: python_sbom.ExternalReference + :no-members: - How an artifact was built: a ``bom-ref`` and one or more ``workflows``, each with the tools it used as - ``resourceReferences``, its ``outputs`` and, optionally, its ``inputs``. See the `formulation - `_ reference. +.. autoclass:: python_sbom.Hash + :no-members: .. autoclass:: python_sbom.License :no-members: -.. autoclass:: python_sbom.OrganizationalEntity +.. autoclass:: python_sbom.LicenseInfo :no-members: -.. autoclass:: python_sbom.Property +.. autoclass:: python_sbom.Evidence :no-members: -.. autoclass:: python_sbom.Spdx +.. autoclass:: python_sbom.Identity :no-members: -Command line ------------- +.. autoclass:: python_sbom.IdentityMethod + :no-members: -.. autofunction:: python_sbom.main +Errors +------ + +.. autoexception:: python_sbom.SbomError diff --git a/docs/usage.rst b/docs/usage.rst index 6756d1c..b953847 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -5,8 +5,9 @@ With hatchling -------------- Add the hook to ``build-system.requires`` and enable it with an empty table, as the quick start on the :doc:`index` -shows. The ``hatch`` extra brings hatchling; the command line and the library need none. The hook acts on wheel builds -only: an sdist and an editable wheel carry no SBOM. +shows. The ``hatch`` extra brings hatchling; the command line needs none. The hook writes the SBOM into wheels only: +an editable wheel carries none, and an sdist carries the source revision instead, so a wheel built from it still names +its commit. The hook writes the SBOM once hatchling has written the wheel, so the document describes the final file name and metadata, whatever dependencies or tag other hooks changed. Hatchling runs the hooks in the order of their tables; see @@ -16,9 +17,10 @@ for this hook. Options ~~~~~~~ -The options live in a ``[tool.python-sbom]`` table. Each is optional; an unknown key fails the build, so a typo cannot -pass silently. A key in the hook's own table, such as ``[tool.hatch.build.hooks.python-sbom]``, wins over the same key -in ``[tool.python-sbom]``. +The options live in a ``[tool.python-sbom]`` table, the one place the hook, ``python-sbom embed`` and ``python-sbom +archive`` all read, so every SBOM of a project names it alike. Each option is optional; an unknown key fails the +build, so a typo cannot pass silently. The hook's own table holds only hatchling's options, and a python-sbom option +there fails the build with a pointer to ``[tool.python-sbom]``. ``supplier`` The organization that supplies the project: a table with a ``name`` string and an optional ``url`` string array. It @@ -58,7 +60,7 @@ in ``[tool.python-sbom]``. ``content-provider`` A :class:`~python_sbom.ContentProvider`, as ``module:attribute`` importable from the project root, that adds what - the project knows beyond its metadata; :doc:`extend` shows one. + the project knows beyond its metadata; :doc:`extend` shows one. Loading it imports, and so runs, that module. .. code-block:: toml @@ -74,11 +76,19 @@ never from its own metadata. The source revision ~~~~~~~~~~~~~~~~~~~ -The SBOM names the commit the wheel was built from. It takes the first of: the ``PYTHON_SBOM_COMMIT`` environment -variable; git, for a checkout or any directory inside one; and the ``node`` line of an expanded ``.git_archival.txt``, -for a ``git archive``. A checkout with uncommitted changes to tracked files gets a ``:vcs-dirty`` property, and -the link to the revision says so. A wheel built from an sdist, as ``python -m build`` and ``uv build`` do by default, -has no checkout, so set ``PYTHON_SBOM_COMMIT`` or ship ``.git_archival.txt`` to keep the commit. +The SBOM names the commit the wheel was built from. It takes the first source that knows it: + +#. the ``PYTHON_SBOM_COMMIT`` environment variable, or ``--commit`` on the command line; +#. git, when the project's ``pyproject.toml`` is tracked by the checkout that holds it, so a project unpacked inside an + unrelated repository never borrows that repository's commit; +#. ``.python-sbom-revision``, which the hook writes into every sdist built from a clean checkout, so a wheel built from + the sdist, as ``python -m build`` and ``uv build`` do by default, keeps the commit; +#. the ``node`` line of an expanded ``.git_archival.txt``, for a ``git archive``. + +A commit must be a full hash of 40 or 64 hex characters. A checkout with changes, including untracked files that are +not ignored, gets a ``:vcs-dirty`` property, and the link to the revision says so. When git cannot read the +checkout, the build warns and moves on to the next source. git runs with ``core.fsmonitor`` off, so a repository's +configuration runs no command. With another build backend -------------------------- @@ -103,13 +113,14 @@ Build, then embed, from the project root: ``--commit`` names the source revision when the wheel came from an sdist, and ``--content-provider`` overrides the ``content-provider`` option. -The build tool record lists the environment named by ``--build-env``, by default the one ``embed`` runs in. To record -the real toolchain, point ``--build-env`` at the build environment's ``site-packages``, or build without isolation in -the same locked environment. On Linux, meson-python also needs ``patchelf`` there: +The build tool record lists the packages of the environment named by ``--build-env``, by default the one ``embed`` runs +in; the interpreter it records is always the one running ``embed``. To record the real toolchain, build without +isolation in the environment ``embed`` runs in, from the same lock. On Linux, meson-python also needs ``patchelf`` +there: .. code-block:: console - $ python -m pip install python-sbom meson-python meson ninja patchelf + $ python -m pip install build python-sbom meson-python meson ninja patchelf $ python -m build --wheel --no-isolation $ python-sbom embed dist/*.whl From d4ebc944d36d6f24381536ce083ba3c691327731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 23:09:23 -0800 Subject: [PATCH 022/115] =?UTF-8?q?=F0=9F=94=A7=20chore:=20follow=20bump-d?= =?UTF-8?q?eps-index=20tooling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .git_archival.txt | 3 +++ .gitattributes | 1 + .github/workflows/check.yaml | 2 +- .github/workflows/prepare-release.yaml | 2 +- .pre-commit-config.yaml | 24 ++++++++++++++++++++++-- .readthedocs.yaml | 6 +++--- 6 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 .git_archival.txt create mode 100644 .gitattributes diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 0000000..7c51009 --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,3 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a94cb2f --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.git_archival.txt export-subst diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index f74b8b8..ad01f2c 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -46,7 +46,7 @@ jobs: dry-run: true hooks: name: ๐Ÿงน Hooks under the cooldown - uses: tox-dev/github-workflows/.github/workflows/cooled-hooks.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/.github/workflows/cooled-hooks.yaml@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 permissions: contents: read gate: diff --git a/.github/workflows/prepare-release.yaml b/.github/workflows/prepare-release.yaml index c0d478c..edfa8aa 100644 --- a/.github/workflows/prepare-release.yaml +++ b/.github/workflows/prepare-release.yaml @@ -22,7 +22,7 @@ permissions: {} jobs: prepare: name: ๐Ÿท๏ธ Prepare Towncrier release - uses: tox-dev/github-workflows/.github/workflows/prepare-towncrier-release.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/.github/workflows/prepare-towncrier-release.yaml@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 permissions: contents: read pull-requests: read diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 467121a..6969ab9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +# prek runs lower priorities first and equal ones concurrently, so hooks sharing a priority never write the same +# file; pre-commit ignores the key and runs hooks in file order. ci: autoupdate_schedule: quarterly default_language_version: @@ -7,42 +9,56 @@ repos: rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: - id: end-of-file-fixer + priority: 0 - id: trailing-whitespace + priority: 1 - repo: https://github.com/python-jsonschema/check-jsonschema rev: a76b92851fc9eede9f9bc4073eb55202f34eb0df # frozen: 0.39.0 hooks: - id: check-github-workflows + priority: 6 args: ["--verbose"] - repo: https://github.com/codespell-project/codespell rev: 57b21406f092110c18776e39b0bda50d37c945c8 # frozen: v2.4.3 hooks: - id: codespell + priority: 6 - repo: https://github.com/tox-dev/tox-toml-fmt rev: "5125f601c3e134347288c3d42f3b34298b7cf1be" # frozen: v1.11.1 hooks: - id: tox-toml-fmt + priority: 2 - repo: https://github.com/tox-dev/pyproject-fmt rev: "50e0f3aee9dc5747a5f173f79a7417403c18dda6" # frozen: v2.30.3 hooks: - id: pyproject-fmt + priority: 2 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "25bcf95026b96e2dd3c0aec3090c3d6ef6ca2e6c" # frozen: v0.17.3 + hooks: + - id: ruff-check + priority: 3 + args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] - repo: https://github.com/gaborbernat/simplint-pre-commit rev: "c8719dca99c13381570bc69f8955c9448fc271b7" # frozen: v0.7.1 hooks: - id: simplint + priority: 4 - repo: https://github.com/astral-sh/ruff-pre-commit rev: "25bcf95026b96e2dd3c0aec3090c3d6ef6ca2e6c" # frozen: v0.17.3 hooks: - id: ruff-format - - id: ruff-check - args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] + priority: 5 - repo: https://github.com/google/yamlfmt rev: "21ca5323a9c87ee37a434e0ca908efc0a89daa07" # frozen: v0.21.0 hooks: - id: yamlfmt + priority: 2 - repo: https://github.com/executablebooks/mdformat rev: 82912cdaea4fb830f751504486a7879c70526547 # frozen: 1.0.0 hooks: - id: mdformat + priority: 2 args: ["--wrap", "120"] additional_dependencies: - mdformat-gfm==1.0.0 @@ -51,6 +67,7 @@ repos: rev: c1813841673cdcd6cf602dde1edb78a5d5e45235 # frozen: v2.2.1 hooks: - id: docstrfmt + priority: 2 files: ^docs/(changelog|contributing)[.]rst$ args: ["-l", "120", "--preserve-adornments"] additional_dependencies: @@ -59,8 +76,11 @@ repos: rev: c92afdf2c6ca336aaa8f39e2bd8d48f62dbd8e1f # frozen: v1.31.1 hooks: - id: zizmor + priority: 6 args: ["--min-severity", "high"] - repo: meta hooks: - id: check-hooks-apply + priority: 6 - id: check-useless-excludes + priority: 6 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index a173db1..6f130a0 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,6 +3,6 @@ build: os: ubuntu-lts-latest tools: {} commands: - - curl -LsSf https://astral.sh/uv/0.13.0/install.sh | sh - - ~/.local/bin/uv tool install tox==4.64.7 --with tox-uv==1.36.1 -p 3.15 --managed-python - - ~/.local/bin/tox run -e docs -- + - curl -LsSf https://astral.sh/uv/install.sh | sh + - ~/.local/bin/uv tool install tox --with tox-uv -p 3.15 --managed-python + - ~/.local/bin/tox run -e docs From 86def472b1a4f2a905fb569b21b090578bd2257c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 6 Nov 2026 23:10:25 -0800 Subject: [PATCH 023/115] =?UTF-8?q?=F0=9F=93=9D=20docs:=20match=20the=20vi?= =?UTF-8?q?rtualenv=20example=20to=20its=20patch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/explanation.rst | 10 +++++----- docs/extend.rst | 34 +++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/docs/explanation.rst b/docs/explanation.rst index 4375ee1..f9796c1 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -49,13 +49,13 @@ points at what a package imports, and a dashed arrow at what it imports only for class hook,cli,collect,api edge; class generate,validate,formats,wheel,model core; -The model imports nothing else of the package but its errors, and only the hook imports hatchling, so the provider API -and the command line work without it. Every package raises subclasses of :class:`~python_sbom.SbomError`, which the command -line reports without a traceback. +The model imports nothing else of the package but its errors, and only the hook imports hatchling, so the provider +API and the command line work without it. Every package raises subclasses of :class:`~python_sbom.SbomError`, which the +command line reports without a traceback. The one extension point is the :class:`~python_sbom.ContentProvider`, which every entry point loads from the -``content-provider`` option. A hatchling custom hook can override :meth:`~python_sbom.hatch.SbomBuildHook.content_provider` to build the provider -itself. +``content-provider`` option. A hatchling custom hook can override +:meth:`~python_sbom.hatch.SbomBuildHook.content_provider` to build the provider itself. What the SBOM records --------------------- diff --git a/docs/extend.rst b/docs/extend.rst index 9ec3d5e..f6c013d 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -57,16 +57,34 @@ and the wheels: [tool.python-sbom] supplier = { name = 'Python Packaging Authority', url = ['https://www.pypa.io'] } bundled-wheels = ['src/virtualenv/seed/wheels/embed/*.whl'] + content-provider = 'tasks.sbom_content:VirtualenvContent' -virtualenv still builds on Python versions python-sbom does not support, so its ``hatch_build.py`` loads the hook -only where the requirement's marker installs it, and hands over a provider that reads the table by name, without -importing the package. The test suite builds a project with virtualenv's layout from this script: +virtualenv still builds on Python versions python-sbom does not support, so its ``hatch_build.py`` loads the hook only +where the requirement's marker installs it, and falls back to hatchling's plain hook elsewhere: + +.. code-block:: python + + import sys + + if sys.version_info >= (3, 11): + from python_sbom.hatch import SbomBuildHook + else: # python-sbom needs Python 3.11, so builds on older interpreters carry no SBOM + from hatchling.builders.hooks.plugin.interface import BuildHookInterface as SbomBuildHook + + + def get_build_hook(): + return SbomBuildHook + +The ``get_build_hook`` function tells hatchling which class to load. The SBOM lists the script among the build tools, +hashed. The provider reads the table next to each wheel through the context, by name, without importing the package, +and names the project's supplier as the wheels' supplier, since virtualenv bundles only PyPA's own projects. The test +suite builds a project with virtualenv's layout with the same provider, handed over from a hook subclass: .. literalinclude:: ../tests/_seeder/hatch_build.py :language: python + :pyobject: BundleSupport -The ``get_build_hook`` function tells hatchling which class to load, since the module also sees the base class it -imports. The SBOM lists the script among the build tools, hashed. The pip component then carries these properties, +The pip component then carries these properties, in this order: .. list-table:: @@ -101,5 +119,7 @@ hears of every bundled wheel, and its context reads the archive's files: wrote virtualenv.pyz.cdx.json wrote virtualenv.pyz.cdx.json into virtualenv.pyz -The document takes its options, source revision, build time and toolchain the way the wheel's does, so both name the -project alike. ``--embed`` adds it to the archive's top level, where ``python-sbom validate virtualenv.pyz`` finds it. +The document takes its options, source revision and toolchain the way the wheel's does, so both name the project +alike. Its build time comes from ``SOURCE_DATE_EPOCH``, or the current time without it; set ``SOURCE_DATE_EPOCH`` for +both builds to give them one time. ``--embed`` adds the document to the archive's top level, where ``python-sbom +validate virtualenv.pyz`` finds it. From 6e4335e945d410b16309803a3874fd93dddfeab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:17:35 -0800 Subject: [PATCH 024/115] =?UTF-8?q?=F0=9F=9A=A7=20wip:=20adopt=20libraries?= =?UTF-8?q?,=20cap=20validation,=20bench?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Work in progress, split across agents from here: - core uses installer, packageurl-python, dep-logic, license-expression and packaging.direct_url; build-system requires them for the self-SBOM - validate caps components at 5,000, raised with --max-components - the sdist revision file is written as bytes, so it holds LF on Windows - bench/ benchmark harness scaffolding --- bench/__init__.py | 1 + bench/__main__.py | 325 +++++++++++++++++++++++++ bench/_cases.py | 215 ++++++++++++++++ bench/_fixtures.py | 316 ++++++++++++++++++++++++ docs/extend.rst | 15 +- docs/usage.rst | 14 +- pyproject.toml | 10 +- src/python_sbom/_cli/_commands.py | 12 +- src/python_sbom/_cli/_options.py | 1 + src/python_sbom/_collect/_content.py | 11 +- src/python_sbom/_collect/_facts.py | 17 +- src/python_sbom/_collect/_source.py | 89 ++++--- src/python_sbom/_generate/_archive.py | 5 +- src/python_sbom/_model/_components.py | 7 +- src/python_sbom/_model/_markers.py | 37 +-- src/python_sbom/_model/_names.py | 3 +- src/python_sbom/_model/_tools.py | 4 +- src/python_sbom/_validate/_artifact.py | 12 +- src/python_sbom/_validate/_document.py | 40 +-- src/python_sbom/_validate/_licenses.py | 130 +++++----- src/python_sbom/_wheel/_archive.py | 55 +++-- src/python_sbom/_wheel/_record.py | 41 +--- src/python_sbom/hatch/_hook.py | 5 +- tests/_cli/test_commands.py | 29 +++ tests/_collect/test_content.py | 18 +- tests/_collect/test_facts.py | 29 +++ tests/_collect/test_source.py | 129 +++++++++- tests/_model/test_components.py | 24 ++ tests/_model/test_markers.py | 10 + tests/_validate/test_artifact.py | 8 + tests/_validate/test_document.py | 26 +- tests/_validate/test_licenses.py | 74 ++++-- tests/_wheel/test_archive.py | 16 +- tests/_wheel/test_record.py | 55 +---- 34 files changed, 1450 insertions(+), 333 deletions(-) create mode 100644 bench/__init__.py create mode 100644 bench/__main__.py create mode 100644 bench/_cases.py create mode 100644 bench/_fixtures.py diff --git a/bench/__init__.py b/bench/__init__.py new file mode 100644 index 0000000..c03b83b --- /dev/null +++ b/bench/__init__.py @@ -0,0 +1 @@ +"""Benchmarks of python-sbom's hot paths, timed with pyperf; run them with ``python -m bench``.""" diff --git a/bench/__main__.py b/bench/__main__.py new file mode 100644 index 0000000..b3d8f71 --- /dev/null +++ b/bench/__main__.py @@ -0,0 +1,325 @@ +""" +CLI entry point: ``python -m bench ``, run from the project root or with the root on ``PYTHONPATH``. + +``prepare`` + build the fixtures into the work directory; the only command that needs the network +``run [CASE ...] [-- PYPERF-OPTION ...]`` + time each case (all by default) in its own pyperf process tree, one JSON file per case under + ``//``; options after ``--`` reach pyperf, overriding the case's defaults (e.g. ``--rigorous``) +``profile [CASE ...]`` + run each case once under cProfile, after one untimed warm-up, and write its stats, the top functions by + cumulative and own time, and with ``--py-spy`` a speedscope flame graph +``report PATH ...`` + render pyperf JSON files, or directories of them, as a Markdown table +``list`` + print the case names + +The timed process imports whichever python-sbom the running interpreter has installed, so comparing two revisions +means running the same harness from two environments. +""" + +from __future__ import annotations + +import argparse +import cProfile +import io +import os +import pstats +import shutil +import subprocess +import sys +import tempfile +import time +from operator import itemgetter +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from ._cases import CASES, TIERS, make_step +from ._fixtures import load, prepare + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + from ._cases import Step + +_ROOT: Final[Path] = Path(__file__).parents[1] +_PYTHON: Final[str] = f"{sys.implementation.name}{sys.version_info.major}.{sys.version_info.minor}" +_TOP: Final[int] = 15 +_CLOCKS: Final[tuple[str, ...]] = ("wall", "cpu") + + +def main(arguments: Sequence[str]) -> None: + if arguments[:1] == ["worker"]: + _worker(arguments[1], Path(arguments[2]), arguments[3], arguments[4:]) + return + if arguments[:1] == ["profile-worker"]: + _profile_case(arguments[1], Path(arguments[2]), Path(arguments[3])) + return + passthrough: list[str] = [] + if "--" in arguments: + split = arguments.index("--") + arguments, passthrough = arguments[:split], list(arguments[split + 1 :]) + options = _parser().parse_args(arguments) + match options.command: + case "prepare": + prepare(options.work, options.project, options.virtualenv) + case "run": + out = options.out / (_PYTHON if options.clock == "wall" else f"{_PYTHON}-{options.clock}") + _run(options.work, out, _selected(options.cases), options.clock, passthrough) + case "profile": + _profile(options.work, options.out / _PYTHON / "profile", _selected(options.cases), py_spy=options.py_spy) + case "once": + _once(options.work, options.case) + case "report": + sys.stdout.write(_report(options.paths)) + case "list": + sys.stdout.write("".join(f"{name}\n" for name in CASES)) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="python -m bench") + commands = parser.add_subparsers(dest="command", required=True) + work = argparse.ArgumentParser(add_help=False) + work.add_argument("--work", type=Path, default=_ROOT / ".tox" / "bench-work", help="fixture directory") + output = argparse.ArgumentParser(add_help=False) + output.add_argument("--out", type=Path, default=_ROOT / ".tox" / "bench-results", help="result directory") + setup = commands.add_parser("prepare", parents=[work], help="build the fixtures") + setup.add_argument("--project", type=Path, default=_ROOT, help="python-sbom checkout whose own wheel to build") + setup.add_argument("--virtualenv", type=Path, help="virtualenv checkout carrying [tool.python-sbom]") + for name, text in (("run", "time cases with pyperf"), ("profile", "profile cases once")): + command = commands.add_parser(name, parents=[work, output], help=text) + command.add_argument("cases", nargs="*", metavar="CASE", help="cases to run (default: all)") + commands.choices["run"].add_argument( + "--clock", + choices=_CLOCKS, + default="wall", + help="wall time, or the CPU time of the process and its children, steadier on a loaded machine", + ) + commands.choices["profile"].add_argument("--py-spy", action="store_true", help="also record with py-spy") + commands.add_parser("once", parents=[work], help="run one case once, for an outside profiler").add_argument("case") + commands.add_parser("report", help="render results as Markdown").add_argument("paths", nargs="+", type=Path) + commands.add_parser("list", help="print the case names") + return parser + + +def _selected(names: list[str]) -> list[str]: + if unknown := sorted(set(names) - CASES.keys()): + msg = f"unknown case(s) {', '.join(unknown)}; known: {', '.join(CASES)}" + raise SystemExit(msg) + return names or list(CASES) + + +def _run(work: Path, out: Path, names: list[str], clock: str, passthrough: list[str]) -> None: + out.mkdir(parents=True, exist_ok=True) + fixtures: Final = load(work) + for name in names: + if missing := [key for key in CASES[name].needs if key not in fixtures]: + sys.stdout.write(f"skip {name}: no fixture {', '.join(missing)}\n") + continue + (result := out / f"{name}.json").unlink(missing_ok=True) + subprocess.run( + [ + sys.executable, + "-m", + "bench", + "worker", + name, + str(work), + clock, + "--output", + str(result), + "--inherit-environ", + "PYTHONPATH", + *TIERS[CASES[name].tier], + *passthrough, + ], + check=True, + ) + + +def _worker(name: str, work: Path, clock: str, arguments: Sequence[str]) -> None: + import pyperf # ruff: ignore[import-outside-top-level] # only the timed processes need it + + runner = pyperf.Runner(program_args=("-m", "bench", "worker", name, str(work), clock)) + runner.parse_args(list(arguments)) + with tempfile.TemporaryDirectory(dir=work, prefix="scratch-") as scratch: + step = make_step(CASES[name], load(work), Path(scratch)) + runner.metadata["python_sbom_version"] = _python_sbom_version() + runner.metadata["bench_clock"] = clock + runner.bench_time_func(name, _timed, step, time.perf_counter if clock == "wall" else _cpu_time) + + +def _timed(loops: int, step: Step, clock: Callable[[], float]) -> float: + total = 0.0 + for _ in range(loops): + if step.reset is not None: + step.reset() + start = clock() + step.run() + total += clock() - start + return total + + +def _cpu_time() -> float: + # children count once waited for, which covers git and the cold cases' interpreter + import resource # ruff: ignore[import-outside-top-level] # Unix only, and only the cpu clock needs it + + own, children = resource.getrusage(resource.RUSAGE_SELF), resource.getrusage(resource.RUSAGE_CHILDREN) + return own.ru_utime + own.ru_stime + children.ru_utime + children.ru_stime + + +def _once(work: Path, name: str) -> None: + with tempfile.TemporaryDirectory(dir=work, prefix="scratch-") as scratch: + step = make_step(CASES[name], load(work), Path(scratch)) + if step.reset is not None: + step.reset() + step.run() + + +def _profile(work: Path, out: Path, names: list[str], *, py_spy: bool) -> None: + out.mkdir(parents=True, exist_ok=True) + fixtures: Final = load(work) + summary: Final[list[str]] = [] + for name in names: + if any(key not in fixtures for key in CASES[name].needs): + continue + stats_file = out / f"{name}.pstats" + # each case in a fresh process, so one case's warm caches never shorten another's profile + subprocess.run([sys.executable, "-m", "bench", "profile-worker", name, str(work), str(stats_file)], check=True) + summary.append(_hot_spots(name, stats_file)) + (out / f"{name}.txt").write_text(_stats_text(stats_file), encoding="utf-8") + # next to the interpreter too, as a venv's scripts are when its python runs without activating it + if py_spy and ( + spy := shutil.which("py-spy", path=f"{Path(sys.executable).parent}{os.pathsep}{os.environ.get('PATH', '')}") + ): + subprocess.run( + [ + spy, + "record", + "--format", + "speedscope", + "--rate", + "500", + "--output", + str(out / f"{name}.speedscope.json"), + "--", + sys.executable, + "-m", + "bench", + "once", + name, + "--work", + str(work), + ], + check=True, + ) + (target := out / "hot-spots.md").write_text("\n".join(summary), encoding="utf-8") + sys.stdout.write(f"wrote {target}\n") + + +def _profile_case(name: str, work: Path, stats_file: Path) -> None: + with tempfile.TemporaryDirectory(dir=work, prefix="scratch-") as scratch: + step = make_step(CASES[name], load(work), Path(scratch)) + if step.command is not None: + if step.reset is not None: + step.reset() + command = [step.command[0], "-m", "cProfile", "-o", str(stats_file), *step.command[1:]] + subprocess.run(command, check=True, stdout=subprocess.DEVNULL) + return + if CASES[name].tier != "huge": # warm, as the timed values are; the huge case loads its validator on import + if step.reset is not None: + step.reset() + step.run() + if step.reset is not None: + step.reset() + profiler = cProfile.Profile() + profiler.runcall(step.run) + profiler.dump_stats(stats_file) + + +def _stats_text(stats_file: Path) -> str: + buffer = io.StringIO() + stats = pstats.Stats(str(stats_file), stream=buffer) + stats.sort_stats("cumulative").print_stats(40) + stats.sort_stats("tottime").print_stats(40) + return buffer.getvalue() + + +def _hot_spots(name: str, stats_file: Path) -> str: + stats = pstats.Stats(str(stats_file)) + entries = stats.stats # ty: ignore[unresolved-attribute] # the table pstats keeps is not in its stubs + total = sum(entry[2] for entry in entries.values()) or 1.0 + rows = [ + (_label(function), entry[1], entry[0], entry[2], entry[3]) + for function, entry in entries.items() + if not _harness(function) + ] + lines = [f"### {name}", "", f"profiled total {total:.3f}s", ""] + for title, key in (("cumulative", 4), ("own time", 3)): + lines += [ + f"by {title}:", + "", + "| function | calls | own s | cumulative s | cumulative % |", + "| --- | ---: | ---: | ---: | ---: |", + ] + lines += [ + f"| `{label}` | {calls if calls == primitive else f'{calls}/{primitive}'} | {own:.3f} | {cumulative:.3f} |" + f" {100 * cumulative / total:.1f} |" + for label, calls, primitive, own, cumulative in sorted(rows, key=itemgetter(key), reverse=True)[:_TOP] + ] + lines.append("") + return "\n".join(lines) + + +def _harness(function: tuple[str, int, str]) -> bool: + file, _, func = function + return Path(file).parent == Path(__file__).parent or func in { + "", + "", + } + + +def _label(function: tuple[str, int, str]) -> str: + file, line, func = function + if file == "~": + return func + parts = Path(file).parts + for anchor in ("site-packages", "lib"): + if anchor in parts: + parts = parts[len(parts) - parts[::-1].index(anchor) :] + break + return f"{'/'.join(parts[-3:])}:{line}({func})" + + +def _report(paths: list[Path]) -> str: + import pyperf # ruff: ignore[import-outside-top-level] # see _worker + + files = sorted(file for path in paths for file in (path.glob("*.json") if path.is_dir() else [path])) + lines = [ + "| path | python | median | IQR (p25 - p75) | min - max | values |", + "| --- | --- | ---: | ---: | ---: | ---: |", + ] + for file in files: + benchmark = pyperf.Benchmark.load(str(file)) + values = benchmark.get_values() + python = benchmark.get_metadata().get("python_version", "?") + lines.append( + f"| {benchmark.get_name()} | {python} | {_seconds(benchmark.median())} |" + f" {_seconds(benchmark.percentile(25))} - {_seconds(benchmark.percentile(75))} |" + f" {_seconds(min(values))} - {_seconds(max(values))} | {len(values)} |" + ) + return "\n".join(lines) + "\n" + + +def _seconds(value: float) -> str: + return f"{value:.2f} s" if value >= 1 else f"{value * 1000:.1f} ms" + + +def _python_sbom_version() -> str: + from python_sbom import __version__ # ruff: ignore[import-outside-top-level] # see _worker + + return __version__ + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/bench/_cases.py b/bench/_cases.py new file mode 100644 index 0000000..eda5998 --- /dev/null +++ b/bench/_cases.py @@ -0,0 +1,215 @@ +""" +The timed paths: each case turns the fixtures into an untimed reset and a timed step. + +A case runs in its own pyperf process tree, so module caches one case fills never speed up another. Steady-state cases +time the step in a warm process, as a build backend or a long-lived tool sees it; the ``cold-`` cases time a fresh +``python -m python_sbom`` process, imports and schema loading included, as a shell user sees it. +""" + +from __future__ import annotations + +import contextlib +import inspect +import io +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from ._fixtures import SOURCE_DATE_EPOCH + +if TYPE_CHECKING: + from collections.abc import Callable + +#: pyperf options per cost: many calibrated loops for cheap steps, fewer single-loop values for slow ones +TIERS: Final[dict[str, tuple[str, ...]]] = { + "fast": ("--processes", "10", "--values", "5", "--warmups", "1"), + "slow": ("--processes", "6", "--values", "3", "--warmups", "1", "--loops", "1"), + "huge": ("--processes", "3", "--values", "1", "--warmups", "0", "--loops", "1"), +} + + +@dataclass(frozen=True) +class Step: + run: Callable[[], object] + reset: Callable[[], object] | None = None + #: the command a cold case runs, which the profiler runs under cProfile instead of profiling the subprocess call + command: tuple[str, ...] | None = None + + +@dataclass(frozen=True) +class Case: + name: str + tier: str + needs: tuple[str, ...] + make: Callable[[dict[str, str], Path], Step] + + +def _hatch(root_key: str, *, hooks: bool) -> Callable[[dict[str, str], Path], Step]: + def make(fixtures: dict[str, str], scratch: Path) -> Step: + from hatchling.builders.wheel import WheelBuilder # ruff: ignore[import-outside-top-level] # only hatch cases + + if not hooks: + os.environ["HATCH_BUILD_NO_HOOKS"] = "true" + root = fixtures[root_key] + return Step( + run=lambda: list(WheelBuilder(root).build(directory=str(scratch), versions=["standard"])), + reset=lambda: _empty(scratch), + ) + + return make + + +def _embed(*, cold: bool) -> Callable[[dict[str, str], Path], Step]: + def make(fixtures: dict[str, str], scratch: Path) -> Step: + wheel = scratch / Path(fixtures["meson-wheel"]).name + arguments = ("embed", str(wheel), "--root", fixtures["meson"], "--build-env", fixtures["fat-env"]) + return Step( + run=_cold(arguments) if cold else _cli(arguments), + reset=lambda: shutil.copyfile(fixtures["meson-wheel"], wheel), + command=_command(arguments) if cold else None, + ) + + return make + + +def _archive(fixtures: dict[str, str], scratch: Path) -> Step: + return Step( + run=_cli(( + "archive", + fixtures["zipapp"], + "--root", + fixtures["virtualenv"], + "--build-script", + "tasks/make_zipapp.py", + "--output", + str(scratch / "virtualenv.pyz.cdx.json"), + )) + ) + + +def _validate( + key: str, *, expect_valid: bool = True, max_components: int | None = None +) -> Callable[[dict[str, str], Path], Step]: + def make(fixtures: dict[str, str], scratch: Path) -> Step: # ruff: ignore[unused-function-argument] # one signature + from python_sbom._validate._artifact import validate # ruff: ignore[import-outside-top-level] # loads the schema + + path = Path(fixtures[key]) + # revisions before the component cap take no limit and check any count, which is what a raised limit times + limit = ( + () + if max_components is None or "max_components" not in inspect.signature(validate).parameters + else (max_components,) + ) + + def run() -> None: + if (problems := validate(path, None, *limit)) and expect_valid: + msg = f"{path.name} is invalid: {problems}" + raise RuntimeError(msg) + + return Step(run=run) + + return make + + +def _cold_validate(fixtures: dict[str, str], scratch: Path) -> Step: # ruff: ignore[unused-function-argument] # one signature + arguments = ("validate", fixtures["own-wheel"]) + return Step(run=_cold(arguments), command=_command(arguments)) + + +def _spdx(fixtures: dict[str, str], scratch: Path) -> Step: + return Step(run=_cli(("spdx", fixtures["zipapp-sbom"], str(scratch / "virtualenv.spdx.json")))) + + +def _collect_facts(fixtures: dict[str, str], scratch: Path) -> Step: # ruff: ignore[unused-function-argument] # one signature + from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # see _validate + + root, path = Path(fixtures["meson"]), [fixtures["fat-env"]] + # no revision, so only reading the environment's distributions is timed + return Step(run=lambda: collect_facts(root, lambda _root: None, path=path)) + + +def _bundled(key: str) -> Callable[[dict[str, str], Path], Step]: + def make(fixtures: dict[str, str], scratch: Path) -> Step: # ruff: ignore[unused-function-argument] # one signature + from python_sbom._collect._source import read_bundled_wheel # ruff: ignore[import-outside-top-level] # see _validate + + wheel = Path(fixtures[key]) + data = wheel.read_bytes() + return Step(run=lambda: read_bundled_wheel(data, wheel.name)) + + return make + + +CASES: Final[dict[str, Case]] = { + case.name: case + for case in ( + Case("hatch-small", "slow", ("small",), _hatch("small", hooks=True)), + Case("hatch-small-nohook", "slow", ("small",), _hatch("small", hooks=False)), + Case("hatch-virtualenv", "slow", ("virtualenv",), _hatch("virtualenv", hooks=True)), + Case("hatch-virtualenv-nohook", "slow", ("virtualenv",), _hatch("virtualenv", hooks=False)), + Case("embed-meson-fat-env", "slow", ("meson", "meson-wheel", "fat-env"), _embed(cold=False)), + Case("cold-embed-meson-fat-env", "slow", ("meson", "meson-wheel", "fat-env"), _embed(cold=True)), + Case("archive-zipapp", "slow", ("virtualenv", "zipapp"), _archive), + Case("validate-own-wheel", "slow", ("own-wheel",), _validate("own-wheel")), + Case("cold-validate-own-wheel", "slow", ("own-wheel",), _cold_validate), + Case("validate-virtualenv-wheel", "slow", ("virtualenv-wheel",), _validate("virtualenv-wheel")), + Case("validate-zipapp", "slow", ("zipapp-embedded",), _validate("zipapp-embedded")), + # the default limit: a revision with the component cap times the rejection, one without the full schema check + Case("validate-hostile-10k", "huge", ("hostile",), _validate("hostile", expect_valid=False)), + # a limit above the count, so every revision times the full schema check + Case( + "validate-hostile-10k-uncapped", + "huge", + ("hostile",), + _validate("hostile", expect_valid=False, max_components=20_000), + ), + Case("spdx-zipapp", "fast", ("zipapp-sbom",), _spdx), + Case("collect-facts-fat-env", "slow", ("meson", "fat-env"), _collect_facts), + Case("bundled-wheel-pip", "fast", ("pip-wheel",), _bundled("pip-wheel")), + Case("bundled-wheel-setuptools", "fast", ("setuptools-wheel",), _bundled("setuptools-wheel")), + ) +} + + +def make_step(case: Case, fixtures: dict[str, str], scratch: Path) -> Step: + os.environ["SOURCE_DATE_EPOCH"] = SOURCE_DATE_EPOCH + if missing := [key for key in case.needs if key not in fixtures]: + msg = f"{case.name} needs the fixture(s) {', '.join(missing)}; prepare with --virtualenv to build them" + raise SystemExit(msg) + return case.make(fixtures, scratch) + + +def _cli(arguments: tuple[str, ...]) -> Callable[[], None]: + from python_sbom._cli._main import main # ruff: ignore[import-outside-top-level] # see _validate + + def run() -> None: + with contextlib.redirect_stdout(io.StringIO()): + main(list(arguments)) + + return run + + +def _command(arguments: tuple[str, ...]) -> tuple[str, ...]: + return (sys.executable, "-m", "python_sbom", *arguments) + + +def _cold(arguments: tuple[str, ...]) -> Callable[[], object]: + command = _command(arguments) + return lambda: subprocess.run(command, check=True, stdout=subprocess.DEVNULL) + + +def _empty(directory: Path) -> None: + shutil.rmtree(directory, ignore_errors=True) + directory.mkdir(parents=True) + + +__all__ = [ + "CASES", + "TIERS", + "Case", + "Step", + "make_step", +] diff --git a/bench/_fixtures.py b/bench/_fixtures.py new file mode 100644 index 0000000..7b9f864 --- /dev/null +++ b/bench/_fixtures.py @@ -0,0 +1,316 @@ +""" +Build every input the benchmarks read, once, into a work directory. + +This is the only step that needs the network: the fat build environment and virtualenv's zipapp download wheels. The +timed runs then read only what this wrote, listed in ``fixtures.json``. +""" + +from __future__ import annotations + +import base64 +import copy +import hashlib +import json +import os +import random +import shutil +import subprocess +import sys +import sysconfig +import zipfile +from pathlib import Path +from typing import Final + +#: a fixed build time, so every SBOM the fixtures and the timed runs write is the same +SOURCE_DATE_EPOCH: Final[str] = "1767225600" +#: resolve the fat environment as of this day, so every machine installs the same distributions +_EXCLUDE_NEWER: Final[str] = "2026-10-01T00:00:00Z" +# packages whose dependency closure gives a build environment of about seventy distributions, pure and platform ones, +# among them botocore, whose RECORD lists thousands of files +_FAT_PACKAGES: Final[tuple[str, ...]] = ( + "black", + "boto3", + "click", + "cryptography", + "httpx", + "hypothesis", + "jinja2", + "pydantic", + "pytest", + "pyyaml", + "requests", + "rich", + "setuptools", + "sphinx", + "tox", + "wheel", +) +_HOSTILE_COMPONENTS: Final[int] = 10_000 +_LICENSE: Final[str] = ( + "MIT License\n\nCopyright (c) 2026 Bench Authors\n\nPermission is hereby granted, free of charge.\n" +) +_GIT_IDENTITY: Final[tuple[str, ...]] = ("-c", "user.name=bench", "-c", "user.email=bench@example.org") + + +def prepare(work: Path, project: Path, virtualenv: Path | None) -> None: + work.mkdir(parents=True, exist_ok=True) + os.environ["SOURCE_DATE_EPOCH"] = SOURCE_DATE_EPOCH + fixtures: Final[dict[str, str]] = {} + fixtures["small"] = str(_small_project(work / "small")) + fixtures["small-wheel"] = str(_hatchling_wheel(work / "small", work / "small-dist")) + fixtures["hostile"] = str(_hostile_document(Path(fixtures["small-wheel"]), work / "hostile-10k.cdx.json")) + fixtures["meson"] = str(_meson_project(work / "meson")) + fixtures["meson-wheel"] = str(_meson_wheel(work / "meson-dist")) + fixtures["fat-env"] = _fat_environment(work / "fat-env") + _snapshot(project, work / "python-sbom") + fixtures["own-wheel"] = str(_hatchling_wheel(work / "python-sbom", work / "own-dist")) + if virtualenv is not None: + fixtures.update(_virtualenv(virtualenv, work)) + (work / "fixtures.json").write_text(json.dumps(fixtures, indent=2) + "\n", encoding="utf-8") + sys.stdout.write(f"wrote {work / 'fixtures.json'}\n") + + +def load(work: Path) -> dict[str, str]: + if not (manifest := work / "fixtures.json").is_file(): + msg = f"{manifest} does not exist; run `python -m bench prepare` first" + raise SystemExit(msg) + return json.loads(manifest.read_text(encoding="utf-8")) + + +def _small_project(root: Path) -> Path: + # a pure-Python project of forty modules that turns the hook on, as a typical hatchling user does + shutil.rmtree(root, ignore_errors=True) + (package := root / "src" / "proj").mkdir(parents=True) + (root / "LICENSE").write_text(_LICENSE, encoding="utf-8") + (root / "pyproject.toml").write_text( + """\ +[build-system] +build-backend = "hatchling.build" +requires = [ "hatchling", "python-sbom[hatch]" ] + +[project] +name = "proj" +version = "1.0" +license = "MIT" +license-files = [ "LICENSE" ] +requires-python = ">=3.11" +dependencies = [ "packaging>=24" ] + +[tool.hatch.build.targets.wheel] +packages = [ "src/proj" ] + +[tool.hatch.build.targets.wheel.hooks.python-sbom] + +[tool.python-sbom] +supplier = { name = "Bench Authors", url = [ "https://example.org" ] } +repository = "https://github.com/example/proj" +""", + encoding="utf-8", + ) + (package / "__init__.py").write_text('"""A small project."""\n', encoding="utf-8") + for index in range(40): + (package / f"module_{index:02d}.py").write_text( + "".join(f"def function_{name}() -> int:\n return {name}\n\n\n" for name in range(50)), encoding="utf-8" + ) + _commit(root) + return root + + +def _meson_project(root: Path) -> Path: + # the project directory `python-sbom embed` reads [tool.python-sbom] and the source revision from + shutil.rmtree(root, ignore_errors=True) + root.mkdir(parents=True) + (root / "LICENSE").write_text(_LICENSE, encoding="utf-8") + (root / "pyproject.toml").write_text( + """\ +[build-system] +build-backend = "mesonpy" +requires = [ "meson-python" ] + +[project] +name = "meson-proj" +version = "1.0" +license = "MIT" +license-files = [ "LICENSE" ] + +[tool.python-sbom] +supplier = { name = "Bench Authors", url = [ "https://example.org" ] } +""", + encoding="utf-8", + ) + _commit(root) + return root + + +def _meson_wheel(directory: Path) -> Path: + # what meson-python writes: a platform wheel holding an extension module next to its Python sources + shutil.rmtree(directory, ignore_errors=True) + directory.mkdir(parents=True) + abi = f"cp{sys.version_info.major}{sys.version_info.minor}" + platform = sysconfig.get_platform().replace("-", "_").replace(".", "_") + dist_info = "meson_proj-1.0.dist-info" + generator = random.Random(0) # ruff: ignore[suspicious-non-cryptographic-random-usage] # only needs the same bytes on every run + members: Final[dict[str, bytes]] = { + "meson_proj/__init__.py": b'"""A meson-python project."""\n', + f"meson_proj/_core{sysconfig.get_config_var('EXT_SUFFIX')}": generator.randbytes(4 * 1024 * 1024), + **{ + f"meson_proj/module_{index:02d}.py": "".join( + f"def function_{name}() -> int:\n return {name}\n\n\n" for name in range(50) + ).encode("utf-8") + for index in range(30) + }, + f"{dist_info}/METADATA": ( + b"Metadata-Version: 2.4\nName: meson-proj\nVersion: 1.0\nLicense-Expression: MIT\nLicense-File: LICENSE\n" + b"Requires-Python: >=3.11\nRequires-Dist: requests>=2\n" + ), + f"{dist_info}/WHEEL": ( + f"Wheel-Version: 1.0\nGenerator: meson\nRoot-Is-Purelib: false\nTag: {abi}-{abi}-{platform}\n" + ).encode(), + f"{dist_info}/licenses/LICENSE": _LICENSE.encode(), + } + record = "".join( + f"{name},sha256={base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b'=').decode()},{len(data)}\n" + for name, data in members.items() + ) + wheel = directory / f"meson_proj-1.0-{abi}-{abi}-{platform}.whl" + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as handle: + for name, data in {**members, f"{dist_info}/RECORD": f"{record}{dist_info}/RECORD,,\n".encode()}.items(): + handle.writestr(zipfile.ZipInfo(name, (2026, 1, 1, 0, 0, 0)), data, zipfile.ZIP_DEFLATED) + return wheel + + +def _fat_environment(target: Path) -> str: + if (uv := shutil.which("uv")) is None: + msg = "the fat build environment needs uv on PATH" + raise SystemExit(msg) + shutil.rmtree(target, ignore_errors=True) + _run([uv, "venv", "--quiet", "--python", sys.executable, str(target)]) + python = target / ("Scripts" if sys.platform == "win32" else "bin") / "python" + _run([ + uv, + "pip", + "install", + "--quiet", + "--python", + str(python), + "--exclude-newer", + _EXCLUDE_NEWER, + *_FAT_PACKAGES, + ]) + return _output([str(python), "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"]).strip() + + +def _hatchling_wheel(root: Path, directory: Path) -> Path: + # without isolation, in this environment, so the hook under test is the one installed here + shutil.rmtree(directory, ignore_errors=True) + _run([sys.executable, "-m", "hatchling", "build", "--target", "wheel", "--directory", str(directory)], cwd=root) + (wheel,) = directory.glob("*.whl") + return wheel + + +def _hostile_document(wheel: Path, target: Path) -> Path: + # ten thousand unique components: over the default component cap, which rejects the document before the schema + # check, and with a raised limit a uniqueItems check that compares every pair of them + with zipfile.ZipFile(wheel) as archive: + (name,) = (member for member in archive.namelist() if "/sboms/" in member) + document = json.loads(archive.read(name)) + hostile = copy.deepcopy(document) + hostile["components"] = [ + {"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(_HOSTILE_COMPONENTS) + ] + target.write_text(json.dumps(hostile), encoding="utf-8") + return target + + +def _virtualenv(source: Path, work: Path) -> dict[str, str]: + root = _snapshot(source, work / "virtualenv") + wheel = _hatchling_wheel(root, work / "virtualenv-dist") + zipapp = work / "virtualenv.pyz" + _run( + [sys.executable, "-m", "tasks.make_zipapp", "--wheel-dir", str(wheel.parent), "--dest", str(zipapp)], + cwd=root, + env={**os.environ, "PYTHONPATH": str(root)}, + ) + embedded = work / "virtualenv-sbom.pyz" + shutil.copyfile(zipapp, embedded) + sbom = work / "virtualenv.pyz.cdx.json" + _run([ + sys.executable, + "-m", + "python_sbom", + "archive", + str(embedded), + "--root", + str(root), + "--build-script", + "tasks/make_zipapp.py", + "--output", + str(sbom), + "--embed", + ]) + embed = root / "src" / "virtualenv" / "seed" / "wheels" / "embed" + return { + "virtualenv": str(root), + "virtualenv-wheel": str(wheel), + "zipapp": str(zipapp), + "zipapp-embedded": str(embedded), + "zipapp-sbom": str(sbom), + # the newest of each, the wheels virtualenv seeds on current Pythons + "pip-wheel": str(max(embed.glob("pip-*.whl"), key=_wheel_version)), + "setuptools-wheel": str(max(embed.glob("setuptools-*.whl"), key=_wheel_version)), + } + + +def _wheel_version(wheel: Path) -> tuple[int, ...]: + return tuple(int(part) for part in wheel.name.split("-")[1].split(".") if part.isdigit()) + + +def _snapshot(source: Path, target: Path) -> Path: + # a clone of the checkout plus its uncommitted changes, so builds write their version files there and never into + # the checkout itself; --shared borrows its objects, so even a detached HEAD resolves + shutil.rmtree(target, ignore_errors=True) + _run(["git", "clone", "--quiet", "--shared", "--no-checkout", str(source), str(target)]) + head = _output(["git", "-C", str(source), "rev-parse", "HEAD"]).strip() + _run(["git", "-C", str(target), "checkout", "--quiet", "--detach", head]) + if diff := subprocess.run( + ["git", "-C", str(source), "diff", "--binary", "HEAD"], check=True, capture_output=True + ).stdout: + subprocess.run(["git", "-C", str(target), "apply", "--index"], check=True, input=diff) + for name in _output(["git", "-C", str(source), "ls-files", "--others", "--exclude-standard", "-z"]).split("\0"): + if name: + (target / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source / name, target / name) + return target + + +def _commit(root: Path) -> None: + # a checkout, as projects build from one, so the timed runs include reading the source revision with git + _run(["git", "init", "--quiet", str(root)]) + _run(["git", "-C", str(root), "add", "."]) + _run([ + "git", + "-C", + str(root), + *_GIT_IDENTITY, + "commit", + "--quiet", + "--no-verify", + "--no-gpg-sign", + "-m", + "bench fixture", + ]) + + +def _run(command: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None) -> None: + subprocess.run(command, check=True, cwd=cwd, env=env) + + +def _output(command: list[str]) -> str: + return subprocess.run(command, check=True, capture_output=True, text=True).stdout + + +__all__ = [ + "SOURCE_DATE_EPOCH", + "load", + "prepare", +] diff --git a/docs/extend.rst b/docs/extend.rst index f6c013d..92e217d 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -1,9 +1,8 @@ Extend it ========= -The options cover what every project shares. A project that knows more than its metadata says adds it through a -content provider, and the same provider serves the wheel's SBOM and the SBOM of any other archive the project -publishes. +The options cover what every project shares. A project that knows more than its metadata says adds it through a content +provider, and the same provider serves the wheel's SBOM and the SBOM of any other archive the project publishes. Add content with a provider --------------------------- @@ -22,10 +21,9 @@ it adds. the root's dependencies and the serial number derive from it. The build fails when a ``bom-ref`` repeats, a dependency names one that no component has, or a root property uses the project's own prefix. -Name the provider in the ``content-provider`` option, as ``module:attribute`` importable from the project root, and -the hook, ``python-sbom embed`` and ``python-sbom archive`` all load it. When the provider's class lives in the -project, the SBOM lists its module among the build tools, hashed, because it shapes the document a rebuild has to -reproduce: +Name the provider in the ``content-provider`` option, as ``module:attribute`` importable from the project root, and the +hook, ``python-sbom embed`` and ``python-sbom archive`` all load it. When the provider's class lives in the project, the +SBOM lists its module among the build tools, hashed, because it shapes the document a rebuild has to reproduce: .. code-block:: python @@ -84,8 +82,7 @@ suite builds a project with virtualenv's layout with the same provider, handed o :language: python :pyobject: BundleSupport -The pip component then carries these properties, -in this order: +The pip component then carries these properties, in this order: .. list-table:: :header-rows: 1 diff --git a/docs/usage.rst b/docs/usage.rst index b953847..8c17a75 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -131,8 +131,7 @@ you publish; auditwheel's own SBOM in the same directory stays untouched. Render it as SPDX ----------------- -Some tools read only `SPDX `_. ``python-sbom spdx`` renders the CycloneDX document as SPDX 2.3 -JSON: +Some tools read only `SPDX `_. ``python-sbom spdx`` renders the CycloneDX document as SPDX 2.3 JSON: .. code-block:: console @@ -159,3 +158,14 @@ and validates the SPDX rendering; an SBOM another tool wrote, such as auditwheel Each ``--allow-license`` adds an SPDX id that bundled wheels may use. With any given, a bundled wheel whose license the ids do not allow fails the check, and so does one that declares a license only by name. + +Before the schema check, ``validate`` counts the components an SBOM lists, nested ones included, and fails an SBOM that +lists more than 5,000 without checking it further. The schema's ``uniqueItems`` rule compares every pair of items in an +array, so its time grows with the square of the count: 5,000 components in one array take about ten seconds to check, +10,000 about forty, and a document crafted that way could stall a CI job. Every file of the wheel, of each bundled wheel +and of each build tool counts, so a project with a large build environment can come close. Raise the cap for an SBOM +you expect to be larger: + +.. code-block:: console + + $ python-sbom validate dist/ --max-components 10000 diff --git a/pyproject.toml b/pyproject.toml index c9ff9c7..90c41cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,12 @@ [build-system] build-backend = "hatchling.build" requires = [ + "dep-logic>=0.7.2", "hatch-fancy-pypi-readme>=25.1", "hatch-vcs>=0.5", "hatchling>=1.32.4", + "installer>=1.1", + "packageurl-python>=0.11.1", ] [project] @@ -46,6 +49,9 @@ dynamic = [ "version", ] dependencies = [ + "dep-logic>=0.7.2", + "installer>=1.1", + "packageurl-python>=0.11.1", "packaging>=26.3", ] optional-dependencies.hatch = [ @@ -53,6 +59,7 @@ optional-dependencies.hatch = [ ] optional-dependencies.validate = [ "cyclonedx-python-lib[json-validation]>=11.12", + "license-expression>=30", "spdx-tools>=0.8.5", ] urls.Changelog = "https://python-sbom.readthedocs.io/en/latest/changelog.html" @@ -74,7 +81,8 @@ test = [ "covdefaults>=2.3", "cyclonedx-python-lib[json-validation]>=11.12", "hatchling>=1.28", - "installer>=0.7", + "dep-logic>=0.7.2", + "installer>=1.1", "pytest>=9.1.1", "pytest-cov>=7.1", "pytest-mock>=3.16", diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py index 16c15a1..2b4805d 100644 --- a/src/python_sbom/_cli/_commands.py +++ b/src/python_sbom/_cli/_commands.py @@ -15,7 +15,7 @@ from python_sbom._generate._archive import describe_archive from python_sbom._generate._embed import embed_sbom, revision_reader from python_sbom._validate._artifact import validate as validate_artifact -from python_sbom._validate._document import is_document +from python_sbom._validate._document import MAX_COMPONENTS, is_document if TYPE_CHECKING: from argparse import ArgumentParser, _SubParsersAction @@ -158,6 +158,14 @@ def add_validate(commands: _SubParsersAction[ArgumentParser]) -> None: metavar="SPDX-ID", help="SPDX license id shipped components may use; repeat it per id, leave it out to skip the license policy", ) + parser.add_argument( + "--max-components", + type=int, + default=MAX_COMPONENTS, + metavar="N", + help="reject an SBOM listing more than N components, nested ones included, without checking it further, since" + f" the schema check slows with the square of the count (default: {MAX_COMPONENTS})", + ) def _validate(options: Options) -> bool: @@ -171,7 +179,7 @@ def _validate(options: Options) -> bool: return False valid = True for artifact in artifacts: - for problem in (problems := validate_artifact(artifact, allowed)): + for problem in (problems := validate_artifact(artifact, allowed, options.max_components)): sys.stderr.write(f"SBOM invalid: {artifact.name}: {problem}\n") if problems: valid = False diff --git a/src/python_sbom/_cli/_options.py b/src/python_sbom/_cli/_options.py index 5d21063..81d0afc 100644 --- a/src/python_sbom/_cli/_options.py +++ b/src/python_sbom/_cli/_options.py @@ -24,6 +24,7 @@ class Options(Namespace): target: Path paths: list[Path] allowed_licenses: list[str] | None + max_components: int __all__ = [ diff --git a/src/python_sbom/_collect/_content.py b/src/python_sbom/_collect/_content.py index 57544f5..a34fac2 100644 --- a/src/python_sbom/_collect/_content.py +++ b/src/python_sbom/_collect/_content.py @@ -1,7 +1,7 @@ from __future__ import annotations -import importlib import inspect +import pkgutil import sys from pathlib import Path, PurePosixPath @@ -12,19 +12,20 @@ def load_content_provider(spec: str | None, root: Path) -> ContentProvider: if spec is None: return ContentProvider() + # resolve_name also takes a bare module, which is no provider module_name, _, attribute = spec.partition(":") if not (module_name and attribute): msg = f"content provider {spec!r} must be `module:attribute`" raise SettingsError(msg) sys.path.insert(0, str(root)) try: - module = importlib.import_module(module_name) - except ImportError as error: - msg = f"content provider {spec!r} does not import from {root}: {error}" + provider = pkgutil.resolve_name(spec) + # ValueError: a name resolve_name cannot parse, such as a second colon + except (ImportError, AttributeError, ValueError) as error: + msg = f"content provider {spec!r} does not load from {root}: {error}" raise SettingsError(msg) from error finally: sys.path.remove(str(root)) - provider = getattr(module, attribute, None) if isinstance(provider, type) and issubclass(provider, ContentProvider): provider = provider() if not isinstance(provider, ContentProvider): diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index 0770589..0153b8a 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -13,6 +13,7 @@ from importlib.metadata import distributions from typing import TYPE_CHECKING, Final, Protocol +from packaging.direct_url import DirectUrl, DirectUrlValidationError from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name @@ -83,9 +84,7 @@ def _installed(path: Sequence[str] | None) -> list[InstalledDistribution]: def _recorded_files(distribution: Distribution) -> list[RecordRow]: # a platform wheel's files are built for this machine's OS and architecture, and an editable install's point at a # checkout, so listing either would tie the document to one machine - if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] != "true" or ( - json.loads(distribution.read_text("direct_url.json") or "{}").get("dir_info", {}).get("editable") - ): + if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] != "true" or _editable(distribution): return [] rows: Final[list[RecordRow]] = [] for file in distribution.files or []: @@ -96,6 +95,18 @@ def _recorded_files(distribution: Distribution) -> list[RecordRow]: return rows +def _editable(distribution: Distribution) -> bool: + if (text := distribution.read_text("direct_url.json")) is None: + return False + try: + dir_info = DirectUrl.from_dict(json.loads(text)).dir_info + # an installer that wrote the file wrong breaks no build: its files get listed, as for any other install + except (ValueError, DirectUrlValidationError) as error: + warnings.warn(f"read the direct_url.json of {distribution.name} as not editable: {error}", stacklevel=2) + return False + return dir_info is not None and dir_info.editable is True + + def _describes_distribution(path: str, parent_suffix: str, name: str) -> bool: # console-script launchers live outside site-packages and embed the environment's interpreter path; the installer # metadata names the frontend; uv's cache files hold paths and times; none of them says anything about the package diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index 3de8a9d..c7a0b81 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -2,27 +2,36 @@ import hashlib from collections import Counter -from email.parser import Parser from io import BytesIO +from itertools import starmap +from pathlib import PurePath from typing import TYPE_CHECKING, Final +from installer.records import InvalidRecordEntry, RecordEntry, parse_record_file +from installer.sources import WheelFile +from installer.utils import parse_metadata_file from packaging.requirements import InvalidRequirement, Requirement -from python_sbom._errors import WheelError -from python_sbom._model._inputs import BuildScript, BundledWheel, SbomRequest, SourceTree -from python_sbom._wheel._archive import member_digest, open_archive, read_member, read_wheel_metadata -from python_sbom._wheel._record import HASHLIB_NAMES, UNHASHED, parse_record +from python_sbom._errors import SettingsError, WheelError +from python_sbom._model._inputs import BuildScript, BundledWheel, RecordRow, SbomRequest, SourceTree +from python_sbom._wheel._archive import open_archive, read_member, read_wheel_metadata +from python_sbom._wheel._record import decode_digest if TYPE_CHECKING: import zipfile from email.message import Message from pathlib import Path, PurePosixPath - from python_sbom._model._inputs import RecordRow, Settings + from python_sbom._model._inputs import Settings def read_source_tree(request: SbomRequest) -> SourceTree: root = request.root + for pattern in request.settings.bundled_wheels: + # a wheel outside the project is none the project ships, and pathlib refuses an absolute pattern outright + if (path := PurePath(pattern)).anchor or ".." in path.parts: + msg = f"`bundled-wheels` pattern {pattern!r} must be relative to the project, without `..`" + raise SettingsError(msg) return SourceTree( copyright=next( ( @@ -53,10 +62,13 @@ def read_source_tree(request: SbomRequest) -> SourceTree: def read_bundled_wheel(data: bytes, path: str) -> BundledWheel: where: Final[str] = f"bundled wheel {path}" with open_archive(BytesIO(data), where) as archive: + # installer takes the project name from the archive's file name, which an archive read from memory lacks + archive.filename = path members: Final[list[str]] = sorted(name for name in archive.namelist() if not name.endswith("/")) - dist_info, metadata = read_wheel_metadata(archive, where) + wheel, metadata = read_wheel_metadata(archive, where) + dist_info: Final[str] = wheel.dist_info_dir _check_named(metadata, where, f"{dist_info}/METADATA") - files: Final[list[RecordRow]] = _verified_record(archive, where, dist_info) + files: Final[list[RecordRow]] = _verified_record(archive, wheel, where) vendored: Final[list[tuple[str, Message]]] = [ *_vendor_manifest(archive, where, members), *( @@ -81,35 +93,48 @@ def _check_named(metadata: Message, where: str, name: str) -> None: raise WheelError(msg) -def _verified_record(archive: zipfile.ZipFile, where: str, dist_info: str) -> list[RecordRow]: +def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> list[RecordRow]: # the SBOM copies these rows, so a RECORD that misstates the wheel would make the SBOM misstate it too - rows: Final[list[RecordRow]] = parse_record( - read_member(archive, f"{dist_info}/RECORD", where).decode("utf-8"), where - ) - if repeated := sorted(path for path, count in Counter(row.path for row in rows).items() if count > 1): + dist_info: Final[str] = wheel.dist_info_dir + # installer decompresses RECORD whole, so a capped read comes first and bounds installer's read of the same bytes + record: Final[str] = read_member(archive, f"{dist_info}/RECORD", where).decode("utf-8") + try: + rows: Final[list[tuple[str, str, str]]] = list(parse_record_file(record.splitlines())) + except InvalidRecordEntry as error: + msg = f"{where}: RECORD does not parse: {error}" + raise WheelError(msg) from error + # installer checks the last row of a path only, skips rows naming no member, and lets a RECORD.jws or RECORD.p7s + # anywhere under the .dist-info, "../" included, go unlisted, so each of these checks is ours + if repeated := sorted(path for path, count in Counter(path for path, _, _ in rows).items() if count > 1): msg = f"{where}: RECORD lists {', '.join(repeated)} more than once" raise WheelError(msg) - members: Final[dict[str, zipfile.ZipInfo]] = { - info.filename: info for info in archive.infolist() if not info.is_dir() - } - recorded: Final[set[str]] = {row.path for row in rows} - if missing := sorted(members.keys() - recorded - {f"{dist_info}/{name}" for name in UNHASHED}): + members: Final[set[str]] = {info.filename for info in archive.infolist() if not info.is_dir()} + recorded: Final[set[str]] = {path for path, _, _ in rows} + # installer checks the row of RECORD itself, the one row without a digest + if missing := sorted( + members - recorded - {f"{dist_info}/{name}" for name in ("RECORD", "RECORD.jws", "RECORD.p7s")} + ): msg = f"{where}: RECORD lacks a digest for {', '.join(missing)}" raise WheelError(msg) - if extra := sorted(recorded - members.keys()): + if extra := sorted(recorded - members): msg = f"{where}: RECORD lists {', '.join(extra)}, which the wheel does not hold" raise WheelError(msg) - if resized := [row.path for row in rows if row.size != str(members[row.path].file_size)]: - msg = f"{where}: the size in RECORD differs from the size of {', '.join(resized)}" - raise WheelError(msg) - if changed := [ - row.path - for row in rows - if member_digest(archive, members[row.path], HASHLIB_NAMES[row.algorithm]) != row.digest - ]: - msg = f"{where}: the digest in RECORD differs from the content of {', '.join(changed)}" - raise WheelError(msg) - return rows + try: + wheel.validate_record() + except WheelFile.validation_error as error: + # installer documents the exception class only, not the list of issues it carries + issues: Final[list[str]] = getattr(error, "issues", [str(error)]) + prefix: Final[str] = f"In {archive.filename}, " + msg = f"{where}: RECORD disagrees with the wheel: {'; '.join(issue.removeprefix(prefix) for issue in issues)}" + raise WheelError(msg) from error + found: Final[list[RecordRow]] = [] + for entry in starmap(RecordEntry.from_elements, rows): + # the row of RECORD itself + if entry.hash_ is None: + continue + algorithm, digest = decode_digest(str(entry.hash_), f"{where}: RECORD row for {entry.path}") + found.append(RecordRow(path=entry.path, algorithm=algorithm, digest=digest, size=str(entry.size))) + return found def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) -> list[tuple[str, Message]]: @@ -127,12 +152,12 @@ def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) - if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: msg = f"{where}: vendored requirement in {name} pins no one exact version: {text}" raise WheelError(msg) - found.append((name, Parser().parsestr(f"Name: {requirement.name}\nVersion: {pins[0].version}\n"))) + found.append((name, parse_metadata_file(f"Name: {requirement.name}\nVersion: {pins[0].version}\n"))) return found def _vendored_metadata(archive: zipfile.ZipFile, where: str, name: str) -> Message: - metadata: Final[Message] = Parser().parsestr(read_member(archive, name, where).decode("utf-8")) + metadata: Final[Message] = parse_metadata_file(read_member(archive, name, where).decode("utf-8")) _check_named(metadata, where, name) return metadata diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index 7a0762a..0d75cbf 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -6,7 +6,8 @@ from email.parser import Parser from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Final -from urllib.parse import quote + +from packageurl import PackageURL from python_sbom._collect._content import load_content_provider, provider_script from python_sbom._collect._settings import settings_from_pyproject @@ -142,7 +143,7 @@ def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, context=ProviderContext(root=root, settings=settings, identity=project, artifact=unpacked), ) version = metadata[""]["Version"] - ref = f"pkg:generic/{quote(archive.name, safe='')}@{quote(version, safe='')}" + ref = PackageURL(type="generic", name=archive.name, version=version).to_string() return _Held( reader=reader, project=reader.project, diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index 8936b61..aa973cf 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -9,7 +9,7 @@ from ._inputs import RecordRow from ._licenses import licenses -from ._names import project_urls, purl, pypi_url, reference_type, tool_ref +from ._names import project_urls, property_prefix, purl, pypi_url, reference_type, tool_ref from ._types import Dependency if TYPE_CHECKING: @@ -51,8 +51,9 @@ def bundled_wheel_component( child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['bom-ref']}" child["properties"] = [*child["properties"], {"name": f"{namespace}:vendored-manifest", "value": source}] child["evidence"] = {"identity": [_evidence("purl", "manifest-analysis", source)]} - # a vendored .dist-info comes after pip's manifest line naming the same package, so its metadata wins - vendored[purl(child["name"], child["version"])] = child + # a vendored .dist-info comes after pip's manifest line naming the same package, so its metadata wins; keyed on + # the PEP 503 name, since the manifest and the .dist-info may spell it differently + vendored[f"{property_prefix(child['name'])}=={child['version']}"] = child component["components"] = [ *(file_component(component["bom-ref"], row) for row in wheel.files), *(vendored[key] for key in sorted(vendored)), diff --git a/src/python_sbom/_model/_markers.py b/src/python_sbom/_model/_markers.py index 0c8f8fc..4c04113 100644 --- a/src/python_sbom/_model/_markers.py +++ b/src/python_sbom/_model/_markers.py @@ -1,42 +1,23 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING -from packaging._parser import Op, Value, Variable # ruff: ignore[import-private-name] # packaging exposes the parsed marker tree only privately -from packaging.markers import Marker +from dep_logic.markers import from_pkg_marker if TYPE_CHECKING: - from packaging._parser import MarkerAtom, MarkerList # see above from packaging.requirements import Requirement def applies_without_extras(requirement: Requirement) -> bool: - # the conditions on extra are decided with no extra requested and every other condition stays unknown, so the - # requirement applies unless its extra conditions alone rule it out + # dep-logic reduces the marker to its conditions on extra, holding where some environment satisfies the rest, so + # the requirement applies unless they fail with no extra requested; a marker no environment satisfies never applies if requirement.marker is None: return True - return _extra_verdict(requirement.marker._markers) is not False # ruff: ignore[private-member-access] # the tree, see the import above - - -def _extra_verdict(markers: MarkerList | MarkerAtom) -> bool | None: - # three-valued, with "and" binding tighter than "or", as packaging's own evaluation groups them - groups: list[list[bool | None]] = [[]] - for item in markers: - if item == "or": - groups.append([]) - elif isinstance(item, list): - groups[-1].append(_extra_verdict(item)) - elif isinstance(item, tuple): - match item: - case Variable(value="extra"), Op(value=operator), Value(value=value): - groups[-1].append(Marker(f"extra {operator} {value!r}").evaluate({"extra": ""})) - case Value(value=value), Op(value=operator), Variable(value="extra"): - groups[-1].append(Marker(f"{value!r} {operator} extra").evaluate({"extra": ""})) - case _: - groups[-1].append(None) - # ranked False < unknown < True: "and" takes the lowest verdict of a group, "or" the highest of the groups - verdicts: Final[tuple[bool | None, ...]] = (False, None, True) - return verdicts[max(min(map(verdicts.index, group), default=2) for group in groups)] + try: + return from_pkg_marker(requirement.marker).only("extra").evaluate({"extra": ""}) + # dep-logic asserts extra is compared through == or != only; listing such a requirement beats leaving it out + except AssertionError: + return True __all__ = [ diff --git a/src/python_sbom/_model/_names.py b/src/python_sbom/_model/_names.py index f740b0c..53effac 100644 --- a/src/python_sbom/_model/_names.py +++ b/src/python_sbom/_model/_names.py @@ -3,6 +3,7 @@ import re from typing import TYPE_CHECKING, Final +from packageurl import PackageURL from packaging.utils import canonicalize_name if TYPE_CHECKING: @@ -56,7 +57,7 @@ def property_prefix(name: str) -> str: def purl(name: str, version: str | None = None) -> str: - return f"pkg:pypi/{property_prefix(name)}@{version}" if version else f"pkg:pypi/{property_prefix(name)}" + return PackageURL(type="pypi", name=name, version=version).to_string() def pypi_url(name: str, version: str | None = None) -> str: diff --git a/src/python_sbom/_model/_tools.py b/src/python_sbom/_model/_tools.py index b10759a..12abb22 100644 --- a/src/python_sbom/_model/_tools.py +++ b/src/python_sbom/_model/_tools.py @@ -2,6 +2,8 @@ from typing import TYPE_CHECKING, Final +from packageurl import PackageURL + from ._components import component_from_metadata, file_component from ._names import purl, tool_ref @@ -12,7 +14,7 @@ def tool_components(facts: BuildFacts) -> tuple[list[Component], list[Dependency]]: interpreter = facts.interpreter - package = f"pkg:generic/{interpreter.name}@{interpreter.version}" + package = PackageURL(type="generic", name=interpreter.name, version=interpreter.version).to_string() tools: Final[list[Component]] = [ { "type": "platform", diff --git a/src/python_sbom/_validate/_artifact.py b/src/python_sbom/_validate/_artifact.py index cf77593..556e753 100644 --- a/src/python_sbom/_validate/_artifact.py +++ b/src/python_sbom/_validate/_artifact.py @@ -5,7 +5,7 @@ from python_sbom._wheel._archive import open_archive, read_member -from ._document import validate_document +from ._document import MAX_COMPONENTS, validate_document if TYPE_CHECKING: from pathlib import Path @@ -15,9 +15,9 @@ _SBOM_LIMIT: Final[int] = 64 * 1024 * 1024 -def validate(path: Path, allowed_licenses: frozenset[str] | None) -> list[str]: +def validate(path: Path, allowed_licenses: frozenset[str] | None, max_components: int = MAX_COMPONENTS) -> list[str]: if not zipfile.is_zipfile(path): - return validate_document(path.read_text(encoding="utf-8"), allowed_licenses) + return validate_document(path.read_text(encoding="utf-8"), allowed_licenses, max_components) with open_archive(path, path.name) as archive: found: Final[dict[str, str]] = { name: read_member(archive, name, path.name, _SBOM_LIMIT).decode("utf-8") @@ -26,7 +26,11 @@ def validate(path: Path, allowed_licenses: frozenset[str] | None) -> list[str]: } if not found: return [f"no SBOM found in {path.name}"] - return [f"{name}: {problem}" for name, raw in found.items() for problem in validate_document(raw, allowed_licenses)] + return [ + f"{name}: {problem}" + for name, raw in found.items() + for problem in validate_document(raw, allowed_licenses, max_components) + ] def _is_sbom(name: str) -> bool: diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index d9d926c..290737d 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -10,8 +10,6 @@ from python_sbom._model._names import SBOM_FILE from python_sbom.version import __version__ -from ._licenses import license_problems - if TYPE_CHECKING: from typing import TypeIs # new in 3.13; annotations here are never evaluated @@ -31,18 +29,21 @@ _SERIAL: Final[re.Pattern[str]] = re.compile(r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") # the archive reader stops at the same size _MAX_CHARACTERS: Final[int] = 16 * 1024 * 1024 -# the schema's uniqueItems compares every pair of items, so an array this long already takes half a minute -_MAX_ITEMS: Final[int] = 10_000 +# the schema's uniqueItems compares every pair of items in an array, so a document whose components share one array +# takes about ten seconds to check at this count and four times that at twice it +MAX_COMPONENTS: Final[int] = 5_000 # an exception's message can quote the hostile value that caused it _MAX_TEXT: Final[int] = 200 -def validate_document(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: +def validate_document( + raw: str, allowed_licenses: frozenset[str] | None, max_components: int = MAX_COMPONENTS +) -> list[str]: if len(raw) > _MAX_CHARACTERS: return [f"is {len(raw)} characters long, over the {_MAX_CHARACTERS} character limit"] # one document's problem never stops the others from being checked, so whatever the checks raise is reported try: - return _problems(raw, allowed_licenses) + return _problems(raw, allowed_licenses, max_components) except MissingExtraError: raise except json.JSONDecodeError as error: @@ -54,13 +55,14 @@ def validate_document(raw: str, allowed_licenses: frozenset[str] | None) -> list return [f"could not be checked, {type(error).__name__}: {str(error)[:_MAX_TEXT]}"] -def _problems(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: +def _problems(raw: str, allowed_licenses: frozenset[str] | None, max_components: int) -> list[str]: # every document must conform to the CycloneDX 1.6 schema; one python-sbom wrote must also carry the urn:uuid serial # actions/attest needs, resolvable and unique bom-refs and a valid SPDX rendering, and meet the license policy + from ._licenses import license_problems # ruff: ignore[import-outside-top-level] # parses with the optional extra from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # the optional extra loads on first use loaded: Final = json.loads(raw) - if census := _census_problems(loaded): + if census := _census_problems(loaded, max_components): return census if error := schema_error(raw): return [f"does not conform to the CycloneDX 1.6 schema: {error}"] @@ -85,10 +87,11 @@ def _problems(raw: str, allowed_licenses: frozenset[str] | None) -> list[str]: return problems -def _census_problems(document: _Json) -> list[str]: - # counts the longest components array and every bom-ref before the schema vouches for any shape; iterative, so - # deep nesting cannot exhaust the stack - largest = 0 +def _census_problems(document: _Json, max_components: int) -> list[str]: + # counts the components and bom-refs in one linear pass before the schema vouches for any shape; iterative, so deep + # nesting cannot exhaust the stack. Nested components count too: the schema checks every components array, and + # one array may hold them all, so only the total bounds its work + components = 0 refs: Final[Counter[str]] = Counter() pending: Final[list[_Json]] = [document] while pending: @@ -98,11 +101,15 @@ def _census_problems(document: _Json) -> list[str]: elif isinstance(node, dict): if isinstance(ref := node.get("bom-ref"), str): refs[ref] += 1 - if isinstance(components := node.get("components"), list): - largest = max(largest, len(components)) + if isinstance(listed := node.get("components"), list): + components += len(listed) pending.extend(node.values()) - if largest > _MAX_ITEMS: - return [f"lists {largest} components in one array, over the {_MAX_ITEMS} the schema check can take"] + if components > max_components: + problem = ( + f"lists more than {max_components} components, nested ones included;" + " pass a higher --max-components to check it" + ) + return [problem] if duplicates := sorted(ref for ref, count in refs.items() if count > 1): return [f"bom-ref values are not unique: {duplicates}"] return [] @@ -163,6 +170,7 @@ def _walk(components: list[Component]) -> list[Component]: __all__ = [ + "MAX_COMPONENTS", "is_document", "validate_document", ] diff --git a/src/python_sbom/_validate/_licenses.py b/src/python_sbom/_validate/_licenses.py index 3c92ee9..a4b22f8 100644 --- a/src/python_sbom/_validate/_licenses.py +++ b/src/python_sbom/_validate/_licenses.py @@ -1,31 +1,44 @@ from __future__ import annotations -import re -from itertools import accumulate from typing import TYPE_CHECKING, Final +from license_expression import ( + AND, + ExpressionError, + LicenseSymbol, + LicenseWithExceptionSymbol, + Licensing, + get_license_index, +) + if TYPE_CHECKING: + from license_expression import LicenseExpression + from python_sbom._model._types import Component, License -# SPDX matches operators only in upper case -_OPERATORS: Final[frozenset[str]] = frozenset({"AND", "OR", "WITH", "(", ")"}) -_TOKEN: Final[re.Pattern[str]] = re.compile(r"[()]|[^\s()]+") -# the parser spends three stack frames per parenthesis, and real expressions nest a level or two -_MAX_NESTING: Final[int] = 32 +# the SPDX ids alone, without the aliases get_spdx_licensing() adds: those rename ids (LicenseRef-Apache becomes +# Apache-2.0, Net-SNMP becomes LicenseRef-scancode-net-snmp), so a policy would judge an id other than the one declared; +# knowing the ids still lets the parser tell "MIT ISC" apart from a single license +_LICENSING: Final[Licensing] = Licensing( + LicenseSymbol(entry["spdx_license_key"], is_exception=entry["is_exception"]) + for entry in get_license_index() + if entry.get("spdx_license_key") +) def license_problems(components: list[Component], allowed: frozenset[str]) -> list[str]: + # SPDX matches license ids without regard to case + folded: Final[frozenset[str]] = frozenset(identifier.casefold() for identifier in allowed) return [ f"{component.get('bom-ref', component['name'])}: {problem}" for component in _shipped(components) - if (problem := _judge(component, allowed)) + if (problem := _judge(component, allowed, folded)) ] def _shipped(components: list[Component]) -> list[Component]: - # only what the artifact carries bytes for is judged: a declared requirement resolves at install time, and a - # library a bundled wheel vendors ships under that wheel's terms; iterative, so deep nesting cannot exhaust the - # stack + # only what the artifact carries bytes for is judged: a declared requirement resolves at install time, and a library + # a bundled wheel vendors ships under that wheel's terms; iterative, so deep nesting cannot exhaust the stack found: Final[list[Component]] = [] pending: Final[list[Component]] = list(reversed(components)) while pending: @@ -38,7 +51,7 @@ def _shipped(components: list[Component]) -> list[Component]: return found -def _judge(component: Component, allowed: frozenset[str]) -> str | None: +def _judge(component: Component, allowed: frozenset[str], folded: frozenset[str]) -> str | None: licenses: Final[list[License]] = component.get("licenses", []) terms: Final[list[str]] = [ entry["expression"] if "expression" in entry else entry["license"]["id"] @@ -52,74 +65,51 @@ def _judge(component: Component, allowed: frozenset[str]) -> str | None: denied: Final[list[str]] = [] for term in terms: try: - if not _allowed(term, allowed): - denied.append(term) + tree = _parse(term) except ValueError as error: return f"license expression {term!r} is malformed: {error}" + if not _allowed(tree, folded): + denied.append(term) if denied: return f"license {' AND '.join(denied)} is outside the allowed licenses {sorted(allowed)}" return None -def _allowed(expression: str, allowed: frozenset[str]) -> bool: - tokens: Final[list[str]] = _tokens(expression) - verdict: Final[bool] = _license_any_of(tokens, allowed) - if tokens: - msg: Final[str] = f"unexpected {tokens[-1]!r}" - raise ValueError(msg) - return verdict - - -def _tokens(expression: str) -> list[str]: - found: Final[list[str]] = _TOKEN.findall(expression) - if (depth := max(accumulate({"(": 1, ")": -1}.get(token, 0) for token in found), default=0)) > _MAX_NESTING: - msg: Final[str] = f"it nests {depth} levels deep, over the {_MAX_NESTING} level limit" - raise ValueError(msg) - # reversed, so the parser takes the next token with a cheap pop from the end - return found[::-1] - - -def _license_any_of(tokens: list[str], allowed: frozenset[str]) -> bool: - verdict = _all_of(tokens, allowed) - while tokens and tokens[-1] == "OR": - tokens.pop() - verdict |= _all_of(tokens, allowed) - return verdict - - -def _all_of(tokens: list[str], allowed: frozenset[str]) -> bool: - verdict = _term(tokens, allowed) - while tokens and tokens[-1] == "AND": - tokens.pop() - verdict &= _term(tokens, allowed) - return verdict - - -def _term(tokens: list[str], allowed: frozenset[str]) -> bool: - if (token := _pop(tokens)) == "(": - verdict: Final[bool] = _license_any_of(tokens, allowed) - if (closing := _pop(tokens)) != ")": - msg = f"expected ')', got {closing!r}" - raise ValueError(msg) - return verdict - if token in _OPERATORS: - msg = f"unexpected {token!r}" +def _parse(term: str) -> LicenseExpression: + try: + tree: Final[LicenseExpression | None] = _LICENSING.parse(term) + except ExpressionError as error: + raise ValueError(str(error)) from error + except (AssertionError, IndexError) as error: + # boolean.py trips over some malformed input, such as "( OR MIT" or "()", before it reports it + msg = "it does not parse" + raise ValueError(msg) from error + if tree is None: + msg = "it is empty" raise ValueError(msg) - if tokens and tokens[-1] == "WITH": - tokens.pop() - # an exception grants extra permissions on top of the license it modifies, so that license alone decides - if (exception := _pop(tokens)) in _OPERATORS: - msg = f"unexpected {exception!r}" - raise ValueError(msg) - # SPDX matches license ids without regard to case - return token.casefold() in {identifier.casefold() for identifier in allowed} + return tree -def _pop(tokens: list[str]) -> str: - if not tokens: - msg: Final[str] = "it ends early" - raise ValueError(msg) - return tokens.pop() +def _allowed(tree: LicenseExpression, folded: frozenset[str]) -> bool: + # the parse tree nests as deep as the expression does, so it is walked with a stack instead of recursion; children + # come after their parent in this order, so the reversed walk judges them first + order: Final[list[LicenseExpression]] = [] + pending: Final[list[LicenseExpression]] = [tree] + while pending: + order.append(node := pending.pop()) + pending.extend(node.args) + verdicts: Final[dict[int, bool]] = {} + for node in reversed(order): + if isinstance(node, LicenseWithExceptionSymbol): + # an exception grants extra permissions on top of the license it modifies, so that license alone decides + verdicts[id(node)] = node.license_symbol.key.casefold() in folded + elif isinstance(node, LicenseSymbol): + verdicts[id(node)] = node.key.casefold() in folded + else: + # the parser builds only AND and OR around the symbols + branches = (verdicts[id(arg)] for arg in node.args) + verdicts[id(node)] = all(branches) if isinstance(node, AND) else any(branches) + return verdicts[id(tree)] __all__ = [ diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index 8a098d6..c1bf79b 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -1,5 +1,6 @@ from __future__ import annotations +import csv import hashlib import os import stat @@ -9,16 +10,18 @@ import zlib from collections import Counter from contextlib import contextmanager -from email.parser import Parser +from io import BytesIO, StringIO from pathlib import Path from typing import IO, TYPE_CHECKING, Final -from python_sbom._errors import WheelError +from installer.records import Hash, RecordEntry +from installer.sources import WheelFile +from installer.utils import copyfileobj_with_hashing, parse_metadata_file -from ._record import record_line +from python_sbom._errors import WheelError if TYPE_CHECKING: - from collections.abc import Generator, Iterable + from collections.abc import Generator from email.message import Message # metadata files beyond this size are refused, so a hostile wheel cannot exhaust memory @@ -28,15 +31,13 @@ def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: with open_archive(wheel, str(wheel)) as archive: + _, dist_info = _wheel_source(archive, str(wheel)) entries: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [ (info, archive.read(info)) for info in archive.infolist() ] - if (dist_info := _find_dist_info(info.filename for info, _ in entries)) is None: - msg = f"{wheel} has no .dist-info/METADATA, so it is not a wheel" - raise WheelError(msg) record: Final[str] = f"{dist_info}/RECORD" - if record not in {info.filename for info, _ in entries}: - msg = f"{wheel} has no {record}" + if missing := sorted({f"{dist_info}/METADATA", record} - {info.filename for info, _ in entries}): + msg = f"{wheel} has no {', '.join(missing)}" raise WheelError(msg) sbom: Final[str] = f"{dist_info}/sboms/{name}" # zip timestamps start at 1980 @@ -71,25 +72,30 @@ def open_archive(file: Path | IO[bytes], where: str) -> Generator[zipfile.ZipFil try: with zipfile.ZipFile(file) as archive: # installers extract the first of two same-named members and zipfile reads the last, and a file system that - # ignores case merges names that differ only by case, so such names would hide what gets installed + # ignores case merges names that differ only by case, so such names would hide what gets installed; + # installer's WheelFile accepts names that differ only by case if clashes := sorted( name for name, count in Counter(name.casefold() for name in archive.namelist()).items() if count > 1 ): msg = f"{where} holds more than one member named {', '.join(clashes)}, ignoring case" raise WheelError(msg) yield archive + # zipfile raises these as late as a member's last read, inside installer's checks too, which let them through except (zipfile.BadZipFile, zlib.error) as error: msg = f"{where} is not a valid zip archive: {error}" raise WheelError(msg) from error -def _find_dist_info(names: Iterable[str]) -> str | None: - found = [name.split("/")[0] for name in names if name.count("/") == 1 and name.endswith(".dist-info/METADATA")] - # a wheel holds one .dist-info; a second one is a decoy that could otherwise decide whose metadata counts - if len(found) > 1: - msg = f"the archive holds {len(found)} .dist-info directories, {', '.join(sorted(found))}, not one" - raise WheelError(msg) - return next(iter(found), None) +def _wheel_source(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, str]: + # installer refuses a wheel whose file name does not parse, or that holds other than one top-level .dist-info, + # since a second one is a decoy that could decide whose metadata counts + try: + wheel = WheelFile(archive) + dist_info = wheel.dist_info_dir + except ValueError as error: + msg = f"{where} is not a wheel: {error}" + raise WheelError(msg) from error + return wheel, dist_info def _rewritten_record(record: bytes, name: str, sbom: str, content: bytes) -> bytes: @@ -99,15 +105,18 @@ def _rewritten_record(record: bytes, name: str, sbom: str, content: bytes) -> by newline: Final[str] = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" # the row of RECORD itself stays last at: Final[int] = next((index for index, line in enumerate(lines) if line.startswith(f"{name},")), len(lines)) - lines.insert(at, f"{record_line(sbom, content)}{newline}") + # installer's helper hashes as it copies; the copy is thrown away + digest, size = copyfileobj_with_hashing(BytesIO(content), BytesIO(), "sha256") + row: Final[StringIO] = StringIO() + csv.writer(row, lineterminator=newline).writerow(RecordEntry(sbom, Hash("sha256", digest), size).to_row()) + lines.insert(at, row.getvalue()) return "".join(lines).encode() -def read_wheel_metadata(archive: zipfile.ZipFile, where: str) -> tuple[str, Message]: - if (dist_info := _find_dist_info(archive.namelist())) is None: - msg = f"{where} has no .dist-info/METADATA, so it is not a wheel" - raise WheelError(msg) - return dist_info, Parser().parsestr(read_member(archive, f"{dist_info}/METADATA", where).decode("utf-8")) +def read_wheel_metadata(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, Message]: + wheel, dist_info = _wheel_source(archive, where) + # read capped, as installer's read_dist_info decompresses the member whole + return wheel, parse_metadata_file(read_member(archive, f"{dist_info}/METADATA", where).decode("utf-8")) def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _METADATA_LIMIT) -> bytes: diff --git a/src/python_sbom/_wheel/_record.py b/src/python_sbom/_wheel/_record.py index e16adf3..a7af8da 100644 --- a/src/python_sbom/_wheel/_record.py +++ b/src/python_sbom/_wheel/_record.py @@ -2,14 +2,9 @@ import base64 import binascii -import csv -import hashlib -from io import StringIO -from pathlib import PurePosixPath from typing import Final from python_sbom._errors import WheelError -from python_sbom._model._inputs import RecordRow # the algorithms a wheel RECORD may name, as CycloneDX spells them _ALGORITHMS: Final[dict[str, str]] = { @@ -19,34 +14,11 @@ "sha384": "SHA-384", "sha512": "SHA-512", } -# the hashlib name of each algorithm, by its CycloneDX spelling -HASHLIB_NAMES: Final[dict[str, str]] = {cyclonedx: name for name, cyclonedx in _ALGORITHMS.items()} -# the files of a .dist-info the wheel specification leaves without a digest: RECORD and its signatures -UNHASHED: Final[frozenset[str]] = frozenset({"RECORD", "RECORD.jws", "RECORD.p7s"}) - - -def parse_record(text: str, where: str) -> list[RecordRow]: - # the rows without a digest, which only RECORD and its signatures may have, are left out - rows: Final[list[RecordRow]] = [] - for number, columns in enumerate(csv.reader(StringIO(text, newline="")), start=1): - match columns: - case []: - pass - case [path, "", _] if PurePosixPath(path).name in UNHASHED: - pass - case [path, "", _]: - msg = f"{where}: RECORD row {number} gives no digest for {path}" - raise WheelError(msg) - case [path, digest, size]: - algorithm, hexdigest = decode_digest(digest, f"{where}: RECORD row {number}") - rows.append(RecordRow(path=path, algorithm=algorithm, digest=hexdigest, size=size)) - case _: - msg = f"{where}: RECORD row {number} has {len(columns)} columns, not 3" - raise WheelError(msg) - return rows def decode_digest(digest: str, where: str) -> tuple[str, str]: + # installer checks RECORD digests as urlsafe base64 under any hashlib name, while CycloneDX takes hex under its own + # names for a few algorithms name, _, encoded = digest.partition("=") if (algorithm := _ALGORITHMS.get(name)) is None: msg = f"{where}: unknown hash algorithm {name!r}" @@ -59,15 +31,6 @@ def decode_digest(digest: str, where: str) -> tuple[str, str]: return algorithm, raw.hex() -def record_line(name: str, content: bytes) -> str: - encoded = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() - return f"{name},sha256={encoded},{len(content)}" - - __all__ = [ - "HASHLIB_NAMES", - "UNHASHED", "decode_digest", - "parse_record", - "record_line", ] diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index 735bf04..365b83a 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -60,9 +60,10 @@ def initialize( # a dirty checkout has no commit that reproduces the sdist, so a wheel built from it names none if self.target_name != "sdist" or (revision := default_revision(Path(self.root))) is None or revision.dirty: return - # outside the project, so the build leaves nothing behind in it + # outside the project, so the build leaves nothing behind in it; bytes, so the sdist holds the same LF line on + # every OS written = Path(tempfile.mkdtemp()) / REVISION_FILE - written.write_text(f"node: {revision.commit}\n", encoding="utf-8") + written.write_bytes(f"node: {revision.commit}\n".encode()) build_data["force_include"][str(written)] = REVISION_FILE def finalize( diff --git a/tests/_cli/test_commands.py b/tests/_cli/test_commands.py index 813969e..cc1a32a 100644 --- a/tests/_cli/test_commands.py +++ b/tests/_cli/test_commands.py @@ -123,6 +123,35 @@ def test_validate_reports_problems(wheel: Path, capsys: pytest.CaptureFixture[st assert capsys.readouterr().err == f"SBOM invalid: {wheel.name}: no SBOM found in {wheel.name}\n" +@pytest.fixture +def crowded_wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: + # 100 groups of 50 parts, 5,100 components in all, yet no array long enough to slow the schema check + groups = [ + { + "type": "library", + "name": f"g{group}", + "components": [{"type": "library", "name": f"p{part}"} for part in range(50)], + } + for group in range(100) + ] + sbom = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "components": groups}).encode() + return write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"", _SBOM: sbom}) + + +def test_validate_caps_components(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["validate", str(crowded_wheel)]) + assert capsys.readouterr().err == ( + f"SBOM invalid: {crowded_wheel.name}: {_SBOM}: lists more than 5000 components, nested ones included;" + " pass a higher --max-components to check it\n" + ) + + +def test_validate_max_components_raises_the_cap(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + main(["validate", str(crowded_wheel), "--max-components", "5100"]) + assert capsys.readouterr().out == f"SBOM in {crowded_wheel.name} is valid\n" + + def test_validate_without_wheels(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: with pytest.raises(SystemExit): main(["validate", str(tmp_path)]) diff --git a/tests/_collect/test_content.py b/tests/_collect/test_content.py index 3cb92ea..73e7da6 100644 --- a/tests/_collect/test_content.py +++ b/tests/_collect/test_content.py @@ -23,12 +23,23 @@ class Content(ContentProvider): instance = Content() other = object() + + +class Holder: + nested = instance """ (tmp_path / "tools" / f"{tmp_path.name}.py").write_text(provider, encoding="utf-8") return f"tools.{tmp_path.name}" -@pytest.mark.parametrize("attribute", [pytest.param("Content", id="class"), pytest.param("instance", id="instance")]) +@pytest.mark.parametrize( + "attribute", + [ + pytest.param("Content", id="class"), + pytest.param("instance", id="instance"), + pytest.param("Holder.nested", id="dotted-attribute"), + ], +) def test_load_from_project(tmp_path: Path, module: str, attribute: str) -> None: provider = load_content_provider(f"{module}:{attribute}", tmp_path) assert (type(provider).__name__, provider_script(provider, tmp_path)) == ( @@ -54,9 +65,10 @@ class Local(ContentProvider): [ pytest.param("tools", "must be `module:attribute`", id="no-attribute"), pytest.param(":Content", "must be `module:attribute`", id="no-module"), - pytest.param("not_a_module_anywhere:Content", "does not import from", id="import-fails"), + pytest.param("tools:Content:x", "does not load from .*: invalid format", id="two-colons"), + pytest.param("not_a_module_anywhere:Content", "does not load from .*: No module named", id="import-fails"), pytest.param("MODULE:other", "is not a ContentProvider subclass or instance", id="not-a-provider"), - pytest.param("MODULE:missing", "is not a ContentProvider subclass or instance", id="missing-attribute"), + pytest.param("MODULE:missing", "does not load from .*: .* has no attribute 'missing'", id="missing-attribute"), ], ) def test_load_refuses(tmp_path: Path, module: str, spec: str, message: str) -> None: diff --git a/tests/_collect/test_facts.py b/tests/_collect/test_facts.py index 4340c36..da345c3 100644 --- a/tests/_collect/test_facts.py +++ b/tests/_collect/test_facts.py @@ -127,6 +127,35 @@ def test_editable_distribution_lists_no_files(tmp_path: Path, install: Install) assert facts_from(tmp_path).distributions[0].files == [] +def test_directory_install_not_editable_lists_files(tmp_path: Path, install: Install) -> None: + install(tmp_path, "Name: local\nVersion: 1.0\n", {}, direct_url='{"url": "file:///checkout", "dir_info": {}}') + assert [row.path for row in facts_from(tmp_path).distributions[0].files] == [ + "local-1.0.dist-info/METADATA", + "local-1.0.dist-info/WHEEL", + ] + + +@pytest.mark.parametrize( + ("direct_url", "message"), + [ + pytest.param("{", "Expecting property name", id="not-json"), + pytest.param('{"dir_info": {"editable": true}}', "Missing required value in 'url'", id="no-url"), + pytest.param( + '{"url": "file:///checkout", "dir_info": {"editable": "yes"}}', + r"Unexpected type str \(expected bool\) in 'dir_info.editable'", + id="editable-not-bool", + ), + ], +) +def test_malformed_direct_url_reads_as_not_editable( + tmp_path: Path, install: Install, direct_url: str, message: str +) -> None: + install(tmp_path, "Name: local\nVersion: 1.0\n", {}, direct_url=direct_url) + with pytest.warns(UserWarning, match=f"read the direct_url.json of local as not editable: {message}"): + files = facts_from(tmp_path).distributions[0].files + assert len(files) == 2 + + def test_first_distribution_on_path_wins(tmp_path: Path, install: Install) -> None: install(first := tmp_path / "first", "Name: alpha\nVersion: 1.0\n", {}) install(second := tmp_path / "second", "Name: Alpha\nVersion: 2.0\n", {}) diff --git a/tests/_collect/test_source.py b/tests/_collect/test_source.py index 137460b..e7f3871 100644 --- a/tests/_collect/test_source.py +++ b/tests/_collect/test_source.py @@ -3,6 +3,7 @@ import base64 import dataclasses import hashlib +import re import zipfile from email import message_from_string from itertools import starmap @@ -12,7 +13,7 @@ import pytest from python_sbom._collect._source import read_bundled_wheel, read_source_tree, wheel_request -from python_sbom._errors import WheelError +from python_sbom._errors import SettingsError, WheelError from python_sbom._model._inputs import BuildScript, RecordRow, SbomRequest if TYPE_CHECKING: @@ -34,6 +35,7 @@ def row(path: str, content: bytes, algorithm: str = "sha256", *, encoded: str = _METADATA_ROW: Final[str] = row(_METADATA, _PIP[_METADATA]) +_RECORD_ROW: Final[str] = "pip-1.0.dist-info/RECORD,,\n" @pytest.mark.parametrize( @@ -73,6 +75,25 @@ def test_bundled_wheels_sorted_once(tmp_path: Path, settings: Settings, write_wh assert paths == ["src/a/pip-1.0-py3-none-any.whl", "src/b/pip-1.0-py3-none-any.whl"] +@pytest.mark.parametrize( + "pattern", + [ + pytest.param("../outside/*.whl", id="parent"), + pytest.param("src/../../outside/*.whl", id="parent-after-directory"), + pytest.param("ROOT/outside/*.whl", id="absolute"), + ], +) +def test_bundled_wheel_pattern_outside_project( + tmp_path: Path, settings: Settings, write_wheel: WriteWheel, pattern: str +) -> None: + write_wheel(tmp_path / "outside" / "pip-1.0-py3-none-any.whl", _PIP) + (project := tmp_path / "project" / "src").mkdir(parents=True) + pattern = pattern.replace("ROOT", tmp_path.as_posix()) + settings = dataclasses.replace(settings, bundled_wheels=(pattern,)) + with pytest.raises(SettingsError, match=f"`bundled-wheels` pattern {re.escape(repr(pattern))} must be relative"): + read_source_tree(request(project.parent, settings)) + + def test_build_scripts_hashed(tmp_path: Path, settings: Settings) -> None: (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") script = PurePosixPath("hatch_build.py") @@ -114,7 +135,17 @@ def test_bundled_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: @pytest.mark.parametrize( ("files", "message"), [ - pytest.param({"pip/__init__.py": b""}, "has no .dist-info/METADATA", id="no-metadata"), + pytest.param( + {"pip/__init__.py": b""}, + r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", + id="no-dist-info", + ), + pytest.param({"pip-1.0.dist-info/WHEEL": b""}, "has no pip-1.0.dist-info/METADATA", id="no-metadata"), + pytest.param( + {"aaa-0.dist-info/METADATA": b"Name: aaa\nVersion: 0\n", **_PIP}, + r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", + id="two-dist-info", + ), pytest.param({"pip-1.0.dist-info/METADATA": b"Name: pip\n"}, "METADATA lacks Name or Version", id="no-version"), pytest.param( {"pip-1.0.dist-info/METADATA": _PIP["pip-1.0.dist-info/METADATA"]}, @@ -126,6 +157,11 @@ def test_bundled_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: "METADATA is over the 16777216 byte limit", id="oversized-metadata", ), + pytest.param( + {**_PIP, "pip-1.0.dist-info/RECORD": b"\n" * (16 * 1024 * 1024 + 1)}, + "RECORD is over the 16777216 byte limit", + id="oversized-record", + ), ], ) def test_malformed_bundled_wheel( @@ -136,6 +172,23 @@ def test_malformed_bundled_wheel( read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") +@pytest.mark.parametrize( + ("name", "message"), + [ + pytest.param("pip.whl", "Not a valid wheel filename: pip.whl", id="unparsable"), + pytest.param( + "other-1.0-py3-none-any.whl", r"Wheel \.dist-info directory doesn't match wheel filename", id="other" + ), + ], +) +def test_bundled_wheel_file_name_names_the_project( + tmp_path: Path, write_wheel: WriteWheel, name: str, message: str +) -> None: + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", _PIP) + with pytest.raises(WheelError, match=f"bundled wheel {name} is not a wheel: {message}"): + read_bundled_wheel(wheel.read_bytes(), name) + + def test_corrupt_bundled_wheel() -> None: with pytest.raises(WheelError, match=r"bundled wheel pip\.whl is not a valid zip archive"): read_bundled_wheel(b"not a zip", "pip.whl") @@ -162,7 +215,7 @@ def test_bundled_wheel_with_duplicate_member(tmp_path: Path) -> None: ], ) def test_bundled_wheel_record_algorithm(tmp_path: Path, write_wheel: WriteWheel, algorithm: str, expected: str) -> None: - record = "".join(row(path, content, algorithm) for path, content in _PIP.items()) + record = "".join(row(path, content, algorithm) for path, content in _PIP.items()) + _RECORD_ROW files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) assert [ @@ -174,8 +227,29 @@ def test_bundled_wheel_record_algorithm(tmp_path: Path, write_wheel: WriteWheel, @pytest.mark.parametrize( ("record", "message"), [ - pytest.param(f"{_INIT},sha256=x\n", "RECORD row 1 has 2 columns, not 3", id="columns"), - pytest.param(f"{_INIT},,0\n", f"RECORD row 1 gives no digest for {_INIT}", id="no-digest"), + pytest.param( + f"{_INIT},sha256=x\n", "RECORD does not parse: Row Index 0: expected 3 elements, got 2", id="columns" + ), + pytest.param( + f"{_METADATA_ROW}\n{_RECORD_ROW}", + "RECORD does not parse: Row Index 1: expected 3 elements, got 0", + id="blank-line", + ), + pytest.param( + f"{_INIT},,0\n{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD disagrees with the wheel: hash / size of {_INIT} is not included in RECORD", + id="no-digest", + ), + pytest.param( + f"{row(_INIT, b'')}{_METADATA_ROW}", + "RECORD disagrees with the wheel: pip-1.0.dist-info/RECORD is not mentioned in RECORD", + id="record-not-recorded", + ), + pytest.param( + f"{row(_INIT, b'', 'sha224')}{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD row for {_INIT}: unknown hash algorithm 'sha224'", + id="algorithm-cyclonedx-lacks", + ), pytest.param(_METADATA_ROW, f"RECORD lacks a digest for {_INIT}", id="member-not-recorded"), pytest.param( f"{row(_INIT, b'', encoded='A' * 43)}{row(_INIT, b'')}{_METADATA_ROW}", @@ -193,13 +267,13 @@ def test_bundled_wheel_record_algorithm(tmp_path: Path, write_wheel: WriteWheel, id="recorded-not-member", ), pytest.param( - f"{row(_INIT, b'', size=999999)}{_METADATA_ROW}", - f"the size in RECORD differs from the size of {_INIT}", + f"{row(_INIT, b'', size=999999)}{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD disagrees with the wheel: hash / size of {_INIT} didn't match RECORD", id="size-mismatch", ), pytest.param( - f"{row(_INIT, b'', 'sha512', encoded='A' * 86)}{_METADATA_ROW}", - f"the digest in RECORD differs from the content of {_INIT}", + f"{row(_INIT, b'', 'sha512', encoded='A' * 86)}{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD disagrees with the wheel: hash / size of {_INIT} didn't match RECORD", id="digest-mismatch", ), ], @@ -212,7 +286,7 @@ def test_record_disagrees_with_wheel(tmp_path: Path, write_wheel: WriteWheel, re def test_record_signatures_need_no_row(tmp_path: Path, write_wheel: WriteWheel) -> None: - record = "".join(starmap(row, _PIP.items())) + record = "".join(starmap(row, _PIP.items())) + _RECORD_ROW files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), "pip-1.0.dist-info/RECORD.jws": b"{}"} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) assert [file.path for file in read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files] == list( @@ -220,6 +294,22 @@ def test_record_signatures_need_no_row(tmp_path: Path, write_wheel: WriteWheel) ) +@pytest.mark.parametrize( + "signature", + [ + pytest.param("pip-1.0.dist-info/sub/RECORD.jws", id="nested"), + pytest.param("pip-1.0.dist-info/../pip/RECORD.p7s", id="parent"), + ], +) +def test_signature_away_from_record_needs_a_row(tmp_path: Path, write_wheel: WriteWheel, signature: str) -> None: + # installer lets these go unlisted, though a signature only sits beside RECORD + record = "".join(starmap(row, _PIP.items())) + _RECORD_ROW + files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), signature: b"import os\n"} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + with pytest.raises(WheelError, match=f"RECORD lacks a digest for {re.escape(signature)}"): + read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + def test_vendored_metadata_needs_name_and_version(tmp_path: Path, write_wheel: WriteWheel) -> None: files = {**_PIP, "pip/_vendor/rich-13.0.dist-info/METADATA": b"Metadata-Version: 2.1\n"} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files) @@ -256,7 +346,20 @@ def test_wheel_request(tmp_path: Path, settings: Settings, write_wheel: WriteWhe ) -def test_wheel_request_needs_a_wheel(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: - archive = write_wheel(tmp_path / "native.whl", {"native/__init__.py": b""}, record=False) - with pytest.raises(WheelError, match=r"has no \.dist-info/METADATA, so it is not a wheel"): +@pytest.mark.parametrize( + ("name", "message"), + [ + pytest.param("native.whl", "is not a wheel: Not a valid wheel filename", id="file-name"), + pytest.param( + "native-1.0-py3-none-any.whl", + r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", + id="no-dist-info", + ), + ], +) +def test_wheel_request_needs_a_wheel( + tmp_path: Path, settings: Settings, write_wheel: WriteWheel, name: str, message: str +) -> None: + archive = write_wheel(tmp_path / name, {"native/__init__.py": b""}, record=False) + with pytest.raises(WheelError, match=message): wheel_request(archive, Path(), settings, ()) diff --git a/tests/_model/test_components.py b/tests/_model/test_components.py index bed0b85..cba1666 100644 --- a/tests/_model/test_components.py +++ b/tests/_model/test_components.py @@ -333,3 +333,27 @@ def test_script_tool_with_link() -> None: def test_script_tool_without_link() -> None: assert "externalReferences" not in script_tool(PurePosixPath("hook.py"), "ef" * 32, "1.0", None) + + +def test_vendored_spellings_of_one_package_merge() -> None: + wheel = BundledWheel( + path=_WHEEL, + sha256="ab" * 32, + metadata_name="pip-1.0.dist-info/METADATA", + metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + files=[], + vendored=[ + ("pip/_vendor/vendor.txt", message_from_string("Name: jaraco.text\nVersion: 4.0\n")), + ( + "pip/_vendor/jaraco_text-4.0.dist-info/METADATA", + message_from_string("Name: jaraco_text\nVersion: 4.0\n"), + ), + ], + ) + component = bundled_wheel_component(wheel, "example", [], None) + assert [child["name"] for child in component.get("components", [])] == ["jaraco_text"] + + +def test_pypi_purl_keeps_dots() -> None: + component = component_from_metadata(message_from_string("Name: zope.interface\nVersion: 7.0\n")) + assert component["purl"] == "pkg:pypi/zope.interface@7.0" diff --git a/tests/_model/test_markers.py b/tests/_model/test_markers.py index 92eeff3..2c7a687 100644 --- a/tests/_model/test_markers.py +++ b/tests/_model/test_markers.py @@ -21,6 +21,16 @@ ), pytest.param('click; python_version >= "3" or extra == "a"', True, id="environment-or-extra"), pytest.param("click", True, id="no-marker"), + pytest.param('click; extra == "A_b" and extra != "a-b"', False, id="extra-names-normalized"), + # no environment satisfies the marker, so no install ever pulls the requirement in + pytest.param('click; python_version < "3" and python_version >= "3"', False, id="environment-contradiction"), + pytest.param( + 'click; (os_name == "nt" and os_name != "nt") or extra == "a"', False, id="contradiction-or-extra" + ), + # dep-logic compares extra through == and != only, so other comparisons keep the requirement listed + pytest.param('click; extra in "a,b"', True, id="extra-in"), + pytest.param('click; extra not in "a,b"', True, id="extra-not-in"), + pytest.param('click; extra ~= "1.0" and os_name == "nt"', True, id="extra-compatible-release"), ], ) def test_applies_without_extras(requirement: str, applies: bool) -> None: diff --git a/tests/_validate/test_artifact.py b/tests/_validate/test_artifact.py index 1336d55..96f17fa 100644 --- a/tests/_validate/test_artifact.py +++ b/tests/_validate/test_artifact.py @@ -93,3 +93,11 @@ def test_corrupt_archive(tmp_path: Path) -> None: def test_json_file(tmp_path: Path, describe: Describe) -> None: (sbom := tmp_path / "example.cdx.json").write_bytes(dumps(describe())) assert validate(sbom, frozenset({"MIT"})) == [] + + +def test_json_file_component_cap(tmp_path: Path) -> None: + components = [{"type": "library", "name": name} for name in ("one", "two", "three")] + (sbom := tmp_path / "example.cdx.json").write_text(json.dumps({"components": components}), encoding="utf-8") + assert validate(sbom, None, max_components=2) == [ + "lists more than 2 components, nested ones included; pass a higher --max-components to check it" + ] diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py index d3f0d8f..64a40bc 100644 --- a/tests/_validate/test_document.py +++ b/tests/_validate/test_document.py @@ -56,13 +56,27 @@ def test_oversized_document_is_not_parsed() -> None: ] -def test_long_components_array_skips_the_schema() -> None: - # the schema compares every pair of array items, which takes minutes at this length - components = [{"type": "library", "bom-ref": f"ref-{index}", "name": "many"} for index in range(10_001)] - raw = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components}) - assert validate_document(raw, None) == [ - "lists 10001 components in one array, over the 10000 the schema check can take" +def test_component_cap_counts_nested_components() -> None: + assert validate_document(many_components(), None) == [ + "lists more than 5000 components, nested ones included; pass a higher --max-components to check it" + ] + + +def many_components() -> str: + # 100 groups of 50 parts, 5,100 components in all, yet no array long enough to slow the schema check + groups = [ + { + "type": "library", + "name": f"group-{group}", + "components": [{"type": "library", "name": f"part-{part}"} for part in range(50)], + } + for group in range(100) ] + return json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "components": groups}) + + +def test_component_cap_can_be_raised() -> None: + assert validate_document(many_components(), None, max_components=5_100) == [] def test_missing_fields(describe: Describe) -> None: diff --git a/tests/_validate/test_licenses.py b/tests/_validate/test_licenses.py index b29fc80..237dc6c 100644 --- a/tests/_validate/test_licenses.py +++ b/tests/_validate/test_licenses.py @@ -51,40 +51,82 @@ def shipped(licenses: list[License]) -> Component: id="names", ), pytest.param([], "declares no license", id="none"), - pytest.param([{"expression": "MIT AND"}], "license expression 'MIT AND' is malformed: it ends early", id="end"), - pytest.param([{"expression": "(MIT"}], "license expression '(MIT' is malformed: it ends early", id="open"), + pytest.param( + [{"expression": "MIT AND"}], + "license expression 'MIT AND' is malformed: AND requires two or more licenses as in: MIT AND BSD", + id="end", + ), + pytest.param([{"expression": "(MIT"}], "license expression '(MIT' is malformed: Invalid expression", id="open"), pytest.param( [{"expression": "(MIT ISC)"}], - "license expression '(MIT ISC)' is malformed: expected ')', got 'ISC'", + "license expression '(MIT ISC)' is malformed: Invalid symbols sequence such as (A B) for token: \"ISC\" at" + " position: 5", id="pair", ), - pytest.param([{"expression": "MIT)"}], "license expression 'MIT)' is malformed: unexpected ')'", id="close"), pytest.param( - [{"expression": "AND MIT"}], "license expression 'AND MIT' is malformed: unexpected 'AND'", id="op" + [{"expression": "MIT)"}], + "license expression 'MIT)' is malformed: Unbalanced parenthesis for token: \")\" at position: 3", + id="close", + ), + pytest.param( + [{"expression": "AND MIT"}], + "license expression 'AND MIT' is malformed: Invalid operator sequence without symbols such as AND OR or OR" + ' OR for token: "AND"', + id="op", + ), + pytest.param( + [{"expression": "MIT WITH OR"}], + "license expression 'MIT WITH OR' is malformed: Invalid expression for token: \"WITH\" at position: 4", + id="with", ), + pytest.param([{"expression": " "}], "license expression ' ' is malformed: it is empty", id="empty"), pytest.param( - [{"expression": "MIT WITH OR"}], "license expression 'MIT WITH OR' is malformed: unexpected 'OR'", id="with" + [{"expression": "( OR MIT"}], + "license expression '( OR MIT' is malformed: it does not parse", + id="assertion", ), + pytest.param([{"expression": "()"}], "license expression '()' is malformed: it does not parse", id="no-group"), ], ) def test_reports(licenses: list[License], problem: str) -> None: assert license_problems([shipped(licenses)], _ALLOWED) == [f"bundled:lib.whl: {problem}"] -def test_deep_nesting_is_malformed() -> None: - # each level costs the parser stack frames, so this deep a nesting would exhaust the stack - expression = f"{'(' * 5000}MIT{')' * 5000}" +@pytest.mark.parametrize( + "expression", + [ + pytest.param(f"{'(' * 5000}MIT{')' * 5000}", id="parentheses"), + pytest.param( + f"{''.join(f'MIT {operator} (' for operator in ('AND', 'OR') * 2500)}MIT{')' * 5000}", id="operators" + ), + ], +) +def test_deep_nesting_is_allowed(expression: str) -> None: + # a recursive walk would exhaust the stack this deep + assert license_problems([shipped([{"expression": expression}])], _ALLOWED) == [] + + +def test_deep_nesting_is_denied() -> None: + expression = f"{'MIT AND (' * 5000}GPL-3.0-only{')' * 5000}" assert license_problems([shipped([{"expression": expression}])], _ALLOWED) == [ - ( - f"bundled:lib.whl: license expression {expression!r} is malformed: it nests 5000 levels deep, over the 32" - " level limit" - ) + f"bundled:lib.whl: license {expression} is outside the allowed licenses ['BSD-3-Clause', 'GPL-2.0-only', 'MIT']" ] -def test_nesting_up_to_the_limit_parses() -> None: - expression = f"{'(' * 32}MIT{')' * 32}" - assert license_problems([shipped([{"expression": expression}])], _ALLOWED) == [] +@pytest.mark.parametrize( + ("expression", "allowed", "problems"), + [ + pytest.param( + "LicenseRef-Apache", + frozenset({"Apache-2.0"}), + ["bundled:lib.whl: license LicenseRef-Apache is outside the allowed licenses ['Apache-2.0']"], + id="license-ref-is-not-apache", + ), + pytest.param("Net-SNMP", frozenset({"Net-SNMP"}), [], id="spdx-id-is-kept"), + ], +) +def test_judges_the_declared_id_not_an_alias(expression: str, allowed: frozenset[str], problems: list[str]) -> None: + assert license_problems([shipped([{"expression": expression}])], allowed) == problems def test_judges_only_shipped_components() -> None: diff --git a/tests/_wheel/test_archive.py b/tests/_wheel/test_archive.py index 63425d2..15f3ef0 100644 --- a/tests/_wheel/test_archive.py +++ b/tests/_wheel/test_archive.py @@ -119,7 +119,16 @@ def test_embed_keeps_wheel_permissions(wheel: Path) -> None: # pragma: win32 no @pytest.mark.parametrize( ("files", "message"), [ - pytest.param({"native/__init__.py": b""}, "has no .dist-info/METADATA, so it is not a wheel", id="no-metadata"), + pytest.param( + {"native/__init__.py": b""}, + r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", + id="no-dist-info", + ), + pytest.param( + {"native-1.0.dist-info/RECORD": b""}, + "has no native-1.0.dist-info/METADATA", + id="no-metadata", + ), pytest.param( {"native-1.0.dist-info/METADATA": b"Name: native\n"}, "has no native-1.0.dist-info/RECORD", @@ -128,7 +137,7 @@ def test_embed_keeps_wheel_permissions(wheel: Path) -> None: # pragma: win32 no ], ) def test_embed_refuses_non_wheel(tmp_path: Path, files: dict[str, bytes], message: str) -> None: - with zipfile.ZipFile(archive := tmp_path / "native.whl", "w") as handle: + with zipfile.ZipFile(archive := tmp_path / "native-1.0-py3-none-any.whl", "w") as handle: for name, content in files.items(): handle.writestr(name, content) with pytest.raises(WheelError, match=message): @@ -230,5 +239,6 @@ def test_wheel_with_two_dist_info_directories(tmp_path: Path, write_wheel: Write }, record=False, ) - with pytest.raises(WheelError, match=r"holds 2 \.dist-info directories, aaa-0\.dist-info, native-1\.0\.dist-info"): + message = r"Wheel doesn't contain exactly one \.dist-info directory .*'aaa-0\.dist-info', 'native-1\.0\.dist-info'" + with pytest.raises(WheelError, match=message): write_sbom(wheel, "native.cdx.json", b"{}\n", 0) diff --git a/tests/_wheel/test_record.py b/tests/_wheel/test_record.py index 46166bf..3bb24a7 100644 --- a/tests/_wheel/test_record.py +++ b/tests/_wheel/test_record.py @@ -1,60 +1,21 @@ from __future__ import annotations import hashlib -from typing import Final import pytest from python_sbom._errors import WheelError -from python_sbom._model._inputs import RecordRow -from python_sbom._wheel._record import decode_digest, parse_record, record_line - -# the SHA-256 of b"", urlsafe base64 without padding -_EMPTY: Final[str] = "47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU" - - -def test_parse_record_row() -> None: - assert parse_record(f"pkg/__init__.py,sha256={_EMPTY},0\n", "w") == [ - RecordRow(path="pkg/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"").hexdigest(), size="0") - ] - - -def test_parse_record_unquotes_csv() -> None: - assert [row.path for row in parse_record(f'"pkg/a,b.py",sha256={_EMPTY},0\n', "w")] == ["pkg/a,b.py"] - - -@pytest.mark.parametrize( - "text", - [ - pytest.param("\n", id="blank"), - pytest.param("pkg-1.0.dist-info/RECORD,,\n", id="record"), - pytest.param("pkg-1.0.dist-info/RECORD.jws,,\n", id="jws-signature"), - pytest.param("pkg-1.0.dist-info/RECORD.p7s,,\n", id="p7s-signature"), - ], -) -def test_parse_record_skips_rows_without_digest(text: str) -> None: - assert parse_record(text, "w") == [] - - -@pytest.mark.parametrize( - ("text", "message"), - [ - pytest.param("pkg/__init__.py,,0\n", "w: RECORD row 1 gives no digest for pkg/__init__.py", id="no-digest"), - pytest.param("pkg/,,\n", "w: RECORD row 1 gives no digest for pkg/", id="directory"), - pytest.param(f"pkg/__init__.py,sha256={_EMPTY}\n", "w: RECORD row 1 has 2 columns, not 3", id="two-columns"), - pytest.param(f"a,sha256={_EMPTY},0,extra\n", "w: RECORD row 1 has 4 columns, not 3", id="four-columns"), - pytest.param("\npkg/__init__.py\n", "w: RECORD row 2 has 1 columns, not 3", id="row-number"), - ], -) -def test_parse_record_rejects_row(text: str, message: str) -> None: - with pytest.raises(WheelError, match=message): - parse_record(text, "w") +from python_sbom._wheel._record import decode_digest @pytest.mark.parametrize( ("digest", "expected"), [ - pytest.param(f"sha256={_EMPTY}", ("SHA-256", hashlib.sha256(b"").hexdigest()), id="sha256"), + pytest.param( + "sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU", + ("SHA-256", hashlib.sha256(b"").hexdigest()), + id="sha256", + ), pytest.param( "md5=1B2M2Y8AsgTpgAmY7PhCfg", ("MD5", hashlib.md5(b"", usedforsecurity=False).hexdigest()), id="md5" ), @@ -74,7 +35,3 @@ def test_decode_digest(digest: str, expected: tuple[str, str]) -> None: def test_decode_digest_rejects(digest: str, message: str) -> None: with pytest.raises(WheelError, match=message): decode_digest(digest, "w") - - -def test_record_line() -> None: - assert record_line("pkg/__init__.py", b"") == f"pkg/__init__.py,sha256={_EMPTY},0" From 764f69a2d0a7cafd8b54fc14244c263bb04cc4b4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:17:49 +0000 Subject: [PATCH 025/115] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- bench/__main__.py | 3 +- bench/_cases.py | 4 +- bench/_fixtures.py | 132 ++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 4 files changed, 70 insertions(+), 71 deletions(-) diff --git a/bench/__main__.py b/bench/__main__.py index b3d8f71..6344f94 100644 --- a/bench/__main__.py +++ b/bench/__main__.py @@ -246,8 +246,7 @@ def _stats_text(stats_file: Path) -> str: def _hot_spots(name: str, stats_file: Path) -> str: - stats = pstats.Stats(str(stats_file)) - entries = stats.stats # ty: ignore[unresolved-attribute] # the table pstats keeps is not in its stubs + entries = pstats.Stats(str(stats_file)).stats # ty: ignore[unresolved-attribute] # the table pstats keeps is not in its stubs total = sum(entry[2] for entry in entries.values()) or 1.0 rows = [ (_label(function), entry[1], entry[0], entry[2], entry[3]) diff --git a/bench/_cases.py b/bench/_cases.py index eda5998..cf0acf6 100644 --- a/bench/_cases.py +++ b/bench/_cases.py @@ -32,7 +32,7 @@ } -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class Step: run: Callable[[], object] reset: Callable[[], object] | None = None @@ -40,7 +40,7 @@ class Step: command: tuple[str, ...] | None = None -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class Case: name: str tier: str diff --git a/bench/_fixtures.py b/bench/_fixtures.py index 7b9f864..fe0f65e 100644 --- a/bench/_fixtures.py +++ b/bench/_fixtures.py @@ -70,13 +70,6 @@ def prepare(work: Path, project: Path, virtualenv: Path | None) -> None: sys.stdout.write(f"wrote {work / 'fixtures.json'}\n") -def load(work: Path) -> dict[str, str]: - if not (manifest := work / "fixtures.json").is_file(): - msg = f"{manifest} does not exist; run `python -m bench prepare` first" - raise SystemExit(msg) - return json.loads(manifest.read_text(encoding="utf-8")) - - def _small_project(root: Path) -> Path: # a pure-Python project of forty modules that turns the hook on, as a typical hatchling user does shutil.rmtree(root, ignore_errors=True) @@ -116,6 +109,50 @@ def _small_project(root: Path) -> Path: return root +def _commit(root: Path) -> None: + # a checkout, as projects build from one, so the timed runs include reading the source revision with git + _run(["git", "init", "--quiet", str(root)]) + _run(["git", "-C", str(root), "add", "."]) + _run([ + "git", + "-C", + str(root), + *_GIT_IDENTITY, + "commit", + "--quiet", + "--no-verify", + "--no-gpg-sign", + "-m", + "bench fixture", + ]) + + +def _run(command: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None) -> None: + subprocess.run(command, check=True, cwd=cwd, env=env) + + +def _hatchling_wheel(root: Path, directory: Path) -> Path: + # without isolation, in this environment, so the hook under test is the one installed here + shutil.rmtree(directory, ignore_errors=True) + _run([sys.executable, "-m", "hatchling", "build", "--target", "wheel", "--directory", str(directory)], cwd=root) + (wheel,) = directory.glob("*.whl") + return wheel + + +def _hostile_document(wheel: Path, target: Path) -> Path: + # ten thousand unique components: over the default component cap, which rejects the document before the schema + # check, and with a raised limit a uniqueItems check that compares every pair of them + with zipfile.ZipFile(wheel) as archive: + (name,) = (member for member in archive.namelist() if "/sboms/" in member) + document = json.loads(archive.read(name)) + hostile = copy.deepcopy(document) + hostile["components"] = [ + {"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(_HOSTILE_COMPONENTS) + ] + target.write_text(json.dumps(hostile), encoding="utf-8") + return target + + def _meson_project(root: Path) -> Path: # the project directory `python-sbom embed` reads [tool.python-sbom] and the source revision from shutil.rmtree(root, ignore_errors=True) @@ -200,25 +237,25 @@ def _fat_environment(target: Path) -> str: return _output([str(python), "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"]).strip() -def _hatchling_wheel(root: Path, directory: Path) -> Path: - # without isolation, in this environment, so the hook under test is the one installed here - shutil.rmtree(directory, ignore_errors=True) - _run([sys.executable, "-m", "hatchling", "build", "--target", "wheel", "--directory", str(directory)], cwd=root) - (wheel,) = directory.glob("*.whl") - return wheel +def _output(command: list[str]) -> str: + return subprocess.run(command, check=True, capture_output=True, text=True).stdout -def _hostile_document(wheel: Path, target: Path) -> Path: - # ten thousand unique components: over the default component cap, which rejects the document before the schema - # check, and with a raised limit a uniqueItems check that compares every pair of them - with zipfile.ZipFile(wheel) as archive: - (name,) = (member for member in archive.namelist() if "/sboms/" in member) - document = json.loads(archive.read(name)) - hostile = copy.deepcopy(document) - hostile["components"] = [ - {"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(_HOSTILE_COMPONENTS) - ] - target.write_text(json.dumps(hostile), encoding="utf-8") +def _snapshot(source: Path, target: Path) -> Path: + # a clone of the checkout plus its uncommitted changes, so builds write their version files there and never into + # the checkout itself; --shared borrows its objects, so even a detached HEAD resolves + shutil.rmtree(target, ignore_errors=True) + _run(["git", "clone", "--quiet", "--shared", "--no-checkout", str(source), str(target)]) + head = _output(["git", "-C", str(source), "rev-parse", "HEAD"]).strip() + _run(["git", "-C", str(target), "checkout", "--quiet", "--detach", head]) + if diff := subprocess.run( + ["git", "-C", str(source), "diff", "--binary", "HEAD"], check=True, capture_output=True + ).stdout: + subprocess.run(["git", "-C", str(target), "apply", "--index"], check=True, input=diff) + for name in _output(["git", "-C", str(source), "ls-files", "--others", "--exclude-standard", "-z"]).split("\0"): + if name: + (target / name).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source / name, target / name) return target @@ -265,48 +302,11 @@ def _wheel_version(wheel: Path) -> tuple[int, ...]: return tuple(int(part) for part in wheel.name.split("-")[1].split(".") if part.isdigit()) -def _snapshot(source: Path, target: Path) -> Path: - # a clone of the checkout plus its uncommitted changes, so builds write their version files there and never into - # the checkout itself; --shared borrows its objects, so even a detached HEAD resolves - shutil.rmtree(target, ignore_errors=True) - _run(["git", "clone", "--quiet", "--shared", "--no-checkout", str(source), str(target)]) - head = _output(["git", "-C", str(source), "rev-parse", "HEAD"]).strip() - _run(["git", "-C", str(target), "checkout", "--quiet", "--detach", head]) - if diff := subprocess.run( - ["git", "-C", str(source), "diff", "--binary", "HEAD"], check=True, capture_output=True - ).stdout: - subprocess.run(["git", "-C", str(target), "apply", "--index"], check=True, input=diff) - for name in _output(["git", "-C", str(source), "ls-files", "--others", "--exclude-standard", "-z"]).split("\0"): - if name: - (target / name).parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source / name, target / name) - return target - - -def _commit(root: Path) -> None: - # a checkout, as projects build from one, so the timed runs include reading the source revision with git - _run(["git", "init", "--quiet", str(root)]) - _run(["git", "-C", str(root), "add", "."]) - _run([ - "git", - "-C", - str(root), - *_GIT_IDENTITY, - "commit", - "--quiet", - "--no-verify", - "--no-gpg-sign", - "-m", - "bench fixture", - ]) - - -def _run(command: list[str], *, cwd: Path | None = None, env: dict[str, str] | None = None) -> None: - subprocess.run(command, check=True, cwd=cwd, env=env) - - -def _output(command: list[str]) -> str: - return subprocess.run(command, check=True, capture_output=True, text=True).stdout +def load(work: Path) -> dict[str, str]: + if not (manifest := work / "fixtures.json").is_file(): + msg = f"{manifest} does not exist; run `python -m bench prepare` first" + raise SystemExit(msg) + return json.loads(manifest.read_text(encoding="utf-8")) __all__ = [ diff --git a/pyproject.toml b/pyproject.toml index 90c41cd..7594c2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,8 +80,8 @@ dev = [ test = [ "covdefaults>=2.3", "cyclonedx-python-lib[json-validation]>=11.12", - "hatchling>=1.28", "dep-logic>=0.7.2", + "hatchling>=1.28", "installer>=1.1", "pytest>=9.1.1", "pytest-cov>=7.1", From 3bdd22e85104bee6ae9c266133a35ee5388e235c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:20:46 -0800 Subject: [PATCH 026/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20parse?= =?UTF-8?q?=20dist-info=20files=20with=20installer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/python_sbom/_collect/_facts.py | 6 ++++-- src/python_sbom/_generate/_archive.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index 0153b8a..8090b34 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -9,10 +9,10 @@ import sys import time import warnings -from email.parser import Parser from importlib.metadata import distributions from typing import TYPE_CHECKING, Final, Protocol +from installer.utils import parse_metadata_file from packaging.direct_url import DirectUrl, DirectUrlValidationError from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name @@ -84,7 +84,9 @@ def _installed(path: Sequence[str] | None) -> list[InstalledDistribution]: def _recorded_files(distribution: Distribution) -> list[RecordRow]: # a platform wheel's files are built for this machine's OS and architecture, and an editable install's point at a # checkout, so listing either would tie the document to one machine - if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] != "true" or _editable(distribution): + if parse_metadata_file(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] != "true" or _editable( + distribution + ): return [] rows: Final[list[RecordRow]] = [] for file in distribution.files or []: diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index 0d75cbf..a4fad4c 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -3,10 +3,10 @@ import hashlib import tempfile from dataclasses import dataclass -from email.parser import Parser from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Final +from installer.utils import parse_metadata_file from packageurl import PackageURL from python_sbom._collect._content import load_content_provider, provider_script @@ -115,7 +115,7 @@ def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, handle.extractall(unpacked) # each distribution by the directory that holds its .dist-info, the project's own at the top level metadata = { - f"{name.rsplit('/', 2)[0]}/" if name.count("/") > 1 else "": Parser().parsestr( + f"{name.rsplit('/', 2)[0]}/" if name.count("/") > 1 else "": parse_metadata_file( (unpacked / name).read_text(encoding="utf-8") ) for name in members From 2d87c0d1f90cc4c3cba1e465a62b393b22c191cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:20:46 -0800 Subject: [PATCH 027/115] =?UTF-8?q?=F0=9F=94=92=20build:=20lock=20dep-logi?= =?UTF-8?q?c?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pylock.toml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/pylock.toml b/pylock.toml index df5ccfc..7574fe4 100644 --- a/pylock.toml +++ b/pylock.toml @@ -658,6 +658,34 @@ size = 25604 [packages.wheels.hashes] sha256 = "a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" +[[packages]] +name = "dep-logic" +version = "0.7.2" +marker = "\"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.10" +dependencies = [ + { name = "packaging" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "dep_logic-0.7.2.tar.gz" +upload-time = 2026-08-28 03:43:17.064829+00:00 +url = "https://files.pythonhosted.org/packages/1b/7a/ac7db322bfc792c237e059e0b048e8e0784eb624d0447e3cf5e231323725/dep_logic-0.7.2.tar.gz" +size = 37962 + +[packages.sdist.hashes] +sha256 = "2a3c49526075b0bc6825f0ea8c2d9f464de48b7b85f3f3d2900b4e5950fefedf" + +[[packages.wheels]] +name = "dep_logic-0.7.2-py3-none-any.whl" +upload-time = 2026-08-28 03:43:15.718045+00:00 +url = "https://files.pythonhosted.org/packages/d0/71/e6979663eb7feaab2a2fcd3a004c0fc33b2a5d0ebafc4c474000b2be6732/dep_logic-0.7.2-py3-none-any.whl" +size = 36183 + +[packages.wheels.hashes] +sha256 = "33640c4ba7fd45f2504442eb9e0c12e57dac253c748ca3925749548a5c556bae" + [[packages]] name = "distlib" version = "0.4.4" @@ -1009,7 +1037,7 @@ sha256 = "9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7" [[packages]] name = "installer" version = "1.1.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -1614,7 +1642,7 @@ sha256 = "bc0fb085d8ed0d67a6471892535f3d9b795eb911b7cd129d3a24967aab7a538f" [[packages]] name = "packageurl-python" version = "0.17.6" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -3580,7 +3608,6 @@ command-line = [ "--all-groups", "--all-extras", "--no-cache", - "--upgrade", ] input-path = "pyproject.toml" mode = "universal" From 0c4aba6d60968a41a8cce8353a7a4334756c8bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:39:53 -0800 Subject: [PATCH 028/115] =?UTF-8?q?=E2=9C=85=20test:=20pin=20embed=20outpu?= =?UTF-8?q?t=20byte=20for=20byte?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fixed project, wheel and build environment must produce the same SBOM and wheel members, so optimisations prove they change nothing. The test wheels are written with Unix attributes, and the pinned ones stored, so the bytes match on every OS and zlib build. --- .gitattributes | 1 + tests/_generate/pinned/embed-members.json | 100 ++++ tests/_generate/pinned/embed.cdx.json | 583 ++++++++++++++++++++++ tests/_generate/test_embed.py | 84 +++- tests/conftest.py | 12 +- 5 files changed, 774 insertions(+), 6 deletions(-) create mode 100644 tests/_generate/pinned/embed-members.json create mode 100644 tests/_generate/pinned/embed.cdx.json diff --git a/.gitattributes b/.gitattributes index a94cb2f..5e8ba4d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ .git_archival.txt export-subst +tests/_generate/pinned/** -text diff --git a/tests/_generate/pinned/embed-members.json b/tests/_generate/pinned/embed-members.json new file mode 100644 index 0000000..f7d3139 --- /dev/null +++ b/tests/_generate/pinned/embed-members.json @@ -0,0 +1,100 @@ +[ + [ + "pkg_name/__init__.py", + [ + 2024, + 1, + 1, + 0, + 0, + 0 + ], + 0, + 25165824, + "9e26bf369911c45c243c684147b23fc9e1dcfcf257d299a1c632016a6fcd33f4" + ], + [ + "pkg_name/wheels/pip-1.0-py3-none-any.whl", + [ + 2024, + 1, + 1, + 0, + 0, + 0 + ], + 0, + 25165824, + "a64ff1b762e23c4eb0a3b4a8fb85fe12d18de3dce86ea6471f3f546285976734" + ], + [ + "pkg_name-1.0.dist-info/METADATA", + [ + 2024, + 1, + 1, + 0, + 0, + 0 + ], + 0, + 25165824, + "f18fd332176c30ba5df08ce78bd605a6616ed6e7e5e165a4a0b05e6db880b0a2" + ], + [ + "pkg_name-1.0.dist-info/licenses/LICENSE", + [ + 2024, + 1, + 1, + 0, + 0, + 0 + ], + 0, + 25165824, + "d0febe6f2e066329c487c2621cd02a4eddce97b51cdacb22600dd5181d843c08" + ], + [ + "pkg_name-1.0.dist-info/WHEEL", + [ + 2024, + 1, + 1, + 0, + 0, + 0 + ], + 0, + 25165824, + "242557f73f15fa3b36695e6a9904761377e20acf98bb7bcf3bdd5c08198ed493" + ], + [ + "pkg_name-1.0.dist-info/sboms/pkg-name.cdx.json", + [ + 2023, + 11, + 14, + 22, + 13, + 20 + ], + 8, + 2175008768, + "63503a8e31088e81123f5b7861da0411bc423174c7353aa74d2d1f9391ce23e4" + ], + [ + "pkg_name-1.0.dist-info/RECORD", + [ + 2024, + 1, + 1, + 0, + 0, + 0 + ], + 0, + 25165824, + "a932f8430d0fec72d6bdfc1aa0afa1700ede4c4b6eec6a825e245e9835ed0a82" + ] +] diff --git a/tests/_generate/pinned/embed.cdx.json b/tests/_generate/pinned/embed.cdx.json new file mode 100644 index 0000000..2df0c7b --- /dev/null +++ b/tests/_generate/pinned/embed.cdx.json @@ -0,0 +1,583 @@ +{ + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": "urn:uuid:d3cce085-f38d-5e88-a7b1-b1b95274c6f3", + "version": 1, + "metadata": { + "timestamp": "2023-11-14T22:13:20+00:00", + "lifecycles": [ + { + "phase": "build" + } + ], + "tools": { + "components": [ + { + "type": "platform", + "bom-ref": "tool:pkg:generic/cpython@3.14.0", + "name": "cpython", + "version": "3.14.0", + "purl": "pkg:generic/cpython@3.14.0", + "properties": [ + { + "name": "python:implementation", + "value": "CPython" + } + ] + }, + { + "type": "library", + "bom-ref": "tool:pkg:pypi/alpha@1.0", + "name": "alpha", + "version": "1.0", + "purl": "pkg:pypi/alpha@1.0", + "externalReferences": [], + "properties": [ + { + "name": "python:requires-dist", + "value": "beta" + } + ], + "components": [ + { + "type": "file", + "bom-ref": "tool:pkg:pypi/alpha@1.0#alpha.py", + "name": "alpha.py", + "hashes": [ + { + "alg": "SHA-256", + "content": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ], + "properties": [ + { + "name": "size", + "value": "0" + } + ] + }, + { + "type": "file", + "bom-ref": "tool:pkg:pypi/alpha@1.0#alpha-1.0.dist-info/METADATA", + "name": "alpha-1.0.dist-info/METADATA", + "hashes": [ + { + "alg": "SHA-256", + "content": "84fff1197cd817e092c798eaadecbb239d6076bda206362533d29a5151059eb1" + } + ], + "properties": [ + { + "name": "size", + "value": "45" + } + ] + }, + { + "type": "file", + "bom-ref": "tool:pkg:pypi/alpha@1.0#alpha-1.0.dist-info/WHEEL", + "name": "alpha-1.0.dist-info/WHEEL", + "hashes": [ + { + "alg": "SHA-256", + "content": "a4e0b653593a8427ffbe465eee1dac9c6456c7e0b6d178ee7376b4d8a427d8e6" + } + ], + "properties": [ + { + "name": "size", + "value": "41" + } + ] + } + ] + }, + { + "type": "library", + "bom-ref": "tool:pkg:pypi/beta@2.0", + "name": "beta", + "version": "2.0", + "purl": "pkg:pypi/beta@2.0", + "externalReferences": [], + "properties": [] + }, + { + "type": "library", + "bom-ref": "tool:pkg:pypi/gamma@3.0", + "name": "gamma", + "version": "3.0", + "purl": "pkg:pypi/gamma@3.0", + "externalReferences": [], + "properties": [] + } + ] + }, + "component": { + "type": "application", + "bom-ref": "pkg:pypi/pkg.name@1.0", + "name": "Pkg.Name", + "version": "1.0", + "purl": "pkg:pypi/pkg.name@1.0", + "description": "a pinned example", + "authors": [ + { + "name": "not-an-email" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0", + "acknowledgement": "declared" + } + ], + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/pkg", + "comment": "Project-URL: Source" + }, + { + "type": "vcs", + "url": "https://gitlab.com/example/pkg", + "comment": "Project-URL: Source" + }, + { + "type": "documentation", + "url": "https://pkg.readthedocs.io", + "comment": "Project-URL: Documentation" + }, + { + "type": "issue-tracker", + "url": "https://github.com/example/pkg/issues", + "comment": "Project-URL: Bug Tracker" + }, + { + "type": "distribution", + "url": "https://pypi.org/project/Pkg.Name/1.0/" + }, + { + "type": "attestation", + "url": "https://pypi.org/integrity/Pkg.Name/1.0/pkg_name-1.0-py3-none-any.whl/provenance" + }, + { + "type": "security-contact", + "url": "https://github.com/example/pkg/security/policy" + }, + { + "type": "advisories", + "url": "https://github.com/example/pkg/security/advisories" + }, + { + "type": "license", + "url": "https://github.com/example/pkg/blob/main/LICENSE" + }, + { + "type": "vcs", + "url": "https://github.com/example/pkg/tree/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "comment": "exact source revision" + } + ], + "properties": [ + { + "name": "python:requires-dist", + "value": "alpha>=1" + }, + { + "name": "python:requires-dist", + "value": "beta; extra == \"cli\"" + }, + { + "name": "python:requires-dist", + "value": "gamma; python_version < \"3\"" + }, + { + "name": "python:requires-dist", + "value": "delta; extra in \"a,b\"" + }, + { + "name": "python:provides-extra", + "value": "cli" + }, + { + "name": "python:classifier", + "value": "License :: OSI Approved :: MIT License" + }, + { + "name": "python:classifier", + "value": "Programming Language :: Python :: 3" + }, + { + "name": "python:keywords", + "value": "sbom,pinned" + }, + { + "name": "pkg-name:vcs-commit", + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ], + "supplier": { + "name": "Example Org", + "url": [ + "https://example.org" + ] + } + }, + "manufacturer": { + "name": "Example Org", + "url": [ + "https://example.org" + ] + }, + "supplier": { + "name": "Example Org", + "url": [ + "https://example.org" + ] + }, + "authors": [ + { + "name": "not-an-email" + } + ], + "licenses": [ + { + "expression": "MIT OR Apache-2.0", + "acknowledgement": "declared" + } + ], + "properties": [ + { + "name": "pkg-name:sbom:scope", + "value": "Components are the wheels bundled as data files under src/pkg/wheels and the direct runtime dependencies declared in the wheel's core metadata, which carry no version because the installer resolves them." + }, + { + "name": "pkg-name:sbom:file", + "value": "pkg-name.cdx.json" + }, + { + "name": "pkg-name:sbom:repository", + "value": "https://github.com/example/pkg" + } + ] + }, + "components": [ + { + "type": "library", + "bom-ref": "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl", + "name": "pip", + "version": "1.0", + "purl": "pkg:pypi/pip@1.0", + "licenses": [ + { + "expression": "MIT", + "acknowledgement": "declared" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://pypi.org/project/pip/1.0/" + } + ], + "properties": [ + { + "name": "pkg-name:bundled-wheel", + "value": "pkg_name/wheels/pip-1.0-py3-none-any.whl" + } + ], + "hashes": [ + { + "alg": "SHA-256", + "content": "a64ff1b762e23c4eb0a3b4a8fb85fe12d18de3dce86ea6471f3f546285976734" + } + ], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [ + { + "technique": "manifest-analysis", + "confidence": 1, + "value": "pip-1.0.dist-info/METADATA" + } + ] + }, + { + "field": "hash", + "confidence": 1, + "methods": [ + { + "technique": "hash-comparison", + "confidence": 1, + "value": "a64ff1b762e23c4eb0a3b4a8fb85fe12d18de3dce86ea6471f3f546285976734" + } + ] + } + ] + }, + "components": [ + { + "type": "file", + "bom-ref": "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl#pip/__init__.py", + "name": "pip/__init__.py", + "hashes": [ + { + "alg": "SHA-256", + "content": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ], + "properties": [ + { + "name": "size", + "value": "0" + } + ] + }, + { + "type": "file", + "bom-ref": "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl#pip/_vendor/vendor.txt", + "name": "pip/_vendor/vendor.txt", + "hashes": [ + { + "alg": "SHA-256", + "content": "26c04392ab4b025968e4dbdaca5fcc4ec977c70607bb02d4ae7bc724d4ba92ea" + } + ], + "properties": [ + { + "name": "size", + "value": "26" + } + ] + }, + { + "type": "file", + "bom-ref": "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl#pip-1.0.dist-info/METADATA", + "name": "pip-1.0.dist-info/METADATA", + "hashes": [ + { + "alg": "SHA-256", + "content": "3ea62ee2e6d4513f0a06630188f10dbeace9ed76fd29897bf76edd496d948e37" + } + ], + "properties": [ + { + "name": "size", + "value": "69" + } + ] + }, + { + "type": "library", + "bom-ref": "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl#vendored/pkg:pypi/idna@3.7", + "name": "idna", + "version": "3.7", + "purl": "pkg:pypi/idna@3.7", + "externalReferences": [], + "properties": [ + { + "name": "pkg-name:vendored-manifest", + "value": "pip/_vendor/vendor.txt" + } + ], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [ + { + "technique": "manifest-analysis", + "confidence": 1, + "value": "pip/_vendor/vendor.txt" + } + ] + } + ] + } + }, + { + "type": "library", + "bom-ref": "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl#vendored/pkg:pypi/urllib3@1.26.4", + "name": "urllib3", + "version": "1.26.4", + "purl": "pkg:pypi/urllib3@1.26.4", + "externalReferences": [], + "properties": [ + { + "name": "pkg-name:vendored-manifest", + "value": "pip/_vendor/vendor.txt" + } + ], + "evidence": { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [ + { + "technique": "manifest-analysis", + "confidence": 1, + "value": "pip/_vendor/vendor.txt" + } + ] + } + ] + } + } + ] + }, + { + "type": "library", + "bom-ref": "requires-dist:alpha>=1", + "name": "alpha", + "purl": "pkg:pypi/alpha", + "externalReferences": [ + { + "type": "distribution", + "url": "https://pypi.org/project/alpha/" + } + ], + "properties": [ + { + "name": "pkg-name:requires-dist", + "value": "alpha>=1" + } + ] + }, + { + "type": "library", + "bom-ref": "requires-dist:gamma; python_version < \"3\"", + "name": "gamma", + "purl": "pkg:pypi/gamma", + "externalReferences": [ + { + "type": "distribution", + "url": "https://pypi.org/project/gamma/" + } + ], + "properties": [ + { + "name": "pkg-name:requires-dist", + "value": "gamma; python_version < \"3\"" + }, + { + "name": "pkg-name:environment-marker", + "value": "python_version < \"3\"" + } + ] + }, + { + "type": "library", + "bom-ref": "requires-dist:delta; extra in \"a,b\"", + "name": "delta", + "purl": "pkg:pypi/delta", + "externalReferences": [ + { + "type": "distribution", + "url": "https://pypi.org/project/delta/" + } + ], + "properties": [ + { + "name": "pkg-name:requires-dist", + "value": "delta; extra in \"a,b\"" + }, + { + "name": "pkg-name:environment-marker", + "value": "extra in \"a,b\"" + } + ] + } + ], + "dependencies": [ + { + "ref": "pkg:pypi/pkg.name@1.0", + "dependsOn": [ + "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl", + "requires-dist:alpha>=1", + "requires-dist:gamma; python_version < \"3\"", + "requires-dist:delta; extra in \"a,b\"" + ] + }, + { + "ref": "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl", + "dependsOn": [ + "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl#vendored/pkg:pypi/idna@3.7", + "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl#vendored/pkg:pypi/urllib3@1.26.4" + ] + }, + { + "ref": "tool:pkg:pypi/alpha@1.0", + "dependsOn": [ + "tool:pkg:pypi/beta@2.0" + ] + }, + { + "ref": "tool:pkg:pypi/beta@2.0", + "dependsOn": [] + }, + { + "ref": "tool:pkg:pypi/gamma@3.0", + "dependsOn": [] + } + ], + "compositions": [ + { + "aggregate": "incomplete", + "dependencies": [ + "pkg:pypi/pkg.name@1.0" + ] + } + ], + "formulation": [ + { + "bom-ref": "formula:wheel-build", + "workflows": [ + { + "bom-ref": "workflow:wheel-build", + "uid": "wheel-build", + "name": "build the wheel and this SBOM", + "taskTypes": [ + "build" + ], + "resourceReferences": [ + { + "ref": "tool:pkg:generic/cpython@3.14.0" + }, + { + "ref": "tool:pkg:pypi/alpha@1.0" + }, + { + "ref": "tool:pkg:pypi/beta@2.0" + }, + { + "ref": "tool:pkg:pypi/gamma@3.0" + } + ], + "outputs": [ + { + "type": "artifact", + "resource": { + "ref": "pkg:pypi/pkg.name@1.0" + } + } + ], + "inputs": [ + { + "environmentVars": [ + { + "name": "SOURCE_DATE_EPOCH", + "value": "1700000000" + } + ] + } + ] + } + ] + } + ] +} diff --git a/tests/_generate/test_embed.py b/tests/_generate/test_embed.py index 8eb2282..1db1afa 100644 --- a/tests/_generate/test_embed.py +++ b/tests/_generate/test_embed.py @@ -1,16 +1,22 @@ from __future__ import annotations +import hashlib import json +import platform import zipfile -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Final from python_sbom import BundledWheelInfo, ContentProvider, ProviderContext from python_sbom._generate._embed import embed_sbom if TYPE_CHECKING: - from pathlib import Path, PurePosixPath + from pathlib import PurePosixPath - from conftest import WriteWheel + import pytest + from conftest import Install, WriteWheel + +_PINNED: Final[Path] = Path(__file__).parent / "pinned" def test_bundled_wheel_named_by_its_path_inside_the_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: @@ -47,3 +53,75 @@ def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> Bundl # the table sits next to the wheel inside the built wheel value = context.read(wheel.parent / self._name).decode().strip() return BundledWheelInfo(properties=[{"name": f"{context.identity.property_prefix}:table", "value": value}]) + + +def test_embed_output_is_pinned( + tmp_path: Path, write_wheel: WriteWheel, install: Install, monkeypatch: pytest.MonkeyPatch +) -> None: + # every input fixed, down to the interpreter's version, so the SBOM and the wheel's members must not change by a + # byte; a change that means to alter them updates the pinned files with it + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + monkeypatch.setattr(platform, "python_version", lambda: "3.14.0") + metadata = b"""\ +Metadata-Version: 2.4 +Name: Pkg.Name +Version: 1.0 +Summary: a pinned example +Keywords: sbom,pinned +Author: Author Name +Author-email: Author , plain@example.com +Maintainer-email: not-an-email +License-Expression: MIT OR Apache-2.0 +License-File: LICENSE +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 +Project-URL: Source, https://github.com/example/pkg +Project-URL: Source, https://gitlab.com/example/pkg +Project-URL: Documentation, https://pkg.readthedocs.io +Project-URL: Bug Tracker, https://github.com/example/pkg/issues +Requires-Dist: alpha>=1 +Requires-Dist: beta; extra == "cli" +Requires-Dist: gamma; python_version < "3" +Requires-Dist: delta; extra in "a,b" +Provides-Extra: cli +""" + (root := tmp_path / "project").mkdir() + (root / "pyproject.toml").write_bytes( + b"[tool.python-sbom]\nsupplier = { name = 'Example Org', url = ['https://example.org'] }\n" + b"bundled-wheels = ['src/pkg/wheels/*.whl']\n" + ) + pip = { + "pip/__init__.py": b"", + "pip/_vendor/vendor.txt": b"urllib3==1.26.4\nidna==3.7\n", + "pip-1.0.dist-info/METADATA": b"Metadata-Version: 2.4\nName: pip\nVersion: 1.0\nLicense-Expression: MIT\n", + } + # stored, as deflate output differs between zlib builds and the SBOM records the bundled wheel's hash + bundled = write_wheel(root / "src" / "pkg" / "wheels" / "pip-1.0-py3-none-any.whl", pip, compression=0) + site = tmp_path / "site-packages" + install(site, "Name: alpha\nVersion: 1.0\nRequires-Dist: beta\n", {"alpha.py": b"", "../../bin/alpha": b""}) + install(site, "Name: beta\nVersion: 2.0\n", {"beta.so": b"\x7fELF"}, purelib=False) + install(site, "Name: gamma\nVersion: 3.0\n", {}, direct_url='{"url": "file:///c", "dir_info": {"editable": true}}') + wheel = write_wheel( + tmp_path / "dist" / "pkg_name-1.0-py3-none-any.whl", + { + "pkg_name/__init__.py": b"x = 1\n", + "pkg_name/wheels/pip-1.0-py3-none-any.whl": bundled.read_bytes(), + "pkg_name-1.0.dist-info/METADATA": metadata, + "pkg_name-1.0.dist-info/licenses/LICENSE": b"MIT terms\n", + "pkg_name-1.0.dist-info/WHEEL": b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + }, + compression=0, + ) + written = embed_sbom(wheel, root, content=None, commit="a" * 40, build_env=[str(site)]) + with zipfile.ZipFile(wheel) as archive: + sbom = archive.read(written) + members = [ + [info.filename, list(info.date_time), info.compress_type, info.external_attr, _sha256(archive.read(info))] + for info in archive.infolist() + ] + assert sbom == (_PINNED / "embed.cdx.json").read_bytes() + assert members == json.loads((_PINNED / "embed-members.json").read_bytes()) + + +def _sha256(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() diff --git a/tests/conftest.py b/tests/conftest.py index dff5c64..371ca0b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,7 +23,9 @@ class WriteWheel(Protocol): - def __call__(self, wheel: Path, files: Mapping[str, bytes], *, record: bool = True) -> Path: ... + def __call__( + self, wheel: Path, files: Mapping[str, bytes], *, record: bool = True, compression: int = ... + ) -> Path: ... class Install(Protocol): @@ -54,7 +56,9 @@ def __call__( @pytest.fixture def write_wheel() -> WriteWheel: - def write(wheel: Path, files: Mapping[str, bytes], *, record: bool = True) -> Path: + def write( + wheel: Path, files: Mapping[str, bytes], *, record: bool = True, compression: int = zipfile.ZIP_DEFLATED + ) -> Path: members = dict(files) if record: dist_info = next(name.split("/")[0] for name in files if name.endswith(".dist-info/METADATA")) @@ -63,7 +67,9 @@ def write(wheel: Path, files: Mapping[str, bytes], *, record: bool = True) -> Pa wheel.parent.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: for name, content in members.items(): - archive.writestr(zipfile.ZipInfo(name, (2024, 1, 1, 0, 0, 0)), content, zipfile.ZIP_DEFLATED) + # Unix, the default everywhere but Windows, so the bytes match across operating systems + (info := zipfile.ZipInfo(name, (2024, 1, 1, 0, 0, 0))).create_system = 3 + archive.writestr(info, content, compression) return wheel return write From 95af3d15b31f7b5f2e9976317731ff7ac99b20c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:39:55 -0800 Subject: [PATCH 029/115] =?UTF-8?q?=E2=8F=AA=20revert:=20evaluate=20marker?= =?UTF-8?q?s=20without=20dep-logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dep-logic rejects extra comparisons other than == and != through assert, which python -O strips, losing requirements that apply, and on import it replaces packaging's marker grammar for the whole build process. The own evaluation returns, and an extra compared as a version now stays unknown instead of raising. A test runs the verdicts under python -O. --- pylock.toml | 28 ------------------- pyproject.toml | 3 -- src/python_sbom/_model/_markers.py | 45 ++++++++++++++++++++++++------ tests/_model/test_markers.py | 36 +++++++++++++++++++----- 4 files changed, 66 insertions(+), 46 deletions(-) diff --git a/pylock.toml b/pylock.toml index 7574fe4..bf48fd3 100644 --- a/pylock.toml +++ b/pylock.toml @@ -658,34 +658,6 @@ size = 25604 [packages.wheels.hashes] sha256 = "a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" -[[packages]] -name = "dep-logic" -version = "0.7.2" -marker = "\"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" -requires-python = ">=3.10" -dependencies = [ - { name = "packaging" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "dep_logic-0.7.2.tar.gz" -upload-time = 2026-08-28 03:43:17.064829+00:00 -url = "https://files.pythonhosted.org/packages/1b/7a/ac7db322bfc792c237e059e0b048e8e0784eb624d0447e3cf5e231323725/dep_logic-0.7.2.tar.gz" -size = 37962 - -[packages.sdist.hashes] -sha256 = "2a3c49526075b0bc6825f0ea8c2d9f464de48b7b85f3f3d2900b4e5950fefedf" - -[[packages.wheels]] -name = "dep_logic-0.7.2-py3-none-any.whl" -upload-time = 2026-08-28 03:43:15.718045+00:00 -url = "https://files.pythonhosted.org/packages/d0/71/e6979663eb7feaab2a2fcd3a004c0fc33b2a5d0ebafc4c474000b2be6732/dep_logic-0.7.2-py3-none-any.whl" -size = 36183 - -[packages.wheels.hashes] -sha256 = "33640c4ba7fd45f2504442eb9e0c12e57dac253c748ca3925749548a5c556bae" - [[packages]] name = "distlib" version = "0.4.4" diff --git a/pyproject.toml b/pyproject.toml index 7594c2f..9ec74df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,6 @@ [build-system] build-backend = "hatchling.build" requires = [ - "dep-logic>=0.7.2", "hatch-fancy-pypi-readme>=25.1", "hatch-vcs>=0.5", "hatchling>=1.32.4", @@ -49,7 +48,6 @@ dynamic = [ "version", ] dependencies = [ - "dep-logic>=0.7.2", "installer>=1.1", "packageurl-python>=0.11.1", "packaging>=26.3", @@ -80,7 +78,6 @@ dev = [ test = [ "covdefaults>=2.3", "cyclonedx-python-lib[json-validation]>=11.12", - "dep-logic>=0.7.2", "hatchling>=1.28", "installer>=1.1", "pytest>=9.1.1", diff --git a/src/python_sbom/_model/_markers.py b/src/python_sbom/_model/_markers.py index 4c04113..e2513ea 100644 --- a/src/python_sbom/_model/_markers.py +++ b/src/python_sbom/_model/_markers.py @@ -1,23 +1,52 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final -from dep_logic.markers import from_pkg_marker +from packaging._parser import Op, Value, Variable # ruff: ignore[import-private-name] # packaging exposes the parsed marker tree only privately +from packaging.markers import Marker, UndefinedComparison if TYPE_CHECKING: + from packaging._parser import MarkerAtom, MarkerList # see above from packaging.requirements import Requirement def applies_without_extras(requirement: Requirement) -> bool: - # dep-logic reduces the marker to its conditions on extra, holding where some environment satisfies the rest, so - # the requirement applies unless they fail with no extra requested; a marker no environment satisfies never applies + # the conditions on extra are decided with no extra requested and every other condition stays unknown, so the + # requirement applies unless its extra conditions alone rule it out; dep-logic reduces markers the same way, but it + # rejects extra comparisons other than == and != through assert, which python -O strips, and on import it replaces + # packaging's marker grammar for the whole build process if requirement.marker is None: return True + return _extra_verdict(requirement.marker._markers) is not False # ruff: ignore[private-member-access] # the tree, see the import above + + +def _extra_verdict(markers: MarkerList | MarkerAtom) -> bool | None: + # three-valued, with "and" binding tighter than "or", as packaging's own evaluation groups them + groups: list[list[bool | None]] = [[]] + for item in markers: + if item == "or": + groups.append([]) + elif isinstance(item, list): + groups[-1].append(_extra_verdict(item)) + elif isinstance(item, tuple): + match item: + case Variable(value="extra"), Op(value=operator), Value(value=value): + groups[-1].append(_no_extra_verdict(f"extra {operator} {value!r}")) + case Value(value=value), Op(value=operator), Variable(value="extra"): + groups[-1].append(_no_extra_verdict(f"{value!r} {operator} extra")) + case _: + groups[-1].append(None) + # ranked False < unknown < True: "and" takes the lowest verdict of a group, "or" the highest of the groups + verdicts: Final[tuple[bool | None, ...]] = (False, None, True) + return verdicts[max(min(map(verdicts.index, group), default=2) for group in groups)] + + +def _no_extra_verdict(condition: str) -> bool | None: try: - return from_pkg_marker(requirement.marker).only("extra").evaluate({"extra": ""}) - # dep-logic asserts extra is compared through == or != only; listing such a requirement beats leaving it out - except AssertionError: - return True + return Marker(condition).evaluate({"extra": ""}) + # extra compared as a version, such as extra ~= "1.0", has no answer, so it stays unknown + except UndefinedComparison: + return None __all__ = [ diff --git a/tests/_model/test_markers.py b/tests/_model/test_markers.py index 2c7a687..5456cea 100644 --- a/tests/_model/test_markers.py +++ b/tests/_model/test_markers.py @@ -1,5 +1,8 @@ from __future__ import annotations +import subprocess +import sys + import pytest from packaging.requirements import Requirement @@ -22,16 +25,35 @@ pytest.param('click; python_version >= "3" or extra == "a"', True, id="environment-or-extra"), pytest.param("click", True, id="no-marker"), pytest.param('click; extra == "A_b" and extra != "a-b"', False, id="extra-names-normalized"), - # no environment satisfies the marker, so no install ever pulls the requirement in - pytest.param('click; python_version < "3" and python_version >= "3"', False, id="environment-contradiction"), - pytest.param( - 'click; (os_name == "nt" and os_name != "nt") or extra == "a"', False, id="contradiction-or-extra" - ), - # dep-logic compares extra through == and != only, so other comparisons keep the requirement listed + # only the conditions on extra decide, so a marker no environment satisfies still leaves the requirement listed + pytest.param('click; python_version < "3" and python_version >= "3"', True, id="environment-contradiction"), + pytest.param('click; (os_name == "nt" and os_name != "nt") or extra == "a"', True, id="contradiction-or-extra"), pytest.param('click; extra in "a,b"', True, id="extra-in"), - pytest.param('click; extra not in "a,b"', True, id="extra-not-in"), + pytest.param('click; extra not in "a,b"', False, id="extra-not-in"), + # extra compared as a version has no answer, so it stays unknown pytest.param('click; extra ~= "1.0" and os_name == "nt"', True, id="extra-compatible-release"), ], ) def test_applies_without_extras(requirement: str, applies: bool) -> None: assert applies_without_extras(Requirement(requirement)) is applies + + +def test_verdicts_hold_under_python_optimize() -> None: + # python -O strips assert statements, through which dep-logic turned these comparisons on extra away and so lost + # requirements that apply; the verdicts here must not depend on the mode + markers = { + 'extra in "a,b" and os_name == "nt"': True, + 'extra not in "a,b" or python_version < "3.11"': True, + 'extra < "b" and sys_platform == "linux"': False, + '"a" in extra or os_name == "nt"': True, + 'extra not in "" and python_version >= "3"': False, + '(extra >= "a" or extra == "cli") and os_name != "nt"': False, + } + script = ( + "import sys\n" + "from packaging.requirements import Requirement\n" + "from python_sbom._model._markers import applies_without_extras\n" + "print(*(applies_without_extras(Requirement(f'click; {marker}')) for marker in sys.argv[1:]))\n" + ) + result = subprocess.run([sys.executable, "-O", "-c", script, *markers], capture_output=True, text=True, check=True) + assert result.stdout.split() == [str(verdict) for verdict in markers.values()] From ed0130b5bda3311b75a9d0622fd6ee991a780334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:39:55 -0800 Subject: [PATCH 030/115] =?UTF-8?q?=E2=9A=A1=20perf:=20read=20build-env=20?= =?UTF-8?q?RECORDs=20directly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distribution.files stats every recorded file and builds a path object per row; reading RECORD once through installer's parser halves the collection time. A file recorded but missing on disk is now listed on every Python, as 3.11 already did, and a malformed RECORD warns instead of raising. --- src/python_sbom/_collect/_facts.py | 24 ++++++++++++++++++------ tests/_collect/test_facts.py | 27 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index 8090b34..c34bf5f 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -10,8 +10,10 @@ import time import warnings from importlib.metadata import distributions +from itertools import starmap from typing import TYPE_CHECKING, Final, Protocol +from installer.records import InvalidRecordEntry, RecordEntry, parse_record_file from installer.utils import parse_metadata_file from packaging.direct_url import DirectUrl, DirectUrlValidationError from packaging.requirements import InvalidRequirement, Requirement @@ -88,12 +90,21 @@ def _recorded_files(distribution: Distribution) -> list[RecordRow]: distribution ): return [] + # RECORD read once and as written: Distribution.files would stat every row to drop files missing on disk, and + # build a path object for each, while the SBOM lists what the install recorded + try: + entries: Final[list[RecordEntry]] = list( + starmap(RecordEntry.from_elements, parse_record_file((distribution.read_text("RECORD") or "").splitlines())) + ) + except InvalidRecordEntry as error: + warnings.warn(f"listed no files of {distribution.name}, its RECORD is malformed: {error}", stacklevel=2) + return [] rows: Final[list[RecordRow]] = [] - for file in distribution.files or []: - if file.hash is None or not _describes_distribution(file.as_posix(), file.parent.suffix, file.name): + for entry in entries: + if entry.hash_ is None or not _describes_distribution(entry.path): continue - algorithm, digest = decode_digest(f"{file.hash.mode}={file.hash.value}", file.as_posix()) - rows.append(RecordRow(path=file.as_posix(), algorithm=algorithm, digest=digest, size=str(file.size))) + algorithm, digest = decode_digest(str(entry.hash_), entry.path) + rows.append(RecordRow(path=entry.path, algorithm=algorithm, digest=digest, size=str(entry.size))) return rows @@ -109,12 +120,13 @@ def _editable(distribution: Distribution) -> bool: return dir_info is not None and dir_info.editable is True -def _describes_distribution(path: str, parent_suffix: str, name: str) -> bool: +def _describes_distribution(path: str) -> bool: # console-script launchers live outside site-packages and embed the environment's interpreter path; the installer # metadata names the frontend; uv's cache files hold paths and times; none of them says anything about the package + directory, _, name = path.rpartition("/") return not ( path.startswith("../") - or (parent_suffix == ".dist-info" and name in {"INSTALLER", "REQUESTED", "direct_url.json"}) + or (directory.endswith(".dist-info") and name in {"INSTALLER", "REQUESTED", "direct_url.json"}) or re.fullmatch(r"uv_.*\.json", name) ) diff --git a/tests/_collect/test_facts.py b/tests/_collect/test_facts.py index da345c3..94ab550 100644 --- a/tests/_collect/test_facts.py +++ b/tests/_collect/test_facts.py @@ -2,6 +2,7 @@ import hashlib import platform +import re import sys import time from typing import TYPE_CHECKING @@ -185,6 +186,32 @@ def test_egg_info_distribution(tmp_path: Path) -> None: assert (distribution.name, distribution.metadata["Version"], distribution.files) == ("legacy", "1.0", []) +def test_recorded_file_missing_on_disk_is_listed(tmp_path: Path, install: Install) -> None: + # the install recorded it, and Python 3.11 lists such a file too, where later versions' Distribution.files drop it + install(tmp_path, "Name: alpha\nVersion: 1.0\n", {"alpha.py": b"x = 1\n"}) + (tmp_path / "alpha.py").unlink() + assert [row.path for row in facts_from(tmp_path).distributions[0].files] == [ + "alpha.py", + "alpha-1.0.dist-info/METADATA", + "alpha-1.0.dist-info/WHEEL", + ] + + +@pytest.mark.parametrize( + ("row", "issue"), + [ + pytest.param("alpha.py,sha256=x", "expected 3 elements, got 2", id="two-columns"), + pytest.param("alpha.py,sha256=x,big", "`size` cannot be non-integer", id="size-not-integer"), + pytest.param("alpha.py,nohash,1", "`hash` does not follow the required format", id="hash-without-name"), + ], +) +def test_malformed_record_lists_no_files(tmp_path: Path, install: Install, row: str, issue: str) -> None: + install(tmp_path, "Name: alpha\nVersion: 1.0\n", {}) + (tmp_path / "alpha-1.0.dist-info" / "RECORD").write_bytes(f"{row}\n".encode()) + with pytest.warns(UserWarning, match=f"listed no files of alpha, its RECORD is malformed: .*{re.escape(issue)}"): + assert facts_from(tmp_path).distributions[0].files == [] + + @pytest.mark.parametrize( "files", [ From ff8e0fc9f97c89f63451feefec5dcd0e99b9e4f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:39:55 -0800 Subject: [PATCH 031/115] =?UTF-8?q?=E2=9A=A1=20perf:=20copy=20wheel=20memb?= =?UTF-8?q?ers=20without=20recompressing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_sbom rewrote every member through zlib. zipremove, the backport of the ZipFile.remove and repack that CPython 3.16 adds, drops the old RECORD and SBOM in place, so the other members keep their bytes as the backend wrote them and only the SBOM and RECORD get compressed. --- pylock.toml | 25 +++++++++++++++++++++ pyproject.toml | 2 ++ src/python_sbom/_wheel/_archive.py | 36 +++++++++++++++--------------- tests/_wheel/test_archive.py | 22 ++++++++++++++++++ 4 files changed, 67 insertions(+), 18 deletions(-) diff --git a/pylock.toml b/pylock.toml index bf48fd3..b04a1ac 100644 --- a/pylock.toml +++ b/pylock.toml @@ -3571,6 +3571,31 @@ size = 13580 [packages.wheels.hashes] sha256 = "a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a" +[[packages]] +name = "zipremove" +version = "0.10.0" +marker = "\"project-runtime\" in dependency_groups" +requires-python = "~=3.7" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "zipremove-0.10.0.tar.gz" +upload-time = 2026-07-17 15:50:45.516180+00:00 +url = "https://files.pythonhosted.org/packages/92/30/47d65b9213f52df8a51f603cb0a2e5ef02c22172a8a17805e46c3beabb0d/zipremove-0.10.0.tar.gz" +size = 25448 + +[packages.sdist.hashes] +sha256 = "3c5957c115447b54299d3a23128764b59c4558fc98c4b878ff2dbc5db8a62021" + +[[packages.wheels]] +name = "zipremove-0.10.0-py3-none-any.whl" +upload-time = 2026-07-17 15:50:44.653422+00:00 +url = "https://files.pythonhosted.org/packages/0a/e4/2225c1890ce6e239ca23efabb80a3481229a14a2deaa489f5e89723a5aa6/zipremove-0.10.0-py3-none-any.whl" +size = 11280 + +[packages.wheels.hashes] +sha256 = "04e15df7083642d5b791e51f597cffef8a92d770ecf5160c529b5a7b53b40100" + [tool.nab] nab-version = "0.0.23" created-at = 2026-11-07 05:12:05.951651+00:00 diff --git a/pyproject.toml b/pyproject.toml index 9ec74df..33c67a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ requires = [ "hatchling>=1.32.4", "installer>=1.1", "packageurl-python>=0.11.1", + "zipremove>=0.10", ] [project] @@ -51,6 +52,7 @@ dependencies = [ "installer>=1.1", "packageurl-python>=0.11.1", "packaging>=26.3", + "zipremove>=0.10", ] optional-dependencies.hatch = [ "hatchling>=1.28", diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index c1bf79b..53aec5d 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -3,6 +3,7 @@ import csv import hashlib import os +import shutil import stat import tempfile import time @@ -14,6 +15,7 @@ from pathlib import Path from typing import IO, TYPE_CHECKING, Final +import zipremove from installer.records import Hash, RecordEntry from installer.sources import WheelFile from installer.utils import copyfileobj_with_hashing, parse_metadata_file @@ -32,33 +34,31 @@ def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: with open_archive(wheel, str(wheel)) as archive: _, dist_info = _wheel_source(archive, str(wheel)) - entries: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [ - (info, archive.read(info)) for info in archive.infolist() - ] - record: Final[str] = f"{dist_info}/RECORD" - if missing := sorted({f"{dist_info}/METADATA", record} - {info.filename for info, _ in entries}): - msg = f"{wheel} has no {', '.join(missing)}" - raise WheelError(msg) + record: Final[str] = f"{dist_info}/RECORD" + if missing := sorted({f"{dist_info}/METADATA", record} - set(archive.namelist())): + msg = f"{wheel} has no {', '.join(missing)}" + raise WheelError(msg) + record_info: Final[zipfile.ZipInfo] = archive.getinfo(record) + record_data: Final[bytes] = archive.read(record_info) sbom: Final[str] = f"{dist_info}/sboms/{name}" # zip timestamps start at 1980 entry = zipfile.ZipInfo(sbom, time.gmtime(max(epoch, 315532800))[:6]) entry.compress_type = zipfile.ZIP_DEFLATED entry.create_system = 3 # Unix, whichever system writes the entry, so the bytes match across operating systems entry.external_attr = (stat.S_IFREG | 0o644) << 16 - # the SBOM goes right before RECORD, which stays the last entry - members: Final[list[tuple[zipfile.ZipInfo, bytes]]] = [] - for info, data in entries: - if info.filename == record: - members.extend([(entry, content), (info, _rewritten_record(data, record, sbom, content))]) - elif info.filename != sbom: - members.append((info, data)) + rewritten: Final[bytes] = _rewritten_record(record_data, record, sbom, content) # a name no "dist/*.whl" glob matches, in the wheel's directory so the final rename stays on one file system descriptor, temporary = tempfile.mkstemp(dir=wheel.parent, prefix=".python-sbom-", suffix=".tmp") os.close(descriptor) - try: - with zipfile.ZipFile(temporary, "w") as archive: - for info, data in members: - archive.writestr(info, data) + try: # ruff: ignore[too-many-statements-in-try-clause] # whichever step fails, the temporary file goes + shutil.copyfile(wheel, temporary) + # the other members keep their bytes as stored, still compressed, so only the SBOM and RECORD get compressed + # here; both go last, RECORD after the SBOM + with zipremove.ZipFile(temporary, "a") as archive: + stale = [info for info in archive.infolist() if info.filename in {record, sbom}] + archive.repack([archive.remove(info) for info in stale]) # ty: ignore[unresolved-attribute] # zipremove adds both + archive.writestr(entry, content) + archive.writestr(record_info, rewritten) Path(temporary).chmod(stat.S_IMODE(wheel.stat().st_mode)) Path(temporary).replace(wheel) except BaseException: diff --git a/tests/_wheel/test_archive.py b/tests/_wheel/test_archive.py index 15f3ef0..24854c3 100644 --- a/tests/_wheel/test_archive.py +++ b/tests/_wheel/test_archive.py @@ -71,6 +71,28 @@ def test_embed_keeps_other_entries(wheel: Path) -> None: assert {name: after[name] for name in before} == before +def test_embed_keeps_other_members_compressed_as_they_were(tmp_path: Path) -> None: + # level 9 over repetitive text gives other bytes than zipfile's default level, so a recompressed member would differ + with zipfile.ZipFile(wheel := tmp_path / "native-1.0-py3-none-any.whl", "w", compresslevel=9) as handle: + handle.writestr( + "native/data.txt", b"".join(b"line %d\n" % number for number in range(5000)), zipfile.ZIP_DEFLATED + ) + handle.writestr("native-1.0.dist-info/METADATA", b"Name: native\nVersion: 1.0\n") + handle.writestr("native-1.0.dist-info/RECORD", b"native-1.0.dist-info/RECORD,,\n") + before = _stored(wheel, "native/data.txt") + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert _stored(wheel, "native/data.txt") == before + + +def _stored(wheel: Path, name: str) -> bytes: + with zipfile.ZipFile(wheel) as archive: + info = archive.getinfo(name) + data = wheel.read_bytes() + name_length, extra_length = struct.unpack(" Date: Sat, 7 Nov 2026 00:39:55 -0800 Subject: [PATCH 032/115] =?UTF-8?q?=F0=9F=93=9D=20docs:=20say=20why=20the?= =?UTF-8?q?=20CycloneDX=20model=20is=20our=20own?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/python_sbom/_model/_types.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/python_sbom/_model/_types.py b/src/python_sbom/_model/_types.py index 0fcdfd5..6c154d4 100644 --- a/src/python_sbom/_model/_types.py +++ b/src/python_sbom/_model/_types.py @@ -2,6 +2,10 @@ from typing import NotRequired, TypedDict +# the document model is our own: cyclonedx-python-lib models neither formulation nor compositions (its model/bom.py +# holds only commented-out stubs, tracked upstream in CycloneDX/cyclonedx-python-lib#578 and #581), and its SortedSet +# collections re-order lists whose order the document keeps on purpose + class OrganizationalEntity(TypedDict): """ From f986ea0c0cd44a862e05dd91e1ef18264451a512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:57:03 -0800 Subject: [PATCH 033/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20parse?= =?UTF-8?q?=20core=20metadata=20with=20packaging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wheel, bundled, vendored and build-environment metadata now goes through packaging.metadata.parse_email, unvalidated so spec-breaking metadata is still described. A field packaging cannot take, such as a repeated label or single-use field, is read from unparsed as written. The build environment parses each METADATA once instead of twice. Across 7,620 distinct METADATA files the components, licenses and project identity match the old output except one: Keywords come back joined by commas, so spacing around them is no longer kept. A Project-URL without a URL is left out. --- docs/conf.py | 2 - src/python_sbom/_collect/_facts.py | 25 +++--- src/python_sbom/_collect/_source.py | 21 +++-- src/python_sbom/_generate/_archive.py | 10 +-- src/python_sbom/_generate/_pipeline.py | 2 +- src/python_sbom/_model/_bom.py | 10 +-- src/python_sbom/_model/_components.py | 39 +++++----- src/python_sbom/_model/_identity.py | 11 ++- src/python_sbom/_model/_inputs.py | 11 ++- src/python_sbom/_model/_licenses.py | 14 ++-- src/python_sbom/_model/_metadata.py | 90 +++++++++++++++++++++ src/python_sbom/_model/_names.py | 14 +--- src/python_sbom/_model/_tools.py | 2 +- src/python_sbom/_wheel/_archive.py | 8 +- tests/_collect/test_facts.py | 8 +- tests/_collect/test_source.py | 14 ++-- tests/_generate/test_pipeline.py | 4 +- tests/_model/test_bom.py | 8 +- tests/_model/test_components.py | 34 ++++---- tests/_model/test_identity.py | 4 +- tests/_model/test_licenses.py | 4 +- tests/_model/test_metadata.py | 103 +++++++++++++++++++++++++ tests/_model/test_tools.py | 6 +- tests/conftest.py | 6 +- tests/test_public_api.py | 5 +- 25 files changed, 314 insertions(+), 141 deletions(-) create mode 100644 src/python_sbom/_model/_metadata.py create mode 100644 tests/_model/test_metadata.py diff --git a/docs/conf.py b/docs/conf.py index afcceed..f2de70c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -77,8 +77,6 @@ def _inheritance_aliases() -> dict[str, str]: ("py:class", "hatchling.builders.hooks.plugin.interface.BuildHookInterface"), ("py:class", "hatchling.metadata.core.ProjectMetadata"), ("py:class", "hatchling.bridge.app.Application"), - # the Python documentation describes the protocol without a reference target - ("py:class", "importlib.metadata._meta.PackageMetadata"), ] diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index c34bf5f..4d9b316 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -21,13 +21,16 @@ from python_sbom._errors import SettingsError from python_sbom._model._inputs import BuildFacts, InstalledDistribution, Interpreter, RecordRow, Revision +from python_sbom._model._metadata import parse_metadata from python_sbom._wheel._record import decode_digest if TYPE_CHECKING: from collections.abc import Container, Sequence - from importlib.metadata import Distribution, PackageMetadata + from importlib.metadata import Distribution from pathlib import Path + from python_sbom._model._metadata import CoreMetadata + #: the file an sdist carries its source revision in, as ``node: ``, for the wheel built from it REVISION_FILE: Final[str] = ".python-sbom-revision" _COMMIT: Final[re.Pattern[str]] = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}") @@ -62,22 +65,19 @@ def collect_facts( def _installed(path: Sequence[str] | None) -> list[InstalledDistribution]: - found: Final[dict[str, tuple[Distribution, PackageMetadata]]] = {} + found: Final[dict[str, tuple[Distribution, CoreMetadata]]] = {} for distribution in distributions(path=list(path)) if path is not None else distributions(): - try: - metadata = distribution.metadata - # a .dist-info without METADATA, which newer Pythons refuse to read - except FileNotFoundError: # pragma: >=3.15 cover - continue + # where Distribution.metadata looks, a .dist-info's METADATA, then an .egg-info's PKG-INFO or the file itself + text = distribution.read_text("METADATA") or distribution.read_text("PKG-INFO") or distribution.read_text("") # the first on the path is the one imports load, so a shadowed copy further down is left out - if "Name" in metadata and "Version" in metadata: - found.setdefault(canonicalize_name(metadata["Name"]), (distribution, metadata)) + if (metadata := parse_metadata(text or "")).name and metadata.version: + found.setdefault(canonicalize_name(metadata.name), (distribution, metadata)) return [ InstalledDistribution( name=name, metadata=metadata, files=_recorded_files(distribution), - depends_on=_depends_on(name, distribution, found), + depends_on=_depends_on(name, distribution, metadata, found), ) for name, (distribution, metadata) in found.items() ] @@ -131,9 +131,10 @@ def _describes_distribution(path: str) -> bool: ) -def _depends_on(name: str, distribution: Distribution, installed: Container[str]) -> list[str]: +def _depends_on(name: str, distribution: Distribution, metadata: CoreMetadata, installed: Container[str]) -> list[str]: names: Final[set[str]] = set() - for line in distribution.requires or []: + # an .egg-info states its requirements in requires.txt, which Distribution.requires reads + for line in metadata.values("requires_dist") or distribution.requires or []: try: requirement = Requirement(line) except InvalidRequirement as error: diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index c7a0b81..d4a49e4 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -9,17 +9,16 @@ from installer.records import InvalidRecordEntry, RecordEntry, parse_record_file from installer.sources import WheelFile -from installer.utils import parse_metadata_file from packaging.requirements import InvalidRequirement, Requirement from python_sbom._errors import SettingsError, WheelError from python_sbom._model._inputs import BuildScript, BundledWheel, RecordRow, SbomRequest, SourceTree +from python_sbom._model._metadata import CoreMetadata, parse_metadata from python_sbom._wheel._archive import open_archive, read_member, read_wheel_metadata from python_sbom._wheel._record import decode_digest if TYPE_CHECKING: import zipfile - from email.message import Message from pathlib import Path, PurePosixPath from python_sbom._model._inputs import Settings @@ -36,7 +35,7 @@ def read_source_tree(request: SbomRequest) -> SourceTree: copyright=next( ( line - for license_file in request.metadata.get_all("License-File", []) + for license_file in request.metadata.values("license_files") if (path := root / license_file).is_file() for line in path.read_text(encoding="utf-8").splitlines() if line.startswith("Copyright") @@ -69,7 +68,7 @@ def read_bundled_wheel(data: bytes, path: str) -> BundledWheel: dist_info: Final[str] = wheel.dist_info_dir _check_named(metadata, where, f"{dist_info}/METADATA") files: Final[list[RecordRow]] = _verified_record(archive, wheel, where) - vendored: Final[list[tuple[str, Message]]] = [ + vendored: Final[list[tuple[str, CoreMetadata]]] = [ *_vendor_manifest(archive, where, members), *( (name, _vendored_metadata(archive, where, name)) @@ -87,8 +86,8 @@ def read_bundled_wheel(data: bytes, path: str) -> BundledWheel: ) -def _check_named(metadata: Message, where: str, name: str) -> None: - if not (metadata["Name"] and metadata["Version"]): +def _check_named(metadata: CoreMetadata, where: str, name: str) -> None: + if not (metadata.name and metadata.version): msg = f"{where}: {name} lacks Name or Version" raise WheelError(msg) @@ -137,9 +136,9 @@ def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> return found -def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) -> list[tuple[str, Message]]: +def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) -> list[tuple[str, CoreMetadata]]: # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info - found: Final[list[tuple[str, Message]]] = [] + found: Final[list[tuple[str, CoreMetadata]]] = [] for name in (member for member in members if member.endswith("/_vendor/vendor.txt")): for line in read_member(archive, name, where).decode("utf-8").splitlines(): if not (text := line.partition("#")[0].strip()): @@ -152,12 +151,12 @@ def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) - if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: msg = f"{where}: vendored requirement in {name} pins no one exact version: {text}" raise WheelError(msg) - found.append((name, parse_metadata_file(f"Name: {requirement.name}\nVersion: {pins[0].version}\n"))) + found.append((name, CoreMetadata(raw={"name": requirement.name, "version": pins[0].version}, unparsed={}))) return found -def _vendored_metadata(archive: zipfile.ZipFile, where: str, name: str) -> Message: - metadata: Final[Message] = parse_metadata_file(read_member(archive, name, where).decode("utf-8")) +def _vendored_metadata(archive: zipfile.ZipFile, where: str, name: str) -> CoreMetadata: + metadata: Final[CoreMetadata] = parse_metadata(read_member(archive, name, where).decode("utf-8")) _check_named(metadata, where, name) return metadata diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index a4fad4c..6b28e6c 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -6,7 +6,6 @@ from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Final -from installer.utils import parse_metadata_file from packageurl import PackageURL from python_sbom._collect._content import load_content_provider, provider_script @@ -23,6 +22,7 @@ from python_sbom._model._content import ProviderContext from python_sbom._model._document import contributed_properties, formulation, timestamp, with_header from python_sbom._model._identity import project_identity +from python_sbom._model._metadata import parse_metadata from python_sbom._model._names import SBOM_FILE, sbom_property from python_sbom._model._tools import tool_components from python_sbom._model._types import Dependency @@ -30,11 +30,11 @@ if TYPE_CHECKING: from collections.abc import Sequence - from email.message import Message from python_sbom._model._content import ContentProvider from python_sbom._model._identity import ProjectIdentity from python_sbom._model._inputs import BuildFacts, Settings + from python_sbom._model._metadata import CoreMetadata from python_sbom._model._types import Component, Document, Metadata @@ -115,7 +115,7 @@ def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, handle.extractall(unpacked) # each distribution by the directory that holds its .dist-info, the project's own at the top level metadata = { - f"{name.rsplit('/', 2)[0]}/" if name.count("/") > 1 else "": parse_metadata_file( + f"{name.rsplit('/', 2)[0]}/" if name.count("/") > 1 else "": parse_metadata( (unpacked / name).read_text(encoding="utf-8") ) for name in members @@ -142,7 +142,7 @@ def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, content=content, context=ProviderContext(root=root, settings=settings, identity=project, artifact=unpacked), ) - version = metadata[""]["Version"] + version = metadata[""].version ref = PackageURL(type="generic", name=archive.name, version=version).to_string() return _Held( reader=reader, @@ -168,7 +168,7 @@ class _Reader: content: ContentProvider context: ProviderContext - def distribution(self, prefix: str, metadata: Message, names: list[str]) -> Component: + def distribution(self, prefix: str, metadata: CoreMetadata, names: list[str]) -> Component: component = component_from_metadata(metadata) if prefix: # a wheel bundled unpacked: the directory tells copies of one release apart, as the path does for a .whl diff --git a/src/python_sbom/_generate/_pipeline.py b/src/python_sbom/_generate/_pipeline.py index b722441..9a751ef 100644 --- a/src/python_sbom/_generate/_pipeline.py +++ b/src/python_sbom/_generate/_pipeline.py @@ -18,7 +18,7 @@ def build_sbom( request: SbomRequest, source: SourceTree, facts: BuildFacts, content: ContentProvider, context: ProviderContext ) -> Sbom: return Sbom( - file_name=sbom_file_name(request.metadata["Name"]), + file_name=sbom_file_name(request.metadata.name), document=describe( request, source, diff --git a/src/python_sbom/_model/_bom.py b/src/python_sbom/_model/_bom.py index 01beebd..530168b 100644 --- a/src/python_sbom/_model/_bom.py +++ b/src/python_sbom/_model/_bom.py @@ -44,7 +44,7 @@ def describe( # hatchling repeats a Requires-Dist when a hook re-adds a dependency, and each bom-ref must be unique *( _declared_dependency(text, requirement, project.property_prefix) - for text in dict.fromkeys(request.metadata.get_all("Requires-Dist", [])) + for text in dict.fromkeys(request.metadata.values("requires_dist")) if applies_without_extras(requirement := Requirement(text)) ), *contributions.components, @@ -55,7 +55,7 @@ def describe( script_tool( script.path, script.sha256, - request.metadata["Version"], + request.metadata.version, _file_link(project, request.settings, str(script.path)), ) for script in source.build_scripts @@ -85,7 +85,7 @@ def _root_component( request: SbomRequest, source: SourceTree, facts: BuildFacts, project: ProjectIdentity, contributions: Contributions ) -> Component: metadata, settings = request.metadata, request.settings - name, version = metadata["Name"], metadata["Version"] + name, version = metadata.name, metadata.version root = component_from_metadata(metadata) root["type"] = "application" if settings.supplier is not None: @@ -102,7 +102,7 @@ def _root_component( {"type": "security-contact", "url": f"{project.repository}/security/policy"}, {"type": "advisories", "url": f"{project.repository}/security/advisories"}, ]) - if (license_files := metadata.get_all("License-File", [])) and ( + if (license_files := metadata.values("license_files")) and ( link := _file_link(project, settings, license_files[0]) ): references.append({"type": "license", "url": link}) @@ -163,7 +163,7 @@ def _metadata( metadata["licenses"] = root["licenses"] properties: Final[list[Property]] = [ {"name": sbom_property(project.property_prefix, "sbom:scope"), "value": _scope(settings)}, - {"name": sbom_property(project.property_prefix, SBOM_FILE), "value": sbom_file_name(request.metadata["Name"])}, + {"name": sbom_property(project.property_prefix, SBOM_FILE), "value": sbom_file_name(request.metadata.name)}, ] if project.repository is not None: properties.append({ diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index aa973cf..ae8e465 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -9,15 +9,14 @@ from ._inputs import RecordRow from ._licenses import licenses -from ._names import project_urls, property_prefix, purl, pypi_url, reference_type, tool_ref +from ._names import property_prefix, purl, pypi_url, reference_type, tool_ref from ._types import Dependency if TYPE_CHECKING: - from email.message import Message - from importlib.metadata import PackageMetadata from pathlib import PurePosixPath from ._inputs import BundledWheel + from ._metadata import CoreMetadata from ._types import Component, Contact, ExternalReference, Identity, OrganizationalEntity, Property @@ -32,7 +31,7 @@ def bundled_wheel_component( component["hashes"] = [{"alg": "SHA-256", "content": wheel.sha256}] component["externalReferences"] = [ *component["externalReferences"], - {"type": "distribution", "url": pypi_url(wheel.metadata["Name"], wheel.metadata["Version"])}, + {"type": "distribution", "url": pypi_url(wheel.metadata.name, wheel.metadata.version)}, ] component["properties"] = [ {"name": f"{namespace}:bundled-wheel", "value": wheel.path}, @@ -61,19 +60,17 @@ def bundled_wheel_component( return component -def component_from_metadata(metadata: PackageMetadata | Message) -> Component: - name, version = metadata["Name"], metadata["Version"] +def component_from_metadata(metadata: CoreMetadata) -> Component: + name, version = metadata.name, metadata.version component: Final[Component] = {"type": "library", "bom-ref": purl(name, version), "name": name, "version": version} component["purl"] = component["bom-ref"] - if summary := metadata.get("Summary"): + if summary := metadata.first("summary"): component["description"] = summary # maintainers are who users reach today, so they win over the original authors when both are named - for names, addresses in (("Maintainer", "Maintainer-email"), ("Author", "Author-email")): - named = (_contact(name, "") for name in metadata.get_all(names, [])) + for names, addresses in (("maintainer", "maintainer_email"), ("author", "author_email")): + named = (_contact(name, "") for name in metadata.values(names)) if people := [ - contact - for contact in (*named, *starmap(_contact, getaddresses(metadata.get_all(addresses, [])))) - if contact + contact for contact in (*named, *starmap(_contact, getaddresses(metadata.values(addresses)))) if contact ]: component["authors"] = people break @@ -82,9 +79,15 @@ def component_from_metadata(metadata: PackageMetadata | Message) -> Component: component["externalReferences"] = _project_references(metadata) # core metadata headers copied verbatim, so a reader sees what the distribution declared component["properties"] = [ - {"name": f"python:{header.lower()}", "value": value} - for header in ("Requires-Python", "Requires-Dist", "Provides-Extra", "Classifier", "Keywords") - for value in metadata.get_all(header, []) + {"name": f"python:{header}", "value": value} + for header, values in ( + ("requires-python", metadata.values("requires_python")), + ("requires-dist", metadata.values("requires_dist")), + ("provides-extra", metadata.values("provides_extra")), + ("classifier", metadata.values("classifiers")), + ("keywords", metadata.keywords()), + ) + for value in values ] return component @@ -101,14 +104,14 @@ def _contact(name: str, email: str) -> Contact: return contact -def _project_references(metadata: PackageMetadata | Message) -> list[ExternalReference]: +def _project_references(metadata: CoreMetadata) -> list[ExternalReference]: # a home page that is not an http(s) URL, such as UNKNOWN, is left out references: Final[list[ExternalReference]] = [] - if (home_page := metadata.get("Home-page")) and urlsplit(home_page).scheme in {"http", "https"}: + if (home_page := metadata.first("home_page")) and urlsplit(home_page).scheme in {"http", "https"}: references.append({"type": "website", "url": home_page}) references.extend( {"type": reference_type(label), "url": url, "comment": f"Project-URL: {label}"} - for label, url in project_urls(metadata) + for label, url in metadata.project_urls() ) return references diff --git a/src/python_sbom/_model/_identity.py b/src/python_sbom/_model/_identity.py index 064d292..c470c8f 100644 --- a/src/python_sbom/_model/_identity.py +++ b/src/python_sbom/_model/_identity.py @@ -4,18 +4,17 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from ._names import project_urls, property_prefix, reference_type +from ._names import property_prefix, reference_type if TYPE_CHECKING: - from email.message import Message - from ._inputs import Settings + from ._metadata import CoreMetadata -def project_identity(metadata: Message, settings: Settings) -> ProjectIdentity: - project = property_prefix(metadata["Name"]) +def project_identity(metadata: CoreMetadata, settings: Settings) -> ProjectIdentity: + project = property_prefix(metadata.name) repository = settings.repository or next( - (url.rstrip("/") for label, url in project_urls(metadata) if reference_type(label) == "vcs"), None + (url.rstrip("/") for label, url in metadata.project_urls() if reference_type(label) == "vcs"), None ) base = repository or f"https://pypi.org/project/{project}" return ProjectIdentity( diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py index 9b10f4f..42af1ae 100644 --- a/src/python_sbom/_model/_inputs.py +++ b/src/python_sbom/_model/_inputs.py @@ -4,10 +4,9 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from email.message import Message - from importlib.metadata import PackageMetadata from pathlib import Path, PurePosixPath + from ._metadata import CoreMetadata from ._types import Component, Dependency, OrganizationalEntity, Property @@ -29,7 +28,7 @@ class Interpreter: @dataclass(frozen=True, slots=True, kw_only=True) class InstalledDistribution: name: str - metadata: PackageMetadata | Message + metadata: CoreMetadata files: list[RecordRow] depends_on: list[str] @@ -73,7 +72,7 @@ class Settings: @dataclass(frozen=True, slots=True, kw_only=True) class SbomRequest: root: Path - metadata: Message + metadata: CoreMetadata wheel_name: str settings: Settings build_scripts: tuple[PurePosixPath, ...] @@ -84,9 +83,9 @@ class BundledWheel: path: str sha256: str metadata_name: str - metadata: Message + metadata: CoreMetadata files: list[RecordRow] - vendored: list[tuple[str, Message]] + vendored: list[tuple[str, CoreMetadata]] @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/src/python_sbom/_model/_licenses.py b/src/python_sbom/_model/_licenses.py index 22877d8..17ff8a1 100644 --- a/src/python_sbom/_model/_licenses.py +++ b/src/python_sbom/_model/_licenses.py @@ -6,25 +6,21 @@ from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression if TYPE_CHECKING: - from email.message import Message - from importlib.metadata import PackageMetadata - + from ._metadata import CoreMetadata from ._types import License -def licenses(metadata: PackageMetadata | Message) -> list[License]: +def licenses(metadata: CoreMetadata) -> list[License]: # a License-Expression is taken as declared; otherwise each License field and License classifier name is # declared, and the SPDX ids they map to are added as concluded - if expression := metadata.get("License-Expression"): + if expression := metadata.first("license_expression"): try: return [{"expression": canonicalize_license_expression(expression), "acknowledgement": "declared"}] except InvalidLicenseExpression: # an expression SPDX rejects, such as "MIT WITH GPL-3.0-only", stays as the name the metadata gives return [{"license": {"name": expression, "acknowledgement": "declared"}}] - names = [ - entry.rsplit(" :: ", 1)[-1] for entry in metadata.get_all("Classifier", []) if entry.startswith("License :: ") - ] - if (declared := metadata.get("License")) and "\n" not in declared: + names = [entry.rsplit(" :: ", 1)[-1] for entry in metadata.values("classifiers") if entry.startswith("License :: ")] + if (declared := metadata.first("license")) and "\n" not in declared: names.append(declared) found: Final[list[License]] = [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] # a license policy judges SPDX ids only, so the names stay bare when any one of them does not resolve, and the diff --git a/src/python_sbom/_model/_metadata.py b/src/python_sbom/_model/_metadata.py new file mode 100644 index 0000000..33edd61 --- /dev/null +++ b/src/python_sbom/_model/_metadata.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final, Literal, TypeAlias + +from packaging.metadata import parse_email + +if TYPE_CHECKING: + from packaging.metadata import RawMetadata + +# the fields python-sbom reads, by their key in packaging's RawMetadata +Field: TypeAlias = Literal[ + "author", + "author_email", + "classifiers", + "home_page", + "license", + "license_expression", + "license_files", + "maintainer", + "maintainer_email", + "name", + "provides_extra", + "requires_dist", + "requires_python", + "summary", + "version", +] + +# unparsed keys a field by its header name, lowercased, which is the RawMetadata key with dashes except for these +_HEADERS: Final[dict[str, str]] = {"classifiers": "classifier", "license_files": "license-file"} + + +def parse_metadata(text: str) -> CoreMetadata: + # not validated: the SBOM describes a distribution whose metadata breaks the specification as well, as written + raw, unparsed = parse_email(text) + return CoreMetadata(raw=raw, unparsed=unparsed) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class CoreMetadata: + raw: RawMetadata + unparsed: dict[str, list[str]] + + @property + def name(self) -> str: + return self.first("name") or "" + + def first(self, field: Field) -> str | None: + return next(iter(self.values(field)), None) + + def values(self, field: Field) -> list[str]: + # packaging keeps a field it cannot take as declared, such as a single-use field given twice, out of raw and + # its values as written in unparsed, from where they are still described + match self.raw.get(field): + case str() as value: + return [value] + case list() as values: + return values + return self.unparsed.get(_HEADERS.get(field, field.replace("_", "-")), []) + + @property + def version(self) -> str: + return self.first("version") or "" + + def keywords(self) -> list[str]: + # packaging splits Keywords at commas, so they are joined back into the one value the distribution declared + if "keywords" in self.raw: + return [",".join(self.raw["keywords"])] + return self.unparsed.get("keywords", []) + + def project_urls(self) -> list[tuple[str, str]]: + # a label given twice sends every Project-URL to unparsed, where each splits at its first comma as packaging + # splits them; an entry without a URL names nothing to reference + pairs: Final[list[tuple[str, str]]] = ( + list(self.raw["project_urls"].items()) + if "project_urls" in self.raw + else [ + (label.strip(), url.strip()) + for label, _, url in (entry.partition(",") for entry in self.unparsed.get("project-url", [])) + ] + ) + return [(label, url) for label, url in pairs if url] + + +__all__ = [ + "CoreMetadata", + "Field", + "parse_metadata", +] diff --git a/src/python_sbom/_model/_names.py b/src/python_sbom/_model/_names.py index 53effac..81c4374 100644 --- a/src/python_sbom/_model/_names.py +++ b/src/python_sbom/_model/_names.py @@ -1,15 +1,11 @@ from __future__ import annotations import re -from typing import TYPE_CHECKING, Final +from typing import Final from packageurl import PackageURL from packaging.utils import canonicalize_name -if TYPE_CHECKING: - from email.message import Message - from importlib.metadata import PackageMetadata - #: the metadata property in which a document names its own file; validation checks such documents in full SBOM_FILE: Final[str] = "sbom:file" #: the metadata property in which a document names the project's repository @@ -73,17 +69,9 @@ def reference_type(label: str) -> str: return _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") -def project_urls(metadata: PackageMetadata | Message) -> list[tuple[str, str]]: - return [ - (label.strip(), url.strip()) - for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", []) if "," in entry) - ] - - __all__ = [ "SBOM_FILE", "SBOM_REPOSITORY", - "project_urls", "property_prefix", "purl", "pypi_url", diff --git a/src/python_sbom/_model/_tools.py b/src/python_sbom/_model/_tools.py index 12abb22..870d6a4 100644 --- a/src/python_sbom/_model/_tools.py +++ b/src/python_sbom/_model/_tools.py @@ -26,7 +26,7 @@ def tool_components(facts: BuildFacts) -> tuple[list[Component], list[Dependency } ] refs: Final[dict[str, str]] = { - distribution.name: tool_ref(purl(distribution.name, distribution.metadata["Version"])) + distribution.name: tool_ref(purl(distribution.name, distribution.metadata.version)) for distribution in facts.distributions } dependencies: Final[list[Dependency]] = [] diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index 53aec5d..791b4d8 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -18,13 +18,13 @@ import zipremove from installer.records import Hash, RecordEntry from installer.sources import WheelFile -from installer.utils import copyfileobj_with_hashing, parse_metadata_file +from installer.utils import copyfileobj_with_hashing from python_sbom._errors import WheelError +from python_sbom._model._metadata import CoreMetadata, parse_metadata if TYPE_CHECKING: from collections.abc import Generator - from email.message import Message # metadata files beyond this size are refused, so a hostile wheel cannot exhaust memory _METADATA_LIMIT: Final[int] = 16 * 1024 * 1024 @@ -113,10 +113,10 @@ def _rewritten_record(record: bytes, name: str, sbom: str, content: bytes) -> by return "".join(lines).encode() -def read_wheel_metadata(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, Message]: +def read_wheel_metadata(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, CoreMetadata]: wheel, dist_info = _wheel_source(archive, where) # read capped, as installer's read_dist_info decompresses the member whole - return wheel, parse_metadata_file(read_member(archive, f"{dist_info}/METADATA", where).decode("utf-8")) + return wheel, parse_metadata(read_member(archive, f"{dist_info}/METADATA", where).decode("utf-8")) def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _METADATA_LIMIT) -> bytes: diff --git a/tests/_collect/test_facts.py b/tests/_collect/test_facts.py index 94ab550..a4b29e2 100644 --- a/tests/_collect/test_facts.py +++ b/tests/_collect/test_facts.py @@ -73,7 +73,7 @@ def reader(root: Path, /) -> Revision: def test_default_path_is_this_interpreter(tmp_path: Path) -> None: names = { - distribution.metadata["Name"] + distribution.metadata.name for distribution in collect_facts(tmp_path, no_revision, default_epoch=0).distributions } assert "pytest" in names @@ -163,7 +163,7 @@ def test_first_distribution_on_path_wins(tmp_path: Path, install: Install) -> No (distribution,) = collect_facts( tmp_path, no_revision, path=[str(first), str(second)], default_epoch=0 ).distributions - assert distribution.metadata["Version"] == "1.0" + assert distribution.metadata.version == "1.0" def test_depends_on_installed_requirements_that_apply(tmp_path: Path, install: Install) -> None: @@ -175,7 +175,7 @@ def test_depends_on_installed_requirements_that_apply(tmp_path: Path, install: I ) for name in ("beta", "gamma", "delta"): install(tmp_path, f"Name: {name}\nVersion: 1.0\n", {}) - by_name = {distribution.metadata["Name"]: distribution for distribution in facts_from(tmp_path).distributions} + by_name = {distribution.metadata.name: distribution for distribution in facts_from(tmp_path).distributions} assert by_name["alpha"].depends_on == ["beta"] @@ -183,7 +183,7 @@ def test_egg_info_distribution(tmp_path: Path) -> None: (egg_info := tmp_path / "legacy-1.0-py3.14.egg-info").mkdir() (egg_info / "PKG-INFO").write_text("Metadata-Version: 2.1\nName: Legacy\nVersion: 1.0\n", encoding="utf-8") (distribution,) = facts_from(tmp_path).distributions - assert (distribution.name, distribution.metadata["Version"], distribution.files) == ("legacy", "1.0", []) + assert (distribution.name, distribution.metadata.version, distribution.files) == ("legacy", "1.0", []) def test_recorded_file_missing_on_disk_is_listed(tmp_path: Path, install: Install) -> None: diff --git a/tests/_collect/test_source.py b/tests/_collect/test_source.py index e7f3871..84cc09a 100644 --- a/tests/_collect/test_source.py +++ b/tests/_collect/test_source.py @@ -5,7 +5,6 @@ import hashlib import re import zipfile -from email import message_from_string from itertools import starmap from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Final @@ -15,6 +14,7 @@ from python_sbom._collect._source import read_bundled_wheel, read_source_tree, wheel_request from python_sbom._errors import SettingsError, WheelError from python_sbom._model._inputs import BuildScript, RecordRow, SbomRequest +from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: from collections.abc import Mapping @@ -60,7 +60,7 @@ def test_copyright(tmp_path: Path, settings: Settings, files: Mapping[str, str], def request(root: Path, settings: Settings, metadata: str = "", scripts: tuple[PurePosixPath, ...] = ()) -> SbomRequest: return SbomRequest( root=root, - metadata=message_from_string(f"Name: example\nVersion: 1.0\n{metadata}"), + metadata=parse_metadata(f"Name: example\nVersion: 1.0\n{metadata}"), wheel_name="example-1.0-py3-none-any.whl", settings=settings, build_scripts=scripts, @@ -119,16 +119,16 @@ def test_bundled_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: }, ) bundled = read_bundled_wheel(wheel.read_bytes(), "src/pip-1.0-py3-none-any.whl") - assert (bundled.path, bundled.sha256, bundled.metadata_name, bundled.metadata["Name"], bundled.files[0]) == ( + assert (bundled.path, bundled.sha256, bundled.metadata_name, bundled.metadata.name, bundled.files[0]) == ( "src/pip-1.0-py3-none-any.whl", hashlib.sha256(wheel.read_bytes()).hexdigest(), "pip-1.0.dist-info/METADATA", "pip", RecordRow(path="pip/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"").hexdigest(), size="0"), ) - assert [(source, str(metadata)) for source, metadata in bundled.vendored] == [ - ("pip/_vendor/vendor.txt", "Name: urllib3\nVersion: 1.26.4\n\n"), - ("pip/_vendor/rich-13.0.dist-info/METADATA", "Name: rich\nVersion: 13.0\n\n"), + assert [(source, metadata.raw) for source, metadata in bundled.vendored] == [ + ("pip/_vendor/vendor.txt", {"name": "urllib3", "version": "1.26.4"}), + ("pip/_vendor/rich-13.0.dist-info/METADATA", {"name": "rich", "version": "13.0"}), ] @@ -337,7 +337,7 @@ def test_vendor_manifest_needs_exact_pins( def test_wheel_request(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: wheel = write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"Name: native\n"}) request = wheel_request(wheel, tmp_path, settings, (PurePosixPath("build.py"),)) - assert (request.root, request.metadata["Name"], request.wheel_name, request.settings, request.build_scripts) == ( + assert (request.root, request.metadata.name, request.wheel_name, request.settings, request.build_scripts) == ( tmp_path, "native", "native-1.0-py3-none-any.whl", diff --git a/tests/_generate/test_pipeline.py b/tests/_generate/test_pipeline.py index dc746c1..e65a885 100644 --- a/tests/_generate/test_pipeline.py +++ b/tests/_generate/test_pipeline.py @@ -1,6 +1,5 @@ from __future__ import annotations -from email import message_from_string from pathlib import Path from typing import TYPE_CHECKING @@ -8,6 +7,7 @@ from python_sbom._formats._json import dumps from python_sbom._generate._pipeline import build_sbom from python_sbom._model._inputs import SbomRequest, SourceTree +from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: from conftest import ContextFor @@ -18,7 +18,7 @@ def test_sbom_file_name_and_content(settings: Settings, facts: BuildFacts, context_for: ContextFor) -> None: request = SbomRequest( root=Path(), - metadata=message_from_string("Name: My.Project\nVersion: 1.0\n"), + metadata=parse_metadata("Name: My.Project\nVersion: 1.0\n"), wheel_name="my_project-1.0-py3-none-any.whl", settings=settings, build_scripts=(), diff --git a/tests/_model/test_bom.py b/tests/_model/test_bom.py index ef64c99..7fc51dd 100644 --- a/tests/_model/test_bom.py +++ b/tests/_model/test_bom.py @@ -3,7 +3,6 @@ import dataclasses import json import uuid -from email import message_from_string from pathlib import PurePosixPath from typing import TYPE_CHECKING, Final @@ -11,6 +10,7 @@ from python_sbom import BundledWheelInfo, ContentProvider, Contributions, ProviderContext from python_sbom._model._inputs import BuildScript, BundledWheel, Revision, SourceTree +from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: from conftest import Describe @@ -243,9 +243,9 @@ def test_bundled_wheel_and_its_vendored_packages(describe: Describe) -> None: path="src/pip-1.0-py3-none-any.whl", sha256="ab" * 32, metadata_name="pip-1.0.dist-info/METADATA", - metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + metadata=parse_metadata("Name: pip\nVersion: 1.0\n"), files=[], - vendored=[("pip/_vendor/vendor.txt", message_from_string("Name: urllib3\nVersion: 1.26.4\n"))], + vendored=[("pip/_vendor/vendor.txt", parse_metadata("Name: urllib3\nVersion: 1.26.4\n"))], ) class Seeded(ContentProvider): @@ -354,7 +354,7 @@ def test_bundled_wheel_supplier_is_the_projects_choice( path="src/pip-1.0-py3-none-any.whl", sha256="ab" * 32, metadata_name="pip-1.0.dist-info/METADATA", - metadata=message_from_string("Name: pip\nVersion: 1.0\nProject-URL: Source, https://github.com/example/pip\n"), + metadata=parse_metadata("Name: pip\nVersion: 1.0\nProject-URL: Source, https://github.com/example/pip\n"), files=[], vendored=[], ) diff --git a/tests/_model/test_components.py b/tests/_model/test_components.py index cba1666..ba274b8 100644 --- a/tests/_model/test_components.py +++ b/tests/_model/test_components.py @@ -1,7 +1,6 @@ from __future__ import annotations import hashlib -from email import message_from_string from pathlib import PurePosixPath from typing import TYPE_CHECKING, Final @@ -14,6 +13,7 @@ script_tool, ) from python_sbom._model._inputs import BundledWheel, RecordRow +from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: from python_sbom import Component, Property @@ -23,7 +23,7 @@ def test_component_from_metadata() -> None: - metadata = message_from_string( + metadata = parse_metadata( "Name: Example\nVersion: 1.0\nSummary: an example\nMaintainer-email: Maintainer \n" "License-Expression: MIT\nRequires-Python: >=3.11\nRequires-Dist: packaging>=24\nProvides-Extra: cli\n" "Classifier: Programming Language :: Python :: 3\nKeywords: sample,demo\n" @@ -66,11 +66,11 @@ def test_component_from_metadata() -> None: ], ) def test_component_authors(fields: str, expected: list[dict[str, str]]) -> None: - assert component_from_metadata(message_from_string(f"Name: a\nVersion: 1\n{fields}"))["authors"] == expected + assert component_from_metadata(parse_metadata(f"Name: a\nVersion: 1\n{fields}"))["authors"] == expected def test_component_without_authors_has_no_key() -> None: - assert "authors" not in component_from_metadata(message_from_string("Name: a\nVersion: 1\n")) + assert "authors" not in component_from_metadata(parse_metadata("Name: a\nVersion: 1\n")) @pytest.mark.parametrize( @@ -127,11 +127,11 @@ def test_component_without_authors_has_no_key() -> None: ], ) def test_component_licenses(fields: str, expected: list[License]) -> None: - assert component_from_metadata(message_from_string(f"Name: a\nVersion: 1\n{fields}"))["licenses"] == expected + assert component_from_metadata(parse_metadata(f"Name: a\nVersion: 1\n{fields}"))["licenses"] == expected def test_component_ignores_license_text() -> None: - metadata = message_from_string("Name: a\nVersion: 1\nLicense: Copyright\n the full text\n") + metadata = parse_metadata("Name: a\nVersion: 1\nLicense: Copyright\n the full text\n") assert "licenses" not in component_from_metadata(metadata) @@ -186,7 +186,7 @@ def test_component_ignores_license_text() -> None: ], ) def test_component_references(fields: str, expected: list[dict[str, str]]) -> None: - component = component_from_metadata(message_from_string(f"Name: a\nVersion: 1\n{fields}")) + component = component_from_metadata(parse_metadata(f"Name: a\nVersion: 1\n{fields}")) assert component["externalReferences"] == expected @@ -195,7 +195,7 @@ def test_bundled_wheel_component() -> None: path=_WHEEL, sha256="ab" * 32, metadata_name="pip-1.0.dist-info/METADATA", - metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + metadata=parse_metadata("Name: pip\nVersion: 1.0\n"), files=[RecordRow(path="pip/__init__.py", algorithm="SHA-256", digest="cd" * 32, size="0")], vendored=[], ) @@ -249,11 +249,11 @@ def test_bundled_wheel_vendored_metadata_wins_over_manifest() -> None: path=_WHEEL, sha256="ab" * 32, metadata_name="pip-1.0.dist-info/METADATA", - metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + metadata=parse_metadata("Name: pip\nVersion: 1.0\n"), files=[], vendored=[ - (manifest, message_from_string("Name: urllib3\nVersion: 1.26.4\n")), - (dist_info, message_from_string("Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n")), + (manifest, parse_metadata("Name: urllib3\nVersion: 1.26.4\n")), + (dist_info, parse_metadata("Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n")), ], ) component = bundled_wheel_component(wheel, "example", [], None) @@ -286,7 +286,7 @@ def component(path: str) -> Component: path=path, sha256="ab" * 32, metadata_name="pip-1.0.dist-info/METADATA", - metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + metadata=parse_metadata("Name: pip\nVersion: 1.0\n"), files=[], vendored=[], ) @@ -301,7 +301,7 @@ def test_bundled_wheel_supplier_never_comes_from_the_wheel() -> None: path=_WHEEL, sha256="ab" * 32, metadata_name="pip-1.0.dist-info/METADATA", - metadata=message_from_string("Name: pip\nVersion: 1.0\nProject-URL: Source, https://github.com/example/pip\n"), + metadata=parse_metadata("Name: pip\nVersion: 1.0\nProject-URL: Source, https://github.com/example/pip\n"), files=[], vendored=[], ) @@ -340,13 +340,13 @@ def test_vendored_spellings_of_one_package_merge() -> None: path=_WHEEL, sha256="ab" * 32, metadata_name="pip-1.0.dist-info/METADATA", - metadata=message_from_string("Name: pip\nVersion: 1.0\n"), + metadata=parse_metadata("Name: pip\nVersion: 1.0\n"), files=[], vendored=[ - ("pip/_vendor/vendor.txt", message_from_string("Name: jaraco.text\nVersion: 4.0\n")), + ("pip/_vendor/vendor.txt", parse_metadata("Name: jaraco.text\nVersion: 4.0\n")), ( "pip/_vendor/jaraco_text-4.0.dist-info/METADATA", - message_from_string("Name: jaraco_text\nVersion: 4.0\n"), + parse_metadata("Name: jaraco_text\nVersion: 4.0\n"), ), ], ) @@ -355,5 +355,5 @@ def test_vendored_spellings_of_one_package_merge() -> None: def test_pypi_purl_keeps_dots() -> None: - component = component_from_metadata(message_from_string("Name: zope.interface\nVersion: 7.0\n")) + component = component_from_metadata(parse_metadata("Name: zope.interface\nVersion: 7.0\n")) assert component["purl"] == "pkg:pypi/zope.interface@7.0" diff --git a/tests/_model/test_identity.py b/tests/_model/test_identity.py index 6aa3616..0a14415 100644 --- a/tests/_model/test_identity.py +++ b/tests/_model/test_identity.py @@ -2,12 +2,12 @@ import dataclasses import uuid -from email import message_from_string from typing import TYPE_CHECKING import pytest from python_sbom._model._identity import ProjectIdentity, project_identity +from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: from python_sbom._model._inputs import Settings @@ -34,7 +34,7 @@ def test_project_identity_repository( settings: Settings, repository: str | None, urls: str, expected: str | None ) -> None: - metadata = message_from_string(f"Name: My_Project\nVersion: 1.0\n{urls}") + metadata = parse_metadata(f"Name: My_Project\nVersion: 1.0\n{urls}") identity = project_identity(metadata, dataclasses.replace(settings, repository=repository)) assert identity == ProjectIdentity( property_prefix="my-project", diff --git a/tests/_model/test_licenses.py b/tests/_model/test_licenses.py index 494896a..fdc2c17 100644 --- a/tests/_model/test_licenses.py +++ b/tests/_model/test_licenses.py @@ -1,11 +1,11 @@ from __future__ import annotations -from email import message_from_string from typing import TYPE_CHECKING import pytest from python_sbom._model._licenses import licenses +from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: from python_sbom import License @@ -25,4 +25,4 @@ ], ) def test_license_expression(expression: str, expected: list[License]) -> None: - assert licenses(message_from_string(f"License-Expression: {expression}\n")) == expected + assert licenses(parse_metadata(f"License-Expression: {expression}\n")) == expected diff --git a/tests/_model/test_metadata.py b/tests/_model/test_metadata.py new file mode 100644 index 0000000..e4e9fb1 --- /dev/null +++ b/tests/_model/test_metadata.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import pytest +from packaging.metadata import Metadata + +from python_sbom._model._components import component_from_metadata +from python_sbom._model._metadata import parse_metadata + + +def test_repeated_single_use_field_keeps_every_value() -> None: + # packaging moves an Author given twice out of raw, and unparsed keeps both as written + metadata = parse_metadata("Name: a\nVersion: 1\nAuthor: Ann\nAuthor: Bob\n") + assert (metadata.values("author"), metadata.first("author")) == (["Ann", "Bob"], "Ann") + + +def test_absent_field() -> None: + metadata = parse_metadata("Name: a\nVersion: 1\n") + assert (metadata.values("classifiers"), metadata.first("summary")) == ([], None) + + +def test_without_name_and_version() -> None: + metadata = parse_metadata("Summary: nameless\n") + assert (metadata.name, metadata.version) == ("", "") + + +@pytest.mark.parametrize( + ("fields", "keywords"), + [ + pytest.param("Keywords: a,b\n", ["a,b"], id="comma-separated"), + pytest.param("Keywords: a b\n", ["a b"], id="space-separated"), + # packaging splits at commas and strips each keyword, so the spacing around them is not kept + pytest.param("Keywords: a, b\n", ["a,b"], id="comma-and-space"), + pytest.param("Keywords: a\nKeywords: b\n", ["a", "b"], id="repeated"), + pytest.param("", [], id="absent"), + ], +) +def test_keywords(fields: str, keywords: list[str]) -> None: + assert parse_metadata(f"Name: a\nVersion: 1\n{fields}").keywords() == keywords + + +@pytest.mark.parametrize( + ("fields", "urls"), + [ + pytest.param( + "Project-URL: Source, https://a\nProject-URL: Docs, https://d\n", + [("Source", "https://a"), ("Docs", "https://d")], + id="distinct-labels", + ), + # packaging moves every Project-URL to unparsed when a label repeats, and none of them is lost + pytest.param( + "Project-URL: Source, https://a\nProject-URL: Source , https://b\nProject-URL: Docs, https://d\n", + [("Source", "https://a"), ("Source", "https://b"), ("Docs", "https://d")], + id="repeated-label", + ), + pytest.param("Project-URL: Source\nProject-URL: Docs, https://d\n", [("Docs", "https://d")], id="without-url"), + pytest.param( + "Project-URL: Source, https://a\nProject-URL: Source, https://b\nProject-URL: Bare\n", + [("Source", "https://a"), ("Source", "https://b")], + id="repeated-label-and-without-url", + ), + ], +) +def test_project_urls(fields: str, urls: list[tuple[str, str]]) -> None: + assert parse_metadata(f"Name: a\nVersion: 1\n{fields}").project_urls() == urls + + +# the headers of installed distributions that packaging's validating parser rejects, which the SBOM still describes +@pytest.mark.parametrize( + ("text", "described"), + [ + pytest.param( + "Metadata-Version: 2.1\nName: six\nVersion: 1.17.0\nLicense: MIT\n" + "Classifier: License :: OSI Approved :: MIT License\nLicense-File: LICENSE\n", + ("six", "1.17.0", ["MIT License", "MIT"]), + id="six-license-file-before-2.4", + ), + pytest.param( + "Metadata-Version: 2.1\nName: covdefaults\nVersion: 2.3.0\nLicense: MIT\n" + "Description-Content-Type: text/markdown\nLicense-File: LICENSE\nRequires-Dist: coverage (>=6.0.2)\n", + ("covdefaults", "2.3.0", ["MIT"]), + id="covdefaults-license-file-before-2.4", + ), + pytest.param( + "Metadata-Version: 2.0\nName: ply\nVersion: 3.11\nLicense: BSD\nDescription-Content-Type: UNKNOWN\n" + "Platform: UNKNOWN\n", + ("ply", "3.11", ["BSD"]), + id="ply-unknown-version-and-content-type", + ), + pytest.param( + "Metadata-Version: 2.1\nName: uri-template\nVersion: 1.3.0\nLicense: MIT License\n" + "Project-URL: homepage, https://gitlab.linss.com/open-source/python/uri-template\nKeywords: config\n" + "License-File: LICENSE\nProvides-Extra: dev\nRequires-Dist: mypy ; extra == 'dev'\n", + ("uri-template", "1.3.0", ["MIT License"]), + id="uri-template-license-file-before-2.4", + ), + ], +) +def test_metadata_the_validating_parser_rejects(text: str, described: tuple[str, str, list[str]]) -> None: + with pytest.raises(ExceptionGroup): + Metadata.from_email(text, validate=True) + component = component_from_metadata(parse_metadata(text)) + names = [entry["license"]["name"] for entry in component["licenses"] if "name" in entry["license"]] + assert (component["name"], component["version"], names) == described diff --git a/tests/_model/test_tools.py b/tests/_model/test_tools.py index d2fff7f..a7c671e 100644 --- a/tests/_model/test_tools.py +++ b/tests/_model/test_tools.py @@ -1,10 +1,10 @@ from __future__ import annotations import dataclasses -from email import message_from_string from typing import TYPE_CHECKING from python_sbom._model._inputs import InstalledDistribution, RecordRow +from python_sbom._model._metadata import parse_metadata from python_sbom._model._tools import tool_components if TYPE_CHECKING: @@ -15,10 +15,10 @@ def test_tools_list_interpreter_then_distributions_by_name(facts: BuildFacts) -> row = RecordRow(path="zeta/__init__.py", algorithm="SHA-256", digest="ab" * 32, size="0") distributions = [ InstalledDistribution( - name="zeta", metadata=message_from_string("Name: Zeta\nVersion: 2\n"), files=[row], depends_on=[] + name="zeta", metadata=parse_metadata("Name: Zeta\nVersion: 2\n"), files=[row], depends_on=[] ), InstalledDistribution( - name="alpha", metadata=message_from_string("Name: alpha\nVersion: 1\n"), files=[], depends_on=["zeta"] + name="alpha", metadata=parse_metadata("Name: alpha\nVersion: 1\n"), files=[], depends_on=["zeta"] ), ] tools, dependencies = tool_components(dataclasses.replace(facts, distributions=distributions)) diff --git a/tests/conftest.py b/tests/conftest.py index 371ca0b..21eee23 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,6 @@ import shutil import subprocess import zipfile -from email import message_from_string from pathlib import Path from typing import TYPE_CHECKING, Protocol @@ -15,6 +14,7 @@ from python_sbom._generate._pipeline import build_sbom from python_sbom._model._identity import project_identity from python_sbom._model._inputs import BuildFacts, Interpreter, SbomRequest, Settings, SourceTree +from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: from collections.abc import Mapping @@ -149,7 +149,7 @@ def describe( ) -> Document: request = SbomRequest( root=Path(), - metadata=message_from_string(f"Name: Example\nVersion: 1.0\n{metadata}"), + metadata=parse_metadata(f"Name: Example\nVersion: 1.0\n{metadata}"), wheel_name="example-1.0-py3-none-any.whl", settings=settings, build_scripts=(), @@ -174,7 +174,7 @@ def describe( @pytest.fixture def context_for(settings: Settings) -> ContextFor: def context(root: Path, artifact: Path) -> ProviderContext: - identity = project_identity(message_from_string("Name: Example\nVersion: 1.0\n"), settings) + identity = project_identity(parse_metadata("Name: Example\nVersion: 1.0\n"), settings) return ProviderContext(root=root, settings=settings, identity=identity, artifact=artifact) return context diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 788bbcc..54ab95f 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -1,9 +1,7 @@ from __future__ import annotations import collections.abc -import email.message import importlib -import importlib.metadata import inspect import pathlib import pkgutil @@ -87,5 +85,4 @@ def _names() -> dict[str, object]: # simplint: ignore[SMP602] # see above typing, *(importlib.import_module(found.name) for found in pkgutil.walk_packages(python_sbom.__path__, "python_sbom.")), ] - names = dict(member for module in modules for member in inspect.getmembers(module)) - return {**names, "Message": email.message.Message, "PackageMetadata": importlib.metadata.PackageMetadata} + return dict(member for module in modules for member in inspect.getmembers(module)) From 778cd1b34eb2df2c96cc2d2046cca3bcd0d1b8c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:28:56 -0800 Subject: [PATCH 034/115] =?UTF-8?q?=E2=9A=A1=20perf:=20finish=20the=20pype?= =?UTF-8?q?rf=20benchmark=20harness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Times the hatch hook on a small project and on virtualenv, embed on a meson-python-shaped wheel in a 66-distribution build environment, archive on virtualenv's zipapp, validate on real and hostile 5k/10k component SBOMs, spdx, build-env collection and bundled-wheel RECORD verification. `compare` renders two result directories side by side. virtualenv's wheel and zipapp SBOMs list over 5,000 components, so their validate cases raise the limit to time the full check. --- bench/__main__.py | 108 ++++++++++++++++++++++++-------------------- bench/_cases.py | 110 ++++++++++++++++++++++++++------------------- bench/_fixtures.py | 25 ++++++----- pyproject.toml | 12 +++++ tox.toml | 7 +++ 5 files changed, 158 insertions(+), 104 deletions(-) diff --git a/bench/__main__.py b/bench/__main__.py index 6344f94..cd0c0ad 100644 --- a/bench/__main__.py +++ b/bench/__main__.py @@ -1,18 +1,12 @@ """ CLI entry point: ``python -m bench ``, run from the project root or with the root on ``PYTHONPATH``. -``prepare`` - build the fixtures into the work directory; the only command that needs the network -``run [CASE ...] [-- PYPERF-OPTION ...]`` - time each case (all by default) in its own pyperf process tree, one JSON file per case under - ``//``; options after ``--`` reach pyperf, overriding the case's defaults (e.g. ``--rigorous``) -``profile [CASE ...]`` - run each case once under cProfile, after one untimed warm-up, and write its stats, the top functions by - cumulative and own time, and with ``--py-spy`` a speedscope flame graph -``report PATH ...`` - render pyperf JSON files, or directories of them, as a Markdown table -``list`` - print the case names +``prepare`` builds the fixtures into the work directory, the only command that needs the network. ``run`` times each +case in its own pyperf process tree, one JSON file per case under ``//``; options after ``--`` reach +pyperf and override the case's defaults (e.g. ``--rigorous``). + +``profile`` runs each case once under cProfile after an untimed warm-up, and under py-spy with ``--py-spy``. ``report`` +renders pyperf JSON as a Markdown table, ``compare`` two result directories side by side, and ``list`` prints the cases. The timed process imports whichever python-sbom the running interpreter has installed, so comparing two revisions means running the same harness from two environments. @@ -30,6 +24,7 @@ import sys import tempfile import time +from importlib.metadata import version from operator import itemgetter from pathlib import Path from typing import TYPE_CHECKING, Final @@ -44,17 +39,12 @@ _ROOT: Final[Path] = Path(__file__).parents[1] _PYTHON: Final[str] = f"{sys.implementation.name}{sys.version_info.major}.{sys.version_info.minor}" -_TOP: Final[int] = 15 -_CLOCKS: Final[tuple[str, ...]] = ("wall", "cpu") def main(arguments: Sequence[str]) -> None: if arguments[:1] == ["worker"]: _worker(arguments[1], Path(arguments[2]), arguments[3], arguments[4:]) return - if arguments[:1] == ["profile-worker"]: - _profile_case(arguments[1], Path(arguments[2]), Path(arguments[3])) - return passthrough: list[str] = [] if "--" in arguments: split = arguments.index("--") @@ -69,9 +59,11 @@ def main(arguments: Sequence[str]) -> None: case "profile": _profile(options.work, options.out / _PYTHON / "profile", _selected(options.cases), py_spy=options.py_spy) case "once": - _once(options.work, options.case) + _once(options.case, options.work, options.cprofile) case "report": sys.stdout.write(_report(options.paths)) + case "compare": + sys.stdout.write(_compare(options.before, options.current)) case "list": sys.stdout.write("".join(f"{name}\n" for name in CASES)) @@ -91,13 +83,18 @@ def _parser() -> argparse.ArgumentParser: command.add_argument("cases", nargs="*", metavar="CASE", help="cases to run (default: all)") commands.choices["run"].add_argument( "--clock", - choices=_CLOCKS, + choices=("wall", "cpu"), default="wall", help="wall time, or the CPU time of the process and its children, steadier on a loaded machine", ) commands.choices["profile"].add_argument("--py-spy", action="store_true", help="also record with py-spy") - commands.add_parser("once", parents=[work], help="run one case once, for an outside profiler").add_argument("case") + once = commands.add_parser("once", parents=[work], help="run one case once, for a profiler") + once.add_argument("case") + once.add_argument("--cprofile", type=Path, metavar="STATS", help="run it under cProfile, writing its stats here") commands.add_parser("report", help="render results as Markdown").add_argument("paths", nargs="+", type=Path) + compare = commands.add_parser("compare", help="render two result directories side by side as Markdown") + compare.add_argument("before", type=Path, help="result directory of the baseline") + compare.add_argument("current", type=Path, help="result directory of the revision under test") commands.add_parser("list", help="print the case names") return parser @@ -143,8 +140,8 @@ def _worker(name: str, work: Path, clock: str, arguments: Sequence[str]) -> None runner = pyperf.Runner(program_args=("-m", "bench", "worker", name, str(work), clock)) runner.parse_args(list(arguments)) with tempfile.TemporaryDirectory(dir=work, prefix="scratch-") as scratch: - step = make_step(CASES[name], load(work), Path(scratch)) - runner.metadata["python_sbom_version"] = _python_sbom_version() + step = make_step(name, load(work), Path(scratch)) + runner.metadata["python_sbom_version"] = version("python-sbom") runner.metadata["bench_clock"] = clock runner.bench_time_func(name, _timed, step, time.perf_counter if clock == "wall" else _cpu_time) @@ -152,8 +149,7 @@ def _worker(name: str, work: Path, clock: str, arguments: Sequence[str]) -> None def _timed(loops: int, step: Step, clock: Callable[[], float]) -> float: total = 0.0 for _ in range(loops): - if step.reset is not None: - step.reset() + step.reset() start = clock() step.run() total += clock() - start @@ -168,14 +164,6 @@ def _cpu_time() -> float: return own.ru_utime + own.ru_stime + children.ru_utime + children.ru_stime -def _once(work: Path, name: str) -> None: - with tempfile.TemporaryDirectory(dir=work, prefix="scratch-") as scratch: - step = make_step(CASES[name], load(work), Path(scratch)) - if step.reset is not None: - step.reset() - step.run() - - def _profile(work: Path, out: Path, names: list[str], *, py_spy: bool) -> None: out.mkdir(parents=True, exist_ok=True) fixtures: Final = load(work) @@ -185,7 +173,10 @@ def _profile(work: Path, out: Path, names: list[str], *, py_spy: bool) -> None: continue stats_file = out / f"{name}.pstats" # each case in a fresh process, so one case's warm caches never shorten another's profile - subprocess.run([sys.executable, "-m", "bench", "profile-worker", name, str(work), str(stats_file)], check=True) + subprocess.run( + [sys.executable, "-m", "bench", "once", name, "--work", str(work), "--cprofile", str(stats_file)], + check=True, + ) summary.append(_hot_spots(name, stats_file)) (out / f"{name}.txt").write_text(_stats_text(stats_file), encoding="utf-8") # next to the interpreter too, as a venv's scripts are when its python runs without activating it @@ -217,20 +208,20 @@ def _profile(work: Path, out: Path, names: list[str], *, py_spy: bool) -> None: sys.stdout.write(f"wrote {target}\n") -def _profile_case(name: str, work: Path, stats_file: Path) -> None: +def _once(name: str, work: Path, stats_file: Path | None) -> None: # simplint: ignore[SMP505] # ruff's C901 on main + # without a stats file for an outside profiler such as py-spy with tempfile.TemporaryDirectory(dir=work, prefix="scratch-") as scratch: - step = make_step(CASES[name], load(work), Path(scratch)) + step = make_step(name, load(work), Path(scratch)) + step.reset() + if stats_file is None: + step.run() + return if step.command is not None: - if step.reset is not None: - step.reset() command = [step.command[0], "-m", "cProfile", "-o", str(stats_file), *step.command[1:]] subprocess.run(command, check=True, stdout=subprocess.DEVNULL) return - if CASES[name].tier != "huge": # warm, as the timed values are; the huge case loads its validator on import - if step.reset is not None: - step.reset() + if CASES[name].tier != "huge": # warm, as the timed values are; huge values are each a process's first call step.run() - if step.reset is not None: step.reset() profiler = cProfile.Profile() profiler.runcall(step.run) @@ -264,7 +255,7 @@ def _hot_spots(name: str, stats_file: Path) -> str: lines += [ f"| `{label}` | {calls if calls == primitive else f'{calls}/{primitive}'} | {own:.3f} | {cumulative:.3f} |" f" {100 * cumulative / total:.1f} |" - for label, calls, primitive, own, cumulative in sorted(rows, key=itemgetter(key), reverse=True)[:_TOP] + for label, calls, primitive, own, cumulative in sorted(rows, key=itemgetter(key), reverse=True)[:15] ] lines.append("") return "\n".join(lines) @@ -307,17 +298,38 @@ def _report(paths: list[Path]) -> str: f" {_seconds(benchmark.percentile(25))} - {_seconds(benchmark.percentile(75))} |" f" {_seconds(min(values))} - {_seconds(max(values))} | {len(values)} |" ) - return "\n".join(lines) + "\n" + return "".join(f"{line}\n" for line in lines) -def _seconds(value: float) -> str: - return f"{value:.2f} s" if value >= 1 else f"{value * 1000:.1f} ms" +def _compare(before: Path, current: Path) -> str: + import pyperf # ruff: ignore[import-outside-top-level] # see _worker + + lines = [ + "| path | before median ยฑ MAD | current median ยฑ MAD | change |", + "| --- | ---: | ---: | ---: |", + ] + for name in sorted({file.stem for file in (*before.glob("*.json"), *current.glob("*.json"))}): + old, new = ( + pyperf.Benchmark.load(str(file)) if (file := directory / f"{name}.json").is_file() else None + for directory in (before, current) + ) + # the median absolute deviation, which one value a loaded host stretched barely moves + cells = [ + "-" if benchmark is None else f"{_seconds(benchmark.median())} ยฑ {_seconds(benchmark.median_abs_dev())}" + for benchmark in (old, new) + ] + change = "-" if old is None or new is None else _change(old.median(), new.median()) + lines.append(f"| {name} | {cells[0]} | {cells[1]} | {change} |") + return "".join(f"{line}\n" for line in lines) -def _python_sbom_version() -> str: - from python_sbom import __version__ # ruff: ignore[import-outside-top-level] # see _worker +def _change(before: float, current: float) -> str: + ratio = before / current + return f"{ratio:.2f}x faster" if ratio >= 1 else f"{1 / ratio:.2f}x slower" - return __version__ + +def _seconds(value: float) -> str: + return f"{value:.2f} s" if value >= 1 else f"{value * 1000:.1f} ms" if __name__ == "__main__": diff --git a/bench/_cases.py b/bench/_cases.py index cf0acf6..06314cc 100644 --- a/bench/_cases.py +++ b/bench/_cases.py @@ -32,17 +32,17 @@ } -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True, slots=True, kw_only=True) class Step: - run: Callable[[], object] - reset: Callable[[], object] | None = None + run: Callable[[], None] + #: undoes what ``run`` changed on disk, untimed, before every call + reset: Callable[[], None] = lambda: None #: the command a cold case runs, which the profiler runs under cProfile instead of profiling the subprocess call command: tuple[str, ...] | None = None -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True, slots=True, kw_only=True) class Case: - name: str tier: str needs: tuple[str, ...] make: Callable[[dict[str, str], Path], Step] @@ -55,10 +55,11 @@ def make(fixtures: dict[str, str], scratch: Path) -> Step: if not hooks: os.environ["HATCH_BUILD_NO_HOOKS"] = "true" root = fixtures[root_key] - return Step( - run=lambda: list(WheelBuilder(root).build(directory=str(scratch), versions=["standard"])), - reset=lambda: _empty(scratch), - ) + + def run() -> None: + list(WheelBuilder(root).build(directory=str(scratch), versions=["standard"])) + + return Step(run=run, reset=lambda: _empty(scratch)) return make @@ -67,9 +68,13 @@ def _embed(*, cold: bool) -> Callable[[dict[str, str], Path], Step]: def make(fixtures: dict[str, str], scratch: Path) -> Step: wheel = scratch / Path(fixtures["meson-wheel"]).name arguments = ("embed", str(wheel), "--root", fixtures["meson"], "--build-env", fixtures["fat-env"]) + + def reset() -> None: + shutil.copyfile(fixtures["meson-wheel"], wheel) + return Step( run=_cold(arguments) if cold else _cli(arguments), - reset=lambda: shutil.copyfile(fixtures["meson-wheel"], wheel), + reset=reset, command=_command(arguments) if cold else None, ) @@ -128,8 +133,12 @@ def _collect_facts(fixtures: dict[str, str], scratch: Path) -> Step: # ruff: ig from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # see _validate root, path = Path(fixtures["meson"]), [fixtures["fat-env"]] - # no revision, so only reading the environment's distributions is timed - return Step(run=lambda: collect_facts(root, lambda _root: None, path=path)) + + def run() -> None: + # no revision, so only reading the environment's distributions is timed + collect_facts(root, lambda _root: None, path=path) + + return Step(run=run) def _bundled(key: str) -> Callable[[dict[str, str], Path], Step]: @@ -138,46 +147,54 @@ def make(fixtures: dict[str, str], scratch: Path) -> Step: # ruff: ignore[unuse wheel = Path(fixtures[key]) data = wheel.read_bytes() - return Step(run=lambda: read_bundled_wheel(data, wheel.name)) + + def run() -> None: + read_bundled_wheel(data, wheel.name) + + return Step(run=run) return make CASES: Final[dict[str, Case]] = { - case.name: case - for case in ( - Case("hatch-small", "slow", ("small",), _hatch("small", hooks=True)), - Case("hatch-small-nohook", "slow", ("small",), _hatch("small", hooks=False)), - Case("hatch-virtualenv", "slow", ("virtualenv",), _hatch("virtualenv", hooks=True)), - Case("hatch-virtualenv-nohook", "slow", ("virtualenv",), _hatch("virtualenv", hooks=False)), - Case("embed-meson-fat-env", "slow", ("meson", "meson-wheel", "fat-env"), _embed(cold=False)), - Case("cold-embed-meson-fat-env", "slow", ("meson", "meson-wheel", "fat-env"), _embed(cold=True)), - Case("archive-zipapp", "slow", ("virtualenv", "zipapp"), _archive), - Case("validate-own-wheel", "slow", ("own-wheel",), _validate("own-wheel")), - Case("cold-validate-own-wheel", "slow", ("own-wheel",), _cold_validate), - Case("validate-virtualenv-wheel", "slow", ("virtualenv-wheel",), _validate("virtualenv-wheel")), - Case("validate-zipapp", "slow", ("zipapp-embedded",), _validate("zipapp-embedded")), - # the default limit: a revision with the component cap times the rejection, one without the full schema check - Case("validate-hostile-10k", "huge", ("hostile",), _validate("hostile", expect_valid=False)), - # a limit above the count, so every revision times the full schema check - Case( - "validate-hostile-10k-uncapped", - "huge", - ("hostile",), - _validate("hostile", expect_valid=False, max_components=20_000), - ), - Case("spdx-zipapp", "fast", ("zipapp-sbom",), _spdx), - Case("collect-facts-fat-env", "slow", ("meson", "fat-env"), _collect_facts), - Case("bundled-wheel-pip", "fast", ("pip-wheel",), _bundled("pip-wheel")), - Case("bundled-wheel-setuptools", "fast", ("setuptools-wheel",), _bundled("setuptools-wheel")), - ) + "hatch-small": Case(tier="slow", needs=("small",), make=_hatch("small", hooks=True)), + "hatch-small-nohook": Case(tier="slow", needs=("small",), make=_hatch("small", hooks=False)), + "hatch-virtualenv": Case(tier="slow", needs=("virtualenv",), make=_hatch("virtualenv", hooks=True)), + "hatch-virtualenv-nohook": Case(tier="slow", needs=("virtualenv",), make=_hatch("virtualenv", hooks=False)), + "embed-meson-fat-env": Case(tier="slow", needs=("meson", "meson-wheel", "fat-env"), make=_embed(cold=False)), + "cold-embed-meson-fat-env": Case(tier="slow", needs=("meson", "meson-wheel", "fat-env"), make=_embed(cold=True)), + "archive-zipapp": Case(tier="slow", needs=("virtualenv", "zipapp"), make=_archive), + "validate-own-wheel": Case(tier="slow", needs=("own-wheel",), make=_validate("own-wheel")), + "cold-validate-own-wheel": Case(tier="slow", needs=("own-wheel",), make=_cold_validate), + # virtualenv's SBOMs list over 5,000 components, files included, so a limit above that times the full check + "validate-virtualenv-wheel": Case( + tier="slow", needs=("virtualenv-wheel",), make=_validate("virtualenv-wheel", max_components=20_000) + ), + "validate-zipapp": Case( + tier="slow", needs=("zipapp-embedded",), make=_validate("zipapp-embedded", max_components=20_000) + ), + # at the default cap, so every revision times the full schema check + "validate-hostile-5k": Case(tier="huge", needs=("hostile-5k",), make=_validate("hostile-5k", expect_valid=False)), + # the default limit: a revision with the component cap times the rejection, one without the full schema check + "validate-hostile-10k": Case( + tier="huge", needs=("hostile-10k",), make=_validate("hostile-10k", expect_valid=False) + ), + # a limit above the count, so every revision times the full schema check + "validate-hostile-10k-uncapped": Case( + tier="huge", needs=("hostile-10k",), make=_validate("hostile-10k", expect_valid=False, max_components=20_000) + ), + "spdx-zipapp": Case(tier="fast", needs=("zipapp-sbom",), make=_spdx), + "collect-facts-fat-env": Case(tier="slow", needs=("meson", "fat-env"), make=_collect_facts), + "bundled-wheel-pip": Case(tier="fast", needs=("pip-wheel",), make=_bundled("pip-wheel")), + "bundled-wheel-setuptools": Case(tier="fast", needs=("setuptools-wheel",), make=_bundled("setuptools-wheel")), } -def make_step(case: Case, fixtures: dict[str, str], scratch: Path) -> Step: +def make_step(name: str, fixtures: dict[str, str], scratch: Path) -> Step: os.environ["SOURCE_DATE_EPOCH"] = SOURCE_DATE_EPOCH + case = CASES[name] if missing := [key for key in case.needs if key not in fixtures]: - msg = f"{case.name} needs the fixture(s) {', '.join(missing)}; prepare with --virtualenv to build them" + msg = f"{name} needs the fixture(s) {', '.join(missing)}; prepare with --virtualenv to build them" raise SystemExit(msg) return case.make(fixtures, scratch) @@ -196,9 +213,13 @@ def _command(arguments: tuple[str, ...]) -> tuple[str, ...]: return (sys.executable, "-m", "python_sbom", *arguments) -def _cold(arguments: tuple[str, ...]) -> Callable[[], object]: +def _cold(arguments: tuple[str, ...]) -> Callable[[], None]: command = _command(arguments) - return lambda: subprocess.run(command, check=True, stdout=subprocess.DEVNULL) + + def run() -> None: + subprocess.run(command, check=True, stdout=subprocess.DEVNULL) + + return run def _empty(directory: Path) -> None: @@ -209,7 +230,6 @@ def _empty(directory: Path) -> None: __all__ = [ "CASES", "TIERS", - "Case", "Step", "make_step", ] diff --git a/bench/_fixtures.py b/bench/_fixtures.py index fe0f65e..daef25d 100644 --- a/bench/_fixtures.py +++ b/bench/_fixtures.py @@ -45,11 +45,11 @@ "tox", "wheel", ) -_HOSTILE_COMPONENTS: Final[int] = 10_000 +#: component counts of the hostile documents: the default cap, which still gets the full schema check, and twice it +_HOSTILE_COMPONENTS: Final[tuple[int, ...]] = (5_000, 10_000) _LICENSE: Final[str] = ( "MIT License\n\nCopyright (c) 2026 Bench Authors\n\nPermission is hereby granted, free of charge.\n" ) -_GIT_IDENTITY: Final[tuple[str, ...]] = ("-c", "user.name=bench", "-c", "user.email=bench@example.org") def prepare(work: Path, project: Path, virtualenv: Path | None) -> None: @@ -58,7 +58,9 @@ def prepare(work: Path, project: Path, virtualenv: Path | None) -> None: fixtures: Final[dict[str, str]] = {} fixtures["small"] = str(_small_project(work / "small")) fixtures["small-wheel"] = str(_hatchling_wheel(work / "small", work / "small-dist")) - fixtures["hostile"] = str(_hostile_document(Path(fixtures["small-wheel"]), work / "hostile-10k.cdx.json")) + for count in _HOSTILE_COMPONENTS: + target = work / f"hostile-{count // 1000}k.cdx.json" + fixtures[f"hostile-{count // 1000}k"] = str(_hostile_document(Path(fixtures["small-wheel"]), target, count)) fixtures["meson"] = str(_meson_project(work / "meson")) fixtures["meson-wheel"] = str(_meson_wheel(work / "meson-dist")) fixtures["fat-env"] = _fat_environment(work / "fat-env") @@ -66,7 +68,7 @@ def prepare(work: Path, project: Path, virtualenv: Path | None) -> None: fixtures["own-wheel"] = str(_hatchling_wheel(work / "python-sbom", work / "own-dist")) if virtualenv is not None: fixtures.update(_virtualenv(virtualenv, work)) - (work / "fixtures.json").write_text(json.dumps(fixtures, indent=2) + "\n", encoding="utf-8") + (work / "fixtures.json").write_text(f"{json.dumps(fixtures, indent=2)}\n", encoding="utf-8") sys.stdout.write(f"wrote {work / 'fixtures.json'}\n") @@ -117,7 +119,10 @@ def _commit(root: Path) -> None: "git", "-C", str(root), - *_GIT_IDENTITY, + "-c", + "user.name=bench", + "-c", + "user.email=bench@example.org", "commit", "--quiet", "--no-verify", @@ -139,16 +144,14 @@ def _hatchling_wheel(root: Path, directory: Path) -> Path: return wheel -def _hostile_document(wheel: Path, target: Path) -> Path: - # ten thousand unique components: over the default component cap, which rejects the document before the schema - # check, and with a raised limit a uniqueItems check that compares every pair of them +def _hostile_document(wheel: Path, target: Path, count: int) -> Path: + # unique components in one array, the small project's SBOM otherwise: over the default component cap the document + # is rejected before the schema check, at or under it the uniqueItems check compares every pair of them with zipfile.ZipFile(wheel) as archive: (name,) = (member for member in archive.namelist() if "/sboms/" in member) document = json.loads(archive.read(name)) hostile = copy.deepcopy(document) - hostile["components"] = [ - {"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(_HOSTILE_COMPONENTS) - ] + hostile["components"] = [{"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(count)] target.write_text(json.dumps(hostile), encoding="utf-8") return target diff --git a/pyproject.toml b/pyproject.toml index 33c67a6..2ac2c9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ test = [ ] type = [ "ty>=0.0.84", + { include-group = "bench" }, { include-group = "docs" }, { include-group = "test" }, ] @@ -100,6 +101,11 @@ docs = [ "sphinx-autodoc-typehints>=3.17", "sphinxcontrib-mermaid>=2.1.1", ] +bench = [ + "pyperf>=2.10", + { include-group = "build" }, + { include-group = "test" }, +] build = [ "hatch-fancy-pypi-readme>=25.1", "hatch-vcs>=0.5", @@ -156,6 +162,12 @@ lint.ignore = [ "undocumented-public-method", # undocumented-public-api (simplint SMP903) covers the public API instead "undocumented-public-nested-class", # undocumented-public-api (simplint SMP903) covers the public API instead ] +lint.per-file-ignores."bench/**/*.py" = [ + "import-private-name", # Benchmarks time the private entry points the public API wraps + "start-process-with-partial-path", # Fixtures run git from PATH, as a project build does + "subprocess-without-shell-equals-true", # Commands are fixed argument lists the harness builds + "suspicious-subprocess-import", # Fixtures and cold cases run git, uv and Python processes +] lint.per-file-ignores."tests/**/*.py" = [ "assert", # Tests use assertions "D", # Test names document behavior diff --git a/tox.toml b/tox.toml index b6e36da..bcb78cd 100644 --- a/tox.toml +++ b/tox.toml @@ -129,6 +129,13 @@ description = "run type check on code base" dependency_groups = [ "type" ] commands = [ [ "ty", "check", "--output-format", "concise", "--error-on-warning", "--python-platform", "all", "." ] ] +[env.bench] +description = "time python-sbom's hot paths with pyperf (not run in CI); `prepare` the fixtures first" +dependency_groups = [ "bench" ] +extras = [ "hatch", "validate" ] +set_env.PYTHONPATH = "{tox_root}" +commands = [ [ "python", "-m", "bench", { replace = "posargs", extend = true } ] ] + [env.dev] description = "dev environment with all deps at {envdir}" package = "editable" From 2832b0cf6119fed4fcec52a55fcfd6cba2380b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 00:28:57 -0800 Subject: [PATCH 035/115] =?UTF-8?q?=F0=9F=94=92=20build:=20lock=20the=20be?= =?UTF-8?q?nch=20dependency=20group?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pylock.toml | 166 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 110 insertions(+), 56 deletions(-) diff --git a/pylock.toml b/pylock.toml index b04a1ac..92fabad 100644 --- a/pylock.toml +++ b/pylock.toml @@ -1,6 +1,6 @@ lock-version = "1.0" environments = [ - "python_version == \"3.15\" and sys_platform == \"linux\" and platform_machine == \"x86_64\" and implementation_name == \"cpython\" and platform_python_implementation == \"CPython\" and platform_system == \"Linux\"", + "python_version == \"3.15\" and sys_platform == \"linux\" and platform_machine == \"x86_64\" and implementation_name == \"cpython\" and os_name == \"posix\" and platform_python_implementation == \"CPython\" and platform_system == \"Linux\"", ] requires-python = ">=3.11" extras = [ @@ -8,6 +8,7 @@ extras = [ "validate", ] dependency-groups = [ + "bench", "build", "dev", "docs", @@ -106,7 +107,7 @@ sha256 = "f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0" [[packages]] name = "arrow" version = "1.4.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" dependencies = [ { name = "python-dateutil" }, @@ -135,7 +136,7 @@ sha256 = "749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205" [[packages]] name = "attrs" version = "26.1.0" -marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -185,7 +186,7 @@ sha256 = "e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35" [[packages]] name = "beartype" version = "0.23.1" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -239,7 +240,7 @@ sha256 = "d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9" [[packages]] name = "boolean-py" version = "5.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" index = "https://pypi.org/simple/" [packages.sdist] @@ -416,7 +417,7 @@ sha256 = "5ae39c8c434b972f0740d04610759168590713175aab584b012b1b84f6771874" [[packages]] name = "click" version = "8.5.0" -marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -441,7 +442,7 @@ sha256 = "255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360" [[packages]] name = "covdefaults" version = "2.3.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.7" dependencies = [ { name = "coverage" }, @@ -469,7 +470,7 @@ sha256 = "2832961f6ffcfe4b57c338bc3418a3526f495c26fb9c54565409c5532f7c41be" [[packages]] name = "coverage" version = "7.17.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.11" index = "https://pypi.org/simple/" @@ -603,7 +604,7 @@ sha256 = "1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c" [[packages]] name = "cyclonedx-python-lib" version = "11.12.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "<4.0,>=3.9" dependencies = [ { name = "jsonschema" }, @@ -636,7 +637,7 @@ sha256 = "0e807521a921a5c3cb8ce1153f8a61d29eedfe76a46aac2796b7c6b573391a54" [[packages]] name = "defusedxml" version = "0.7.1" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" index = "https://pypi.org/simple/" @@ -735,7 +736,7 @@ sha256 = "46ce42d27e5cdde09ee7197349663e97cb106679bd9e75db3a09679fbc7aa1a8" [[packages]] name = "fqdn" version = "1.6.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<4,>=2.7" index = "https://pypi.org/simple/" @@ -792,7 +793,7 @@ sha256 = "bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f" [[packages]] name = "hatch-fancy-pypi-readme" version = "25.1.0" -marker = "\"build\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.8" dependencies = [ { name = "hatchling" }, @@ -820,7 +821,7 @@ sha256 = "ce0134c40d63d874ac48f48ccc678b8f3b62b8e50e9318520d2bffc752eedaf3" [[packages]] name = "hatch-vcs" version = "0.5.0" -marker = "\"build\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" dependencies = [ { name = "hatchling" }, @@ -849,7 +850,7 @@ sha256 = "b49677dbdc597460cc22d01b27ab3696f5e16a21ecf2700fb01bc28e1f2a04a7" [[packages]] name = "hatchling" version = "1.33.0" -marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" requires-python = ">=3.10" dependencies = [ { name = "packaging" }, @@ -934,7 +935,7 @@ sha256 = "6a16b69b93187244e0548cbfd25b3e4a6f9a7a2ad784625c3bec2b8d27b81aaa" [[packages]] name = "idna" version = "3.20" -marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -984,7 +985,7 @@ sha256 = "ea0c9a0384df69ed86a943a15cde37d0360b82491b3910dc2215e202e62b5b02" [[packages]] name = "iniconfig" version = "2.3.1" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -1009,7 +1010,7 @@ sha256 = "9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7" [[packages]] name = "installer" version = "1.1.0" -marker = "\"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -1034,7 +1035,7 @@ sha256 = "f55f5728ff61d3dd54dbc5a397cf3f7307cb64b180c20b72afbebf01c9e2c8a0" [[packages]] name = "isoduration" version = "20.11.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.7" dependencies = [ { name = "arrow" }, @@ -1196,7 +1197,7 @@ sha256 = "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" [[packages]] name = "jsonpointer" version = "3.6" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -1221,7 +1222,7 @@ sha256 = "a45c858f8a9e171f6b3790ed9446c9382782f6c4cb4589176813c5a553f4cfcd" [[packages]] name = "jsonschema" version = "4.26.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" dependencies = [ { name = "attrs" }, @@ -1261,7 +1262,7 @@ sha256 = "d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce" [[packages]] name = "jsonschema-specifications" version = "2025.9.1" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" dependencies = [ { name = "referencing" }, @@ -1321,7 +1322,7 @@ sha256 = "be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f" [[packages]] name = "lark" version = "1.3.1" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -1346,7 +1347,7 @@ sha256 = "c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12" [[packages]] name = "license-expression" version = "30.4.4" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" dependencies = [ { name = "boolean-py" }, @@ -1614,7 +1615,7 @@ sha256 = "bc0fb085d8ed0d67a6471892535f3d9b795eb911b7cd129d3a24967aab7a538f" [[packages]] name = "packageurl-python" version = "0.17.6" -marker = "\"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -1639,7 +1640,7 @@ sha256 = "31a85c2717bc41dd818f3c62908685ff9eebcb68588213745b14a6ee9e7df7c9" [[packages]] name = "packaging" version = "26.3" -marker = "\"hatch\" in extras or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"lint\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"project-runtime\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"hatch\" in extras or \"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"lint\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"project-runtime\" in dependency_groups or \"release-news\" in dependency_groups or \"release-notes\" in dependency_groups or \"release-prepare\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1664,7 +1665,7 @@ sha256 = "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c" [[packages]] name = "pathspec" version = "1.1.1" -marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1714,7 +1715,7 @@ sha256 = "1dadfc59a8a5d7565b3d9270ae8fa71c065595352c4277b41c11ee6c222c4b91" [[packages]] name = "pluggy" version = "1.7.0" -marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -1739,7 +1740,7 @@ sha256 = "7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec" [[packages]] name = "ply" version = "3.11" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" index = "https://pypi.org/simple/" [packages.sdist] @@ -1846,10 +1847,35 @@ size = 5942047 [packages.wheels.hashes] sha256 = "1ef04fd7b4bae3a31e1180bc87e877a3a5584063fe79320a125267afee47090d" +[[packages]] +name = "psutil" +version = "7.2.2" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.6" +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "psutil-7.2.2.tar.gz" +upload-time = 2026-01-28 18:14:54.428400+00:00 +url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz" +size = 493740 + +[packages.sdist.hashes] +sha256 = "0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372" + +[[packages.wheels]] +name = "psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl" +upload-time = 2026-01-28 18:15:25.976610+00:00 +url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl" +size = 155560 + +[packages.wheels.hashes] +sha256 = "076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9" + [[packages]] name = "py-serializable" version = "2.1.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "<4.0,>=3.8" dependencies = [ { name = "defusedxml" }, @@ -1961,7 +1987,7 @@ sha256 = "5299296ff5cafcb69851fd652de8f9268a0bb036bcbedbfcaa94cf743baf3986" [[packages]] name = "pygments" version = "2.21.0" -marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1986,7 +2012,7 @@ sha256 = "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9" [[packages]] name = "pyparsing" version = "3.3.3" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -2008,10 +2034,38 @@ size = 126420 [packages.wheels.hashes] sha256 = "ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4" +[[packages]] +name = "pyperf" +version = "2.10.0" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"type\" in dependency_groups" +requires-python = ">=3.9" +dependencies = [ + { name = "psutil" }, +] +index = "https://pypi.org/simple/" + +[packages.sdist] +name = "pyperf-2.10.0.tar.gz" +upload-time = 2026-02-07 11:35:14.693655+00:00 +url = "https://files.pythonhosted.org/packages/16/91/39ca77aa58f13e8c65d747ac7e06584b55acabfa98987fb8d546bc24860d/pyperf-2.10.0.tar.gz" +size = 227609 + +[packages.sdist.hashes] +sha256 = "dd93ccfda79214725293e95f1fa6e00cb4a64adcf1326039486d4e1f91caaa62" + +[[packages.wheels]] +name = "pyperf-2.10.0-py3-none-any.whl" +upload-time = 2026-02-07 11:35:12.636042+00:00 +url = "https://files.pythonhosted.org/packages/91/26/f7bd5e37c254c2671f4dfff316d123eba13663bdfee941a01b09ab02d72d/pyperf-2.10.0-py3-none-any.whl" +size = 144430 + +[packages.wheels.hashes] +sha256 = "79196bc4a11e3c926dd4c6b14c80136c6b37f884fe913cbc57037f37636e9841" + [[packages]] name = "pytest" version = "9.2.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "iniconfig" }, @@ -2042,7 +2096,7 @@ sha256 = "07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588" [[packages]] name = "pytest-cov" version = "7.1.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" dependencies = [ { name = "coverage" }, @@ -2072,7 +2126,7 @@ sha256 = "a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678" [[packages]] name = "pytest-mock" version = "3.17.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" dependencies = [ { name = "pytest" }, @@ -2100,7 +2154,7 @@ sha256 = "1b82aa4b37137248bd687055e109723debeb07b489de1565297dc9589c5f2d6b" [[packages]] name = "python-dateutil" version = "3.0.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" dependencies = [ { name = "six" }, @@ -2157,7 +2211,7 @@ sha256 = "fb65edfff3541b7ceb3a39be334a0f69f4cb5148287a053e1874748742fda93b" [[packages]] name = "pyyaml" version = "6.0.3" -marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"lint\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"lint\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -2173,7 +2227,7 @@ sha256 = "d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f" [[packages]] name = "rdflib" version = "7.6.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.8.1" dependencies = [ { name = "pyparsing" }, @@ -2231,7 +2285,7 @@ sha256 = "d0dae1f74bb273b534770cb4cccb6bb78735540afdb03c2146f4e19dcd412560" [[packages]] name = "referencing" version = "0.37.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" dependencies = [ { name = "attrs" }, @@ -2352,7 +2406,7 @@ sha256 = "cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" [[packages]] name = "rfc3339-validator" version = "0.1.4" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" dependencies = [ { name = "six" }, @@ -2405,7 +2459,7 @@ sha256 = "50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd" [[packages]] name = "rfc3986-validator" version = "0.1.1" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" index = "https://pypi.org/simple/" @@ -2430,7 +2484,7 @@ sha256 = "2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9" [[packages]] name = "rfc3987-syntax" version = "1.1.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" dependencies = [ { name = "lark" }, @@ -2512,7 +2566,7 @@ sha256 = "647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7" [[packages]] name = "rpds-py" version = "2026.9.1" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.11" index = "https://pypi.org/simple/" @@ -2566,7 +2620,7 @@ sha256 = "0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137" [[packages]] name = "semantic-version" version = "2.10.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=2.7" index = "https://pypi.org/simple/" @@ -2591,7 +2645,7 @@ sha256 = "de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177" [[packages]] name = "setuptools" version = "84.2.1" -marker = "\"build\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -2616,7 +2670,7 @@ sha256 = "ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2" [[packages]] name = "setuptools-scm" version = "10.3.4" -marker = "\"build\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.8" dependencies = [ { name = "packaging" }, @@ -2646,7 +2700,7 @@ sha256 = "82f34c3e3084fc2b57d397200637cc13f2338004052d2fee47baa5f0e902464e" [[packages]] name = "six" version = "1.17.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" index = "https://pypi.org/simple/" @@ -2696,7 +2750,7 @@ sha256 = "7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752" [[packages]] name = "sortedcontainers" version = "2.4.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" index = "https://pypi.org/simple/" [packages.sdist] @@ -2745,7 +2799,7 @@ sha256 = "8412826614f9c632227067d0985aeb9f584bd15efc852955430e097a9fe7afbf" [[packages]] name = "spdx-tools" version = "0.8.5" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" dependencies = [ { name = "beartype" }, @@ -3088,7 +3142,7 @@ sha256 = "6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331" [[packages]] name = "tomlkit" version = "0.15.2" -marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -3142,7 +3196,7 @@ sha256 = "ae4d223e6aadaff98d29b7f09e58d9c9ccf4cd3b455a2d2e0486d432cd8bf660" [[packages]] name = "trove-classifiers" version = "2026.9.21.13" -marker = "\"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"hatch\" in extras" index = "https://pypi.org/simple/" [packages.sdist] @@ -3280,7 +3334,7 @@ sha256 = "65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147" [[packages]] name = "tzdata" version = "2026.6" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=2" index = "https://pypi.org/simple/" @@ -3305,7 +3359,7 @@ sha256 = "0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec" [[packages]] name = "uri-template" version = "1.3.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.7" index = "https://pypi.org/simple/" @@ -3330,7 +3384,7 @@ sha256 = "a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363" [[packages]] name = "uritools" version = "6.1.3" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -3405,7 +3459,7 @@ sha256 = "22759b8895e1103578de6fb4cd27e94b4399d021023378d29f01333717751fba" [[packages]] name = "vcs-versioning" version = "2.6.0" -marker = "\"build\" in dependency_groups" +marker = "\"bench\" in dependency_groups or \"build\" in dependency_groups or \"dev\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.8" dependencies = [ { name = "packaging" }, @@ -3499,7 +3553,7 @@ sha256 = "901eace6d2dd25ebb917d0ce5db18e5c537e1504c905cf32f0d006c12098b05d" [[packages]] name = "webcolors" version = "25.10.0" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" index = "https://pypi.org/simple/" @@ -3549,7 +3603,7 @@ sha256 = "3fa599046443d4ca830d06e3d180cd0a675d5871af0a68daa5623318bb4d17e3" [[packages]] name = "xmltodict" version = "1.0.4" -marker = "\"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.9" index = "https://pypi.org/simple/" From d901f728ee51428692b54de56a80098806952821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 01:22:35 -0800 Subject: [PATCH 036/115] =?UTF-8?q?=F0=9F=90=9B=20fix(bench):=20keep=20hos?= =?UTF-8?q?tile=20documents=20at=20their=20count?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/_fixtures.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bench/_fixtures.py b/bench/_fixtures.py index daef25d..e39d761 100644 --- a/bench/_fixtures.py +++ b/bench/_fixtures.py @@ -8,7 +8,6 @@ from __future__ import annotations import base64 -import copy import hashlib import json import os @@ -145,12 +144,13 @@ def _hatchling_wheel(root: Path, directory: Path) -> Path: def _hostile_document(wheel: Path, target: Path, count: int) -> Path: - # unique components in one array, the small project's SBOM otherwise: over the default component cap the document - # is rejected before the schema check, at or under it the uniqueItems check compares every pair of them + # the small project's SBOM with every component in one array of unique ones: over the default component cap the + # document is rejected before the schema check, at or under it the uniqueItems check compares every pair of them. + # The build tools go, as their file components would count toward the cap too with zipfile.ZipFile(wheel) as archive: (name,) = (member for member in archive.namelist() if "/sboms/" in member) - document = json.loads(archive.read(name)) - hostile = copy.deepcopy(document) + hostile = json.loads(archive.read(name)) + hostile["metadata"]["tools"]["components"] = [] hostile["components"] = [{"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(count)] target.write_text(json.dumps(hostile), encoding="utf-8") return target From 2bdde37ca64d1c8f8080489d28fdea9d76d6e0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 01:11:22 -0800 Subject: [PATCH 037/115] =?UTF-8?q?=E2=9A=A1=20perf:=20check=20uniqueItems?= =?UTF-8?q?=20in=20one=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jsonschema's uniqueItems compares every pair of items, so 10,000 unique components took 54 s of CPU to validate. Items now freeze to hashable values that compare equal exactly when jsonschema's equal() holds, so one pass over a dict finds the first repeat; an item too deep to freeze falls back to jsonschema's own check. The schema check also reuses the parsed document instead of parsing the text again. The new message names the repeated items instead of quoting the array. --- src/python_sbom/_cli/_commands.py | 4 +- src/python_sbom/_validate/_document.py | 6 +- src/python_sbom/_validate/_schemas.py | 82 ++++++++++++++++++++++---- tests/_validate/test_schemas.py | 67 +++++++++++++++++++-- 4 files changed, 137 insertions(+), 22 deletions(-) diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py index 2b4805d..39e716e 100644 --- a/src/python_sbom/_cli/_commands.py +++ b/src/python_sbom/_cli/_commands.py @@ -163,8 +163,8 @@ def add_validate(commands: _SubParsersAction[ArgumentParser]) -> None: type=int, default=MAX_COMPONENTS, metavar="N", - help="reject an SBOM listing more than N components, nested ones included, without checking it further, since" - f" the schema check slows with the square of the count (default: {MAX_COMPONENTS})", + help="reject an SBOM listing more than N components, nested ones included, without checking it further, which" + f" bounds the work an untrusted SBOM can ask for (default: {MAX_COMPONENTS})", ) diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 290737d..53e863c 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -29,8 +29,8 @@ _SERIAL: Final[re.Pattern[str]] = re.compile(r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") # the archive reader stops at the same size _MAX_CHARACTERS: Final[int] = 16 * 1024 * 1024 -# the schema's uniqueItems compares every pair of items in an array, so a document whose components share one array -# takes about ten seconds to check at this count and four times that at twice it +# every check takes time in proportion to the count, about half a second of CPU per ten thousand components, so the cap +# bounds the work an untrusted document can ask for MAX_COMPONENTS: Final[int] = 5_000 # an exception's message can quote the hostile value that caused it _MAX_TEXT: Final[int] = 200 @@ -64,7 +64,7 @@ def _problems(raw: str, allowed_licenses: frozenset[str] | None, max_components: loaded: Final = json.loads(raw) if census := _census_problems(loaded, max_components): return census - if error := schema_error(raw): + if error := schema_error(loaded): return [f"does not conform to the CycloneDX 1.6 schema: {error}"] # python-sbom names its file in a metadata property, so documents other tools wrote get the schema check alone; the # schema already guarantees the shape of what this reads diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index a744482..a36bba8 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -1,12 +1,15 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Final +from enum import Enum +from typing import TYPE_CHECKING, Final, TypeAlias from python_sbom._errors import MissingExtraError -try: +try: # ruff: ignore[too-many-statements-in-try-clause] # one guard names the extra every import here needs from cyclonedx.schema import SchemaVersion - from cyclonedx.validation.json import JsonStrictValidator, JsonValidationError + from cyclonedx.validation.json import JsonStrictValidator + from jsonschema.exceptions import ValidationError + from jsonschema.validators import Draft7Validator, extend from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document @@ -15,22 +18,31 @@ raise MissingExtraError(_MESSAGE) from error if TYPE_CHECKING: - from jsonschema.exceptions import ValidationError + from collections.abc import Callable, Iterator + + from jsonschema.protocols import Validator from python_sbom._formats._spdx import Spdx -# loading the schema takes about a second, so the validator is built once -_VALIDATOR: Final[JsonStrictValidator] = JsonStrictValidator(SchemaVersion.V1_6) +# a JSON value, as json.loads hands it over +_Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" +# a JSON value made hashable, keeping jsonschema's notion of equality +_Frozen: TypeAlias = "str | int | float | _Truth | tuple[_Frozen, ...] | frozenset[tuple[str, _Frozen]] | None" +_Keyword: TypeAlias = "Callable[[Validator, bool, _Json, dict[str, _Json]], Iterator[ValidationError]]" # the message quotes the offending value, which for a document root runs to megabytes _MAX_TEXT: Final[int] = 200 -def schema_error(raw: str) -> str | None: - error: Final = _VALIDATOR.validate_str(raw, all_errors=False) - # all_errors=False returns one error or None, but under a runtime check type checkers do not follow - if not isinstance(error, JsonValidationError): +class _Truth(Enum): + # jsonschema tells true and false apart from 1 and 0, which Python's equality does not + TRUE = True + FALSE = False + + +def schema_error(document: _Json) -> str | None: + detail: Final[ValidationError | None] = next(iter(_VALIDATOR.iter_errors(document)), None) + if detail is None: return None - detail: Final[ValidationError] = error.data return f"{detail.message[:_MAX_TEXT]} at {detail.json_path[:_MAX_TEXT]}" @@ -42,6 +54,54 @@ def spdx_problems(spdx: Spdx) -> list[str]: return [f"SPDX rendering invalid: {message.validation_message}" for message in validate_full_spdx_document(parsed)] +def _unique_items( + validator: Validator, + unique: bool, # ruff: ignore[boolean-type-hint-positional-argument] # jsonschema calls keywords positionally + instance: _Json, + schema: dict[str, _Json], +) -> Iterator[ValidationError]: + # jsonschema compares every pair of items, ten seconds for five thousand components; two items freeze alike exactly + # when its equal() holds for them, so one pass over the frozen items gives the same verdict + if not unique or not isinstance(instance, list): + return + seen: Final[dict[_Frozen, int]] = {} + for index, item in enumerate(instance): + try: + frozen = _freeze(item) + except RecursionError: + # freezing recurses once per level of an item, so a deeply nested one gets jsonschema's own check + yield from _UNIQUE_ITEMS(validator, unique, instance, schema) + return + if (first := seen.setdefault(frozen, index)) != index: + # jsonschema's message quotes the whole array, which the report clips before it names the problem + yield ValidationError(f"item {index} repeats item {first}, yet the items must be unique") + return + + +def _freeze(value: _Json) -> _Frozen: + # a list freezes to a tuple and an object to a frozenset of its pairs, so the two never compare equal, as in + # equal(); numbers keep Python's equality, where 1 equals 1.0 as it does there + if isinstance(value, dict): + return frozenset({(key, _freeze(item)) for key, item in value.items()}) + if isinstance(value, list): + return tuple([_freeze(item) for item in value]) + if isinstance(value, bool): + return _Truth(value) + return value + + +# loading the schema takes about a second, so the validator is built once: the Draft 7 validator and format checks +# cyclonedx uses over its bundled schema and registry, with the uniqueItems check above swapped in; validating the +# parsed document skips the second parse validate_str does +_BASE: Final[Validator] = JsonStrictValidator(SchemaVersion.V1_6)._validator # ruff: ignore[private-member-access] # no public accessor +_UNIQUE_ITEMS: Final[_Keyword] = Draft7Validator.VALIDATORS["uniqueItems"] +_VALIDATOR: Final[Validator] = extend(Draft7Validator, {"uniqueItems": _unique_items})( + _BASE.schema, + registry=_BASE._registry, # ruff: ignore[private-member-access] # ty: ignore[unresolved-attribute] # see above + format_checker=Draft7Validator.FORMAT_CHECKER, +) + + __all__ = [ "schema_error", "spdx_problems", diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index 9221ff9..262ceae 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -1,13 +1,17 @@ from __future__ import annotations -import json import sys +from typing import TypeAlias import pytest +from jsonschema.validators import Draft7Validator from python_sbom._errors import MissingExtraError from python_sbom._validate._document import validate_document -from python_sbom._validate._schemas import schema_error +from python_sbom._validate._schemas import _unique_items, schema_error + +# a JSON value, as json.loads hands it over +_Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch) -> None: @@ -19,10 +23,61 @@ def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch) -> None: def test_schema_error_names_the_path() -> None: - raw = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"}) - assert schema_error(raw) == "'one' is not of type 'integer' at $.version" + document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"} + assert schema_error(document) == "'one' is not of type 'integer' at $.version" def test_schema_error_clips_the_offending_value() -> None: - raw = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "x" * 1000}) - assert schema_error(raw) == f"'{'x' * 199} at $.version" + document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "x" * 1000} + assert schema_error(document) == f"'{'x' * 199} at $.version" + + +@pytest.mark.parametrize( + "items", + [ + pytest.param([1, 1.0], id="int-equals-float"), + pytest.param([True, 1], id="true-is-not-one"), + pytest.param([False, 0], id="false-is-not-zero"), + pytest.param([True, True], id="repeated-boolean"), + pytest.param(["1", 1], id="string-is-not-number"), + pytest.param([None, None], id="repeated-null"), + pytest.param([[1, 2], [2, 1]], id="list-order-counts"), + pytest.param([[1], {"0": 1}], id="list-is-not-object"), + pytest.param([{"a": 1, "b": [True]}, {"b": [True], "a": 1.0}], id="object-key-order-does-not-count"), + pytest.param([{"a": True}, {"a": 1}], id="nested-boolean"), + pytest.param([], id="empty"), + ], +) +def test_unique_items_matches_jsonschema(items: list[_Json]) -> None: + validator = Draft7Validator({}) + expected = bool(list(Draft7Validator.VALIDATORS["uniqueItems"](validator, True, items, {}))) + assert bool(list(_unique_items(validator, True, items, {}))) is expected + + +@pytest.mark.parametrize( + ("unique", "instance"), [pytest.param(False, [1, 1], id="not-asked"), pytest.param(True, "11", id="not-an-array")] +) +def test_unique_items_skips(unique: bool, instance: _Json) -> None: + assert list(_unique_items(Draft7Validator({}), unique, instance, {})) == [] + + +def test_unique_items_too_deep_to_freeze() -> None: + # freezing the item overflows the stack, so jsonschema's own check judges the array, and a lone item is unique + nested: list[_Json] = [] + for _ in range(sys.getrecursionlimit()): + nested = [nested] + assert list(_unique_items(Draft7Validator({}), True, [nested], {})) == [] + + +def test_duplicate_components_are_rejected() -> None: + components: list[_Json] = [{"type": "library", "name": f"part-{index}"} for index in range(2_000)] + components.append({"type": "library", "name": "part-0"}) + document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components} + assert schema_error(document) == "item 2000 repeats item 0, yet the items must be unique at $.components" + + +def test_many_unique_components_pass() -> None: + components: list[_Json] = [ + {"type": "library", "bom-ref": f"ref-{index}", "name": f"part-{index}"} for index in range(20_000) + ] + assert schema_error({"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components}) is None From be210dc900b84485030684297ae8ad68ad54639c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 01:11:30 -0800 Subject: [PATCH 038/115] =?UTF-8?q?=E2=9A=A1=20perf:=20settle=20most=20iri?= =?UTF-8?q?-reference=20checks=20by=20regex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rfc3987-syntax runs an Earley parser on every URL and took most of the schema check's time on real SBOMs. Every RFC 3986 URI reference is an RFC 3987 IRI reference, so rfc3986-validator's regex, installed through jsonschema's format-nongpl extra, accepts most values first. Values it rejects still go to rfc3987-syntax, and so do IP literals and values ending in a newline, where the regex accepts strings rfc3987-syntax rejects. No string the fast path accepts was rejected by rfc3987-syntax across 2,000,000 generated strings, nor by rfc3987 across 300,000. Over 3,397 documents (real SBOMs, test fixtures, mutants and hostile ones) every accept/reject verdict matches the previous code. --- src/python_sbom/_validate/_schemas.py | 27 ++++++++++++++++++++++++--- tests/_validate/test_schemas.py | 23 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index a36bba8..fde8d89 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy from enum import Enum from typing import TYPE_CHECKING, Final, TypeAlias @@ -10,6 +11,7 @@ from cyclonedx.validation.json import JsonStrictValidator from jsonschema.exceptions import ValidationError from jsonschema.validators import Draft7Validator, extend + from rfc3986_validator import validate_rfc3986 from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document @@ -20,6 +22,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterator + from jsonschema import FormatChecker from jsonschema.protocols import Validator from python_sbom._formats._spdx import Spdx @@ -90,15 +93,33 @@ def _freeze(value: _Json) -> _Frozen: return value +def _iri_reference(instance: object) -> bool: # simplint: ignore[SMP602] # a format check takes any value + # rfc3987-syntax runs an Earley parser on every URL, most of the schema check's time; every RFC 3986 URI reference + # is an RFC 3987 IRI reference, so rfc3986-validator's regular expression accepts nearly all of them first. That + # expression lets a trailing newline through and accepts IPv6 literals rfc3987-syntax rejects, so those, and + # whatever it rejects, get the original check + if ( + isinstance(instance, str) + and "[" not in instance + and not instance.endswith("\n") + and validate_rfc3986(instance, rule="URI_reference") + ): + return True + original: Final = Draft7Validator.FORMAT_CHECKER.checkers["iri-reference"][0] + return original(instance) + + # loading the schema takes about a second, so the validator is built once: the Draft 7 validator and format checks -# cyclonedx uses over its bundled schema and registry, with the uniqueItems check above swapped in; validating the -# parsed document skips the second parse validate_str does +# cyclonedx uses over its bundled schema and registry, with the two checks above swapped in; validating the parsed +# document skips the second parse validate_str does _BASE: Final[Validator] = JsonStrictValidator(SchemaVersion.V1_6)._validator # ruff: ignore[private-member-access] # no public accessor _UNIQUE_ITEMS: Final[_Keyword] = Draft7Validator.VALIDATORS["uniqueItems"] +_CHECKER: Final[FormatChecker] = copy.deepcopy(Draft7Validator.FORMAT_CHECKER) +_CHECKER.checks("iri-reference", raises=_CHECKER.checkers["iri-reference"][1])(_iri_reference) _VALIDATOR: Final[Validator] = extend(Draft7Validator, {"uniqueItems": _unique_items})( _BASE.schema, registry=_BASE._registry, # ruff: ignore[private-member-access] # ty: ignore[unresolved-attribute] # see above - format_checker=Draft7Validator.FORMAT_CHECKER, + format_checker=_CHECKER, ) diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index 262ceae..fce519a 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -32,6 +32,29 @@ def test_schema_error_clips_the_offending_value() -> None: assert schema_error(document) == f"'{'x' * 199} at $.version" +@pytest.mark.parametrize( + "url", + [ + pytest.param("https://example.org/a?b=c#d", id="url"), + pytest.param("git+https://github.com/a/b.git@v1#egg=b", id="vcs-url"), + pytest.param("https://exรคmple.org/รผ", id="non-ascii"), + pytest.param("http://[::1]/", id="ipv6-literal"), + pytest.param("http://[v1.x]/", id="ipvfuture-literal"), + pytest.param("https://example.org/\n", id="trailing-newline"), + pytest.param("https://exa mple.org", id="space"), + pytest.param("https://example.org/%zz", id="bad-percent-escape"), + pytest.param("", id="empty"), + ], +) +def test_iri_reference_matches_jsonschema(url: str) -> None: + document: _Json = { + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "externalReferences": [{"type": "website", "url": url}], + } + assert (schema_error(document) is None) is Draft7Validator.FORMAT_CHECKER.conforms(url, "iri-reference") + + @pytest.mark.parametrize( "items", [ From cfe70dc9bdd1b04b8cf5d27bc6690164353ac74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 01:16:50 -0800 Subject: [PATCH 039/115] =?UTF-8?q?=E2=9A=A1=20perf:=20validate=20the=20sc?= =?UTF-8?q?hema=20with=20jsonschema-rs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jsonschema-rs checks the CycloneDX 1.6 schema, from the files cyclonedx-python-lib bundles, in milliseconds where python-jsonschema took seconds per SBOM. Over 3,397 documents it differs from python-jsonschema on 19, and on each it follows the RFC: IPv6 literals (RFC 3986 3.2.2) and characters past U+FFFF (RFC 3987 2.2) in iri-reference values, which rfc3987-syntax rejects. Probes found the same for leap seconds and trailing newlines in date-time (RFC 3339), malformed idn-email values (RFC 6531) and ECMA-262 "$" in patterns. jsonschema-rs recurses once per level and crashes the process on a deep enough document, so documents nesting past 128 levels are refused before the schema check, as are NaN and Infinity, which json.loads accepts but RFC 8259 leaves out of JSON and jsonschema-rs reads as null. pylock.toml still needs relocking for the new dependency. --- pyproject.toml | 6 +- src/python_sbom/_validate/_document.py | 27 ++++-- src/python_sbom/_validate/_schemas.py | 108 +++++------------------- tests/_validate/test_document.py | 24 ++++-- tests/_validate/test_schemas.py | 110 +++++++++++-------------- 5 files changed, 112 insertions(+), 163 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2ac2c9d..874e89c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,8 @@ optional-dependencies.hatch = [ "hatchling>=1.28", ] optional-dependencies.validate = [ - "cyclonedx-python-lib[json-validation]>=11.12", + "cyclonedx-python-lib>=11.12", + "jsonschema-rs>=0.60.3", "license-expression>=30", "spdx-tools>=0.8.5", ] @@ -79,9 +80,10 @@ dev = [ ] test = [ "covdefaults>=2.3", - "cyclonedx-python-lib[json-validation]>=11.12", + "cyclonedx-python-lib>=11.12", "hatchling>=1.28", "installer>=1.1", + "jsonschema-rs>=0.60.3", "pytest>=9.1.1", "pytest-cov>=7.1", "pytest-mock>=3.16", diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 53e863c..f121a4a 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import math import re from collections import Counter from typing import TYPE_CHECKING, Final, TypeAlias @@ -34,6 +35,9 @@ MAX_COMPONENTS: Final[int] = 5_000 # an exception's message can quote the hostile value that caused it _MAX_TEXT: Final[int] = 200 +# the schema validator recurses once per level and crashes the process past a few thousand levels on a small thread +# stack; the SBOMs python-sbom and other tools write nest about a dozen levels deep +_MAX_DEPTH: Final[int] = 128 def validate_document( @@ -49,8 +53,8 @@ def validate_document( except json.JSONDecodeError as error: return [f"is not JSON: {error}"] except RecursionError: - # the JSON parser and the schema validator recurse once per level, how deep depends on the Python version - return ["does not conform to the CycloneDX 1.6 schema: it nests too deeply to check"] + # the JSON parser recurses once per level, how deep it gets depends on the Python version + return [f"nests more than {_MAX_DEPTH} levels deep, too deep to check"] except Exception as error: # ruff: ignore[blind-except] # a hostile document may trip any check, see above return [f"could not be checked, {type(error).__name__}: {str(error)[:_MAX_TEXT]}"] @@ -88,22 +92,27 @@ def _problems(raw: str, allowed_licenses: frozenset[str] | None, max_components: def _census_problems(document: _Json, max_components: int) -> list[str]: - # counts the components and bom-refs in one linear pass before the schema vouches for any shape; iterative, so deep - # nesting cannot exhaust the stack. Nested components count too: the schema checks every components array, and - # one array may hold them all, so only the total bounds its work + # counts the components and bom-refs and measures the depth in one linear pass before the schema vouches for any + # shape; iterative, so deep nesting cannot exhaust the stack. Nested components count too: the schema checks every + # components array, and one array may hold them all, so only the total bounds its work components = 0 refs: Final[Counter[str]] = Counter() - pending: Final[list[_Json]] = [document] + pending: Final[list[tuple[_Json, int]]] = [(document, 1)] while pending: - node = pending.pop() + node, depth = pending.pop() + if depth > _MAX_DEPTH: + return [f"nests more than {_MAX_DEPTH} levels deep, too deep to check"] if isinstance(node, list): - pending.extend(node) + pending.extend((item, depth + 1) for item in node) elif isinstance(node, dict): if isinstance(ref := node.get("bom-ref"), str): refs[ref] += 1 if isinstance(listed := node.get("components"), list): components += len(listed) - pending.extend(node.values()) + pending.extend((item, depth + 1) for item in node.values()) + elif isinstance(node, float) and not math.isfinite(node): + # json.loads takes NaN and Infinity, which RFC 8259 leaves out of JSON and the schema check reads as null + return [f"holds {node}, which is not a JSON number"] if components > max_components: problem = ( f"lists more than {max_components} components, nested ones included;" diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index fde8d89..efa44f9 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -1,17 +1,14 @@ from __future__ import annotations -import copy -from enum import Enum +import json +from pathlib import Path from typing import TYPE_CHECKING, Final, TypeAlias from python_sbom._errors import MissingExtraError -try: # ruff: ignore[too-many-statements-in-try-clause] # one guard names the extra every import here needs - from cyclonedx.schema import SchemaVersion - from cyclonedx.validation.json import JsonStrictValidator - from jsonschema.exceptions import ValidationError - from jsonschema.validators import Draft7Validator, extend - from rfc3986_validator import validate_rfc3986 +try: + from cyclonedx.schema._res import CRYPTOGRAPHY_DEFS, JSF, SPDX_JSON # ruff: ignore[import-private-name] # its bundled schemas + from jsonschema_rs import Draft7Validator, Registry from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document @@ -20,33 +17,25 @@ raise MissingExtraError(_MESSAGE) from error if TYPE_CHECKING: - from collections.abc import Callable, Iterator - - from jsonschema import FormatChecker - from jsonschema.protocols import Validator + from jsonschema_rs import ValidationError from python_sbom._formats._spdx import Spdx # a JSON value, as json.loads hands it over _Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" -# a JSON value made hashable, keeping jsonschema's notion of equality -_Frozen: TypeAlias = "str | int | float | _Truth | tuple[_Frozen, ...] | frozenset[tuple[str, _Frozen]] | None" -_Keyword: TypeAlias = "Callable[[Validator, bool, _Json, dict[str, _Json]], Iterator[ValidationError]]" # the message quotes the offending value, which for a document root runs to megabytes _MAX_TEXT: Final[int] = 200 -class _Truth(Enum): - # jsonschema tells true and false apart from 1 and 0, which Python's equality does not - TRUE = True - FALSE = False - - def schema_error(document: _Json) -> str | None: - detail: Final[ValidationError | None] = next(iter(_VALIDATOR.iter_errors(document)), None) + detail: Final[ValidationError | None] = next(_VALIDATOR.iter_errors(document), None) if detail is None: return None - return f"{detail.message[:_MAX_TEXT]} at {detail.json_path[:_MAX_TEXT]}" + steps: Final[str] = "".join( + f"[{step}]" if isinstance(step, int) else f".{step}" if step.isidentifier() else f"[{step!r}]" + for step in detail.instance_path + ) + return f"{detail.message[:_MAX_TEXT]} at {f'${steps}'[:_MAX_TEXT]}" def spdx_problems(spdx: Spdx) -> list[str]: @@ -57,69 +46,16 @@ def spdx_problems(spdx: Spdx) -> list[str]: return [f"SPDX rendering invalid: {message.validation_message}" for message in validate_full_spdx_document(parsed)] -def _unique_items( - validator: Validator, - unique: bool, # ruff: ignore[boolean-type-hint-positional-argument] # jsonschema calls keywords positionally - instance: _Json, - schema: dict[str, _Json], -) -> Iterator[ValidationError]: - # jsonschema compares every pair of items, ten seconds for five thousand components; two items freeze alike exactly - # when its equal() holds for them, so one pass over the frozen items gives the same verdict - if not unique or not isinstance(instance, list): - return - seen: Final[dict[_Frozen, int]] = {} - for index, item in enumerate(instance): - try: - frozen = _freeze(item) - except RecursionError: - # freezing recurses once per level of an item, so a deeply nested one gets jsonschema's own check - yield from _UNIQUE_ITEMS(validator, unique, instance, schema) - return - if (first := seen.setdefault(frozen, index)) != index: - # jsonschema's message quotes the whole array, which the report clips before it names the problem - yield ValidationError(f"item {index} repeats item {first}, yet the items must be unique") - return - - -def _freeze(value: _Json) -> _Frozen: - # a list freezes to a tuple and an object to a frozenset of its pairs, so the two never compare equal, as in - # equal(); numbers keep Python's equality, where 1 equals 1.0 as it does there - if isinstance(value, dict): - return frozenset({(key, _freeze(item)) for key, item in value.items()}) - if isinstance(value, list): - return tuple([_freeze(item) for item in value]) - if isinstance(value, bool): - return _Truth(value) - return value - - -def _iri_reference(instance: object) -> bool: # simplint: ignore[SMP602] # a format check takes any value - # rfc3987-syntax runs an Earley parser on every URL, most of the schema check's time; every RFC 3986 URI reference - # is an RFC 3987 IRI reference, so rfc3986-validator's regular expression accepts nearly all of them first. That - # expression lets a trailing newline through and accepts IPv6 literals rfc3987-syntax rejects, so those, and - # whatever it rejects, get the original check - if ( - isinstance(instance, str) - and "[" not in instance - and not instance.endswith("\n") - and validate_rfc3986(instance, rule="URI_reference") - ): - return True - original: Final = Draft7Validator.FORMAT_CHECKER.checkers["iri-reference"][0] - return original(instance) - - -# loading the schema takes about a second, so the validator is built once: the Draft 7 validator and format checks -# cyclonedx uses over its bundled schema and registry, with the two checks above swapped in; validating the parsed -# document skips the second parse validate_str does -_BASE: Final[Validator] = JsonStrictValidator(SchemaVersion.V1_6)._validator # ruff: ignore[private-member-access] # no public accessor -_UNIQUE_ITEMS: Final[_Keyword] = Draft7Validator.VALIDATORS["uniqueItems"] -_CHECKER: Final[FormatChecker] = copy.deepcopy(Draft7Validator.FORMAT_CHECKER) -_CHECKER.checks("iri-reference", raises=_CHECKER.checkers["iri-reference"][1])(_iri_reference) -_VALIDATOR: Final[Validator] = extend(Draft7Validator, {"uniqueItems": _unique_items})( - _BASE.schema, - registry=_BASE._registry, # ruff: ignore[private-member-access] # ty: ignore[unresolved-attribute] # see above - format_checker=_CHECKER, +# the schema and the files it references are the ones cyclonedx's own Draft 7 validator reads, with format assertion on +# as there; offline, so a reference missing from the registry fails instead of reaching the network +_VALIDATOR: Final[Draft7Validator] = Draft7Validator( + json.loads(Path(SPDX_JSON).with_name("bom-1.6.SNAPSHOT.schema.json").read_text(encoding="utf-8")), + validate_formats=True, + offline=True, + registry=Registry([ + (f"http://cyclonedx.org/schema/{Path(path).name}", json.loads(Path(path).read_text(encoding="utf-8"))) + for path in (SPDX_JSON, CRYPTOGRAPHY_DEFS, JSF) + ]), ) diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py index 64a40bc..5ff4b5b 100644 --- a/tests/_validate/test_document.py +++ b/tests/_validate/test_document.py @@ -43,7 +43,7 @@ def test_not_json() -> None: def test_schema_violation() -> None: (problem,) = validate_document(json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"}), None) - assert problem == "does not conform to the CycloneDX 1.6 schema: 'one' is not of type 'integer' at $.version" + assert problem == 'does not conform to the CycloneDX 1.6 schema: "one" is not of type "integer" at $.version' def test_other_tools_documents_get_the_schema_check_alone() -> None: @@ -208,16 +208,30 @@ def shipped(licenses: list[License]) -> Component: def test_deep_nesting_is_reported(describe: Describe) -> None: + # two levels per component, deep enough to refuse and shallow enough for any Python's JSON parser nested = "".join( - f'{{"type": "library", "bom-ref": "deep-{level}", "name": "deep", "components": [' for level in range(500) + f'{{"type": "library", "bom-ref": "deep-{level}", "name": "deep", "components": [' for level in range(100) ) - nested += "]}" * 500 + nested += "]}" * 100 raw = json.dumps(describe()).replace('"components": []', f'"components": [{nested}]', 1) - assert validate_document(raw, None) == [ - "does not conform to the CycloneDX 1.6 schema: it nests too deeply to check" + assert validate_document(raw, None) == ["nests more than 128 levels deep, too deep to check"] + + +def test_nesting_too_deep_to_parse_is_reported() -> None: + assert validate_document("[" * 1_000_000 + "]" * 1_000_000, None) == [ + "nests more than 128 levels deep, too deep to check" ] +@pytest.mark.parametrize( + "constant", + [pytest.param("NaN", id="nan"), pytest.param("Infinity", id="infinity"), pytest.param("-Infinity", id="minus")], +) +def test_non_finite_numbers_are_rejected(constant: str) -> None: + raw = f'{{"bomFormat": "CycloneDX", "specVersion": "1.6", "version": {constant}}}' + assert validate_document(raw, None) == [f"holds {float(constant)}, which is not a JSON number"] + + def test_license_policy(describe: Describe) -> None: raw = with_components(describe(), shipped([{"expression": "MIT AND GPL-3.0-only"}])) assert validate_document(raw, frozenset({"MIT"})) == [ diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index fce519a..6631cc4 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -4,11 +4,10 @@ from typing import TypeAlias import pytest -from jsonschema.validators import Draft7Validator from python_sbom._errors import MissingExtraError from python_sbom._validate._document import validate_document -from python_sbom._validate._schemas import _unique_items, schema_error +from python_sbom._validate._schemas import schema_error # a JSON value, as json.loads hands it over _Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" @@ -16,7 +15,7 @@ def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch) -> None: # None in sys.modules makes the import fail, as when the extra is not installed - monkeypatch.setitem(sys.modules, "cyclonedx.schema", None) + monkeypatch.setitem(sys.modules, "jsonschema_rs", None) monkeypatch.delitem(sys.modules, "python_sbom._validate._schemas", raising=False) with pytest.raises(MissingExtraError, match=r"pip install 'python-sbom\[validate\]'"): validate_document("{}", None) @@ -24,83 +23,72 @@ def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch) -> None: def test_schema_error_names_the_path() -> None: document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"} - assert schema_error(document) == "'one' is not of type 'integer' at $.version" + assert schema_error(document) == '"one" is not of type "integer" at $.version' + + +def test_schema_error_quotes_keys_that_are_not_names() -> None: + document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "metadata": {"component": {"bom-ref": ""}}} + assert schema_error(document) == '"type" is a required property at $.metadata.component' + document = { + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "metadata": {"component": {"type": "library", "name": "a", "bom-ref": ""}}, + } + assert schema_error(document) == "\"\" is shorter than 1 character at $.metadata.component['bom-ref']" def test_schema_error_clips_the_offending_value() -> None: document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "x" * 1000} - assert schema_error(document) == f"'{'x' * 199} at $.version" + assert schema_error(document) == f'"{"x" * 199} at $.version' -@pytest.mark.parametrize( - "url", - [ - pytest.param("https://example.org/a?b=c#d", id="url"), - pytest.param("git+https://github.com/a/b.git@v1#egg=b", id="vcs-url"), - pytest.param("https://exรคmple.org/รผ", id="non-ascii"), - pytest.param("http://[::1]/", id="ipv6-literal"), - pytest.param("http://[v1.x]/", id="ipvfuture-literal"), - pytest.param("https://example.org/\n", id="trailing-newline"), - pytest.param("https://exa mple.org", id="space"), - pytest.param("https://example.org/%zz", id="bad-percent-escape"), - pytest.param("", id="empty"), - ], -) -def test_iri_reference_matches_jsonschema(url: str) -> None: - document: _Json = { - "bomFormat": "CycloneDX", - "specVersion": "1.6", - "externalReferences": [{"type": "website", "url": url}], - } - assert (schema_error(document) is None) is Draft7Validator.FORMAT_CHECKER.conforms(url, "iri-reference") +def _url(value: str) -> dict[str, _Json]: + return {"externalReferences": [{"type": "website", "url": value}]} @pytest.mark.parametrize( - "items", + ("fields", "valid"), [ - pytest.param([1, 1.0], id="int-equals-float"), - pytest.param([True, 1], id="true-is-not-one"), - pytest.param([False, 0], id="false-is-not-zero"), - pytest.param([True, True], id="repeated-boolean"), - pytest.param(["1", 1], id="string-is-not-number"), - pytest.param([None, None], id="repeated-null"), - pytest.param([[1, 2], [2, 1]], id="list-order-counts"), - pytest.param([[1], {"0": 1}], id="list-is-not-object"), - pytest.param([{"a": 1, "b": [True]}, {"b": [True], "a": 1.0}], id="object-key-order-does-not-count"), - pytest.param([{"a": True}, {"a": 1}], id="nested-boolean"), - pytest.param([], id="empty"), + pytest.param(_url("https://example.org/a?b=c#d"), True, id="url"), + pytest.param(_url("https://exรคmple.org/รผ"), True, id="url-non-ascii"), + # RFC 3987 section 2.2: ucschar runs to U+1FFFD, past the Basic Multilingual Plane + pytest.param(_url("https://example.org/๐Ÿ˜€"), True, id="url-astral"), + # RFC 3986 section 3.2.2: IP-literal = "[" ( IPv6address / IPvFuture ) "]" + pytest.param(_url("http://[::1]/"), True, id="url-ipv6-literal"), + pytest.param(_url("https://example.org/\n"), False, id="url-trailing-newline"), + pytest.param(_url("https://exa mple.org"), False, id="url-space"), + pytest.param(_url("https://example.org/%zz"), False, id="url-bad-percent-escape"), + # RFC 3339 section 5.6: time-second runs to 60 for a leap second + pytest.param({"metadata": {"timestamp": "2016-12-31T23:59:60Z"}}, True, id="timestamp-leap-second"), + pytest.param({"metadata": {"timestamp": "2016-12-31T23:59:59Z\n"}}, False, id="timestamp-trailing-newline"), + # RFC 6531 Mailbox: a local part and a domain, the domain's labels starting with a letter or digit + pytest.param({"metadata": {"authors": [{"email": "a@b.org"}]}}, True, id="email"), + pytest.param({"metadata": {"authors": [{"email": "@b.org"}]}}, False, id="email-without-local-part"), + pytest.param({"metadata": {"authors": [{"email": "a@-b.org"}]}}, False, id="email-label-starts-with-hyphen"), + # draft 7 section 6.3.3: patterns are ECMA-262, whose $ does not match before a final newline + pytest.param({"serialNumber": "urn:uuid:bde8d34d-2da7-5045-bd69-644208002436\n"}, False, id="pattern-newline"), ], ) -def test_unique_items_matches_jsonschema(items: list[_Json]) -> None: - validator = Draft7Validator({}) - expected = bool(list(Draft7Validator.VALIDATORS["uniqueItems"](validator, True, items, {}))) - assert bool(list(_unique_items(validator, True, items, {}))) is expected +def test_formats_follow_their_rfcs(fields: dict[str, _Json], valid: bool) -> None: + assert (schema_error({"bomFormat": "CycloneDX", "specVersion": "1.6", **fields}) is None) is valid @pytest.mark.parametrize( - ("unique", "instance"), [pytest.param(False, [1, 1], id="not-asked"), pytest.param(True, "11", id="not-an-array")] + ("first", "second", "unique"), + [ + pytest.param({"version": 1}, {"version": 1.0}, False, id="int-equals-float"), + pytest.param({"version": "1"}, {"version": "1"}, False, id="same"), + pytest.param({"version": "1", "group": "g"}, {"group": "g", "version": "1"}, False, id="key-order"), + pytest.param({"version": "1"}, {"version": "2"}, True, id="different"), + ], ) -def test_unique_items_skips(unique: bool, instance: _Json) -> None: - assert list(_unique_items(Draft7Validator({}), unique, instance, {})) == [] - - -def test_unique_items_too_deep_to_freeze() -> None: - # freezing the item overflows the stack, so jsonschema's own check judges the array, and a lone item is unique - nested: list[_Json] = [] - for _ in range(sys.getrecursionlimit()): - nested = [nested] - assert list(_unique_items(Draft7Validator({}), True, [nested], {})) == [] - - -def test_duplicate_components_are_rejected() -> None: - components: list[_Json] = [{"type": "library", "name": f"part-{index}"} for index in range(2_000)] - components.append({"type": "library", "name": "part-0"}) - document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components} - assert schema_error(document) == "item 2000 repeats item 0, yet the items must be unique at $.components" +def test_unique_items(first: dict[str, _Json], second: dict[str, _Json], unique: bool) -> None: + components: list[_Json] = [{"type": "library", "name": "a", **first}, {"type": "library", "name": "a", **second}] + assert (schema_error({"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components}) is None) is unique def test_many_unique_components_pass() -> None: components: list[_Json] = [ - {"type": "library", "bom-ref": f"ref-{index}", "name": f"part-{index}"} for index in range(20_000) + {"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(20_000) ] assert schema_error({"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components}) is None From 9749e0e1284b384a31677f5a7f5b9d6be900c986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:12:20 -0800 Subject: [PATCH 040/115] =?UTF-8?q?=F0=9F=93=A6=20build:=20ship=20the=20Cy?= =?UTF-8?q?cloneDX=20schema=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jsonschema-rs read the schemas from cyclonedx-python-lib's private cyclonedx.schema._res, and the library has no public way to reach them. The official files now ship in the package, unchanged: bom-1.6 and jsf-0.82 from the specification's 1.6.2 release, spdx from 1.7.2, the newest license list (3.29, a superset of the 3.28 the library bundled), with the Apache-2.0 LICENSE. A test pins each to the released file's SHA-256. cyclonedx-python-lib leaves the validate extra and the test group, and with it 21 locked packages; jsonschema-rs joins the lock. --- .gitattributes | 1 + .pre-commit-config.yaml | 2 + pylock.toml | 579 +- pyproject.toml | 2 - src/python_sbom/_validate/_schemas.py | 23 +- src/python_sbom/_validate/schemas/LICENSE | 201 + .../_validate/schemas/bom-1.6.schema.json | 5699 +++++++++++++++++ .../_validate/schemas/jsf-0.82.schema.json | 240 + .../_validate/schemas/spdx.schema.json | 1662 +++++ tests/_validate/test_schemas.py | 35 + 10 files changed, 7868 insertions(+), 576 deletions(-) create mode 100644 src/python_sbom/_validate/schemas/LICENSE create mode 100644 src/python_sbom/_validate/schemas/bom-1.6.schema.json create mode 100644 src/python_sbom/_validate/schemas/jsf-0.82.schema.json create mode 100644 src/python_sbom/_validate/schemas/spdx.schema.json diff --git a/.gitattributes b/.gitattributes index 5e8ba4d..04e00a9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ .git_archival.txt export-subst tests/_generate/pinned/** -text +src/python_sbom/_validate/schemas/** -text diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6969ab9..bb0ca1b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,8 @@ ci: autoupdate_schedule: quarterly default_language_version: python: python3.15 +# the CycloneDX schema files ship byte for byte as upstream released them +exclude: ^src/python_sbom/_validate/schemas/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 diff --git a/pylock.toml b/pylock.toml index 92fabad..1fa3661 100644 --- a/pylock.toml +++ b/pylock.toml @@ -104,39 +104,10 @@ size = 13427 [packages.wheels.hashes] sha256 = "f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0" -[[packages]] -name = "arrow" -version = "1.4.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.8" -dependencies = [ - { name = "python-dateutil" }, - { name = "tzdata" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "arrow-1.4.0.tar.gz" -upload-time = 2025-10-18 17:46:46.761994+00:00 -url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz" -size = 152931 - -[packages.sdist.hashes] -sha256 = "ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7" - -[[packages.wheels]] -name = "arrow-1.4.0-py3-none-any.whl" -upload-time = 2025-10-18 17:46:45.663184+00:00 -url = "https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl" -size = 68797 - -[packages.wheels.hashes] -sha256 = "749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205" - [[packages]] name = "attrs" version = "26.1.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"pkg-meta\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -601,64 +572,6 @@ size = 4780973 [packages.wheels.hashes] sha256 = "1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c" -[[packages]] -name = "cyclonedx-python-lib" -version = "11.12.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = "<4.0,>=3.9" -dependencies = [ - { name = "jsonschema" }, - { name = "license-expression" }, - { name = "packageurl-python" }, - { name = "py-serializable" }, - { name = "referencing" }, - { name = "sortedcontainers" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "cyclonedx_python_lib-11.12.0.tar.gz" -upload-time = 2026-08-13 07:52:18.675291+00:00 -url = "https://files.pythonhosted.org/packages/17/40/6509e6cfd7f2f3255501690f46375fdd224949e21fd1e96f4f4c8a9041b1/cyclonedx_python_lib-11.12.0.tar.gz" -size = 1445378 - -[packages.sdist.hashes] -sha256 = "16767c4039de90c04e9f03348f8f0ed4b8ff842eaa7eefcad3a95685f970dacf" - -[[packages.wheels]] -name = "cyclonedx_python_lib-11.12.0-py3-none-any.whl" -upload-time = 2026-08-13 07:52:16.836824+00:00 -url = "https://files.pythonhosted.org/packages/cf/f0/b2cb999244f2f4194df63ecff6939228178fa63a33be809c412684ca8db7/cyclonedx_python_lib-11.12.0-py3-none-any.whl" -size = 529453 - -[packages.wheels.hashes] -sha256 = "0e807521a921a5c3cb8ce1153f8a61d29eedfe76a46aac2796b7c6b573391a54" - -[[packages]] -name = "defusedxml" -version = "0.7.1" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "defusedxml-0.7.1.tar.gz" -upload-time = 2021-03-08 10:59:26.269532+00:00 -url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz" -size = 75520 - -[packages.sdist.hashes] -sha256 = "1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69" - -[[packages.wheels]] -name = "defusedxml-0.7.1-py2.py3-none-any.whl" -upload-time = 2021-03-08 10:59:24.450195+00:00 -url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl" -size = 25604 - -[packages.wheels.hashes] -sha256 = "a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" - [[packages]] name = "distlib" version = "0.4.4" @@ -733,31 +646,6 @@ size = 163877 [packages.wheels.hashes] sha256 = "46ce42d27e5cdde09ee7197349663e97cb106679bd9e75db3a09679fbc7aa1a8" -[[packages]] -name = "fqdn" -version = "1.6.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<4,>=2.7" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "fqdn-1.6.0.tar.gz" -upload-time = 2026-09-28 05:43:35.482270+00:00 -url = "https://files.pythonhosted.org/packages/02/01/248d44fbda9e78fa1a7b5a475194bef4c4a3e285db286419f3bb42f2efdf/fqdn-1.6.0.tar.gz" -size = 130737 - -[packages.sdist.hashes] -sha256 = "e39bf62e2a9481aa2cb780554fdfd944dd8ee5f230eb831684d539cbafed8de2" - -[[packages.wheels]] -name = "fqdn-1.6.0-py3-none-any.whl" -upload-time = 2026-09-28 05:43:33.951821+00:00 -url = "https://files.pythonhosted.org/packages/15/d5/a8f9149b97cced9e1f4bff92ff0b6206120e2c853ae774d7020d6ecd8fa8/fqdn-1.6.0-py3-none-any.whl" -size = 12569 - -[packages.wheels.hashes] -sha256 = "6c793c312ccfa29981e1135b3ef5e1277ebe639731a6b0f7350d90e96e63f0b4" - [[packages]] name = "furo" version = "2025.12.19" @@ -935,7 +823,7 @@ sha256 = "6a16b69b93187244e0548cbfd25b3e4a6f9a7a2ad784625c3bec2b8d27b81aaa" [[packages]] name = "idna" version = "3.20" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"dev\" in dependency_groups or \"docs\" in dependency_groups or \"pkg-meta\" in dependency_groups or \"type\" in dependency_groups" requires-python = ">=3.9" index = "https://pypi.org/simple/" @@ -1032,34 +920,6 @@ size = 464466 [packages.wheels.hashes] sha256 = "f55f5728ff61d3dd54dbc5a397cf3f7307cb64b180c20b72afbebf01c9e2c8a0" -[[packages]] -name = "isoduration" -version = "20.11.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.7" -dependencies = [ - { name = "arrow" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "isoduration-20.11.0.tar.gz" -upload-time = 2020-11-01 11:00:00.312591+00:00 -url = "https://files.pythonhosted.org/packages/7c/1a/3c8edc664e06e6bd06cce40c6b22da5f1429aa4224d0c590f3be21c91ead/isoduration-20.11.0.tar.gz" -size = 11649 - -[packages.sdist.hashes] -sha256 = "ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9" - -[[packages.wheels]] -name = "isoduration-20.11.0-py3-none-any.whl" -upload-time = 2020-11-01 10:59:58.020369+00:00 -url = "https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl" -size = 11321 - -[packages.wheels.hashes] -sha256 = "b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042" - [[packages]] name = "jaraco-classes" version = "3.4.0" @@ -1195,97 +1055,29 @@ size = 134899 sha256 = "85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67" [[packages]] -name = "jsonpointer" -version = "3.6" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.10" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "jsonpointer-3.6.tar.gz" -upload-time = 2026-10-26 20:59:26.307086+00:00 -url = "https://files.pythonhosted.org/packages/54/04/e35b34f4e441473e62d82d6ed9a8f748d93d895f4c05d4e645aa0fde51d7/jsonpointer-3.6.tar.gz" -size = 14352 - -[packages.sdist.hashes] -sha256 = "82e839a3ba3a9e2b54394fda8b3d276c2459d80408d06ef9a6fbc369d7544ffc" - -[[packages.wheels]] -name = "jsonpointer-3.6-py3-none-any.whl" -upload-time = 2026-10-26 20:59:24.947043+00:00 -url = "https://files.pythonhosted.org/packages/6e/34/cee746fb30473bfaa92f19267fcc102a1118eaec83e7e2b5652a91ead01c/jsonpointer-3.6-py3-none-any.whl" -size = 9333 - -[packages.wheels.hashes] -sha256 = "a45c858f8a9e171f6b3790ed9446c9382782f6c4cb4589176813c5a553f4cfcd" - -[[packages]] -name = "jsonschema" -version = "4.26.0" +name = "jsonschema-rs" +version = "0.60.3" marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" requires-python = ">=3.10" -dependencies = [ - { name = "attrs" }, - { name = "fqdn" }, - { name = "idna" }, - { name = "isoduration" }, - { name = "jsonpointer" }, - { name = "jsonschema-specifications" }, - { name = "referencing" }, - { name = "rfc3339-validator" }, - { name = "rfc3986-validator" }, - { name = "rfc3987-syntax" }, - { name = "rpds-py" }, - { name = "uri-template" }, - { name = "webcolors" }, -] index = "https://pypi.org/simple/" [packages.sdist] -name = "jsonschema-4.26.0.tar.gz" -upload-time = 2026-01-07 13:41:07.246891+00:00 -url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz" -size = 366583 +name = "jsonschema_rs-0.60.3.tar.gz" +upload-time = 2026-10-27 10:34:36.803131+00:00 +url = "https://files.pythonhosted.org/packages/36/17/383e687f63ba7c8c730ff770cb5ff70df32bd7f5114bfdcbd4387b19af54/jsonschema_rs-0.60.3.tar.gz" +size = 2926972 [packages.sdist.hashes] -sha256 = "0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326" +sha256 = "1d831d8eeb775af547aec0e9f75da51122a9c3563486d02e43db88d425b32c86" [[packages.wheels]] -name = "jsonschema-4.26.0-py3-none-any.whl" -upload-time = 2026-01-07 13:41:05.306692+00:00 -url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl" -size = 90630 +name = "jsonschema_rs-0.60.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +upload-time = 2026-10-27 10:33:53.610644+00:00 +url = "https://files.pythonhosted.org/packages/df/5d/f16cf6b7616b812c057921d31b9d58f42e311808afbcbb7bd3c58e608b65/jsonschema_rs-0.60.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" +size = 6183925 [packages.wheels.hashes] -sha256 = "d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce" - -[[packages]] -name = "jsonschema-specifications" -version = "2025.9.1" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.9" -dependencies = [ - { name = "referencing" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "jsonschema_specifications-2025.9.1.tar.gz" -upload-time = 2025-09-08 01:34:59.186960+00:00 -url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz" -size = 32855 - -[packages.sdist.hashes] -sha256 = "b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d" - -[[packages.wheels]] -name = "jsonschema_specifications-2025.9.1-py3-none-any.whl" -upload-time = 2025-09-08 01:34:57.871954+00:00 -url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl" -size = 18437 - -[packages.wheels.hashes] -sha256 = "98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe" +sha256 = "326ec5b6800552ccd76a1acca131f70511ac013b377230bdd541db3244adc9fa" [[packages]] name = "keyring" @@ -1319,31 +1111,6 @@ size = 39160 [packages.wheels.hashes] sha256 = "be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f" -[[packages]] -name = "lark" -version = "1.3.1" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.8" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "lark-1.3.1.tar.gz" -upload-time = 2025-10-27 18:25:56.653447+00:00 -url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz" -size = 382732 - -[packages.sdist.hashes] -sha256 = "b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905" - -[[packages.wheels]] -name = "lark-1.3.1-py3-none-any.whl" -upload-time = 2025-10-27 18:25:54.882451+00:00 -url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl" -size = 113151 - -[packages.wheels.hashes] -sha256 = "c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12" - [[packages]] name = "license-expression" version = "30.4.4" @@ -1615,7 +1382,7 @@ sha256 = "bc0fb085d8ed0d67a6471892535f3d9b795eb911b7cd129d3a24967aab7a538f" [[packages]] name = "packageurl-python" version = "0.17.6" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"project-runtime\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" +marker = "\"project-runtime\" in dependency_groups" requires-python = ">=3.8" index = "https://pypi.org/simple/" @@ -1872,34 +1639,6 @@ size = 155560 [packages.wheels.hashes] sha256 = "076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9" -[[packages]] -name = "py-serializable" -version = "2.1.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = "<4.0,>=3.8" -dependencies = [ - { name = "defusedxml" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "py_serializable-2.1.0.tar.gz" -upload-time = 2025-07-21 09:56:48.070686+00:00 -url = "https://files.pythonhosted.org/packages/73/21/d250cfca8ff30c2e5a7447bc13861541126ce9bd4426cd5d0c9f08b5547d/py_serializable-2.1.0.tar.gz" -size = 52368 - -[packages.sdist.hashes] -sha256 = "9d5db56154a867a9b897c0163b33a793c804c80cee984116d02d49e4578fc103" - -[[packages.wheels]] -name = "py_serializable-2.1.0-py3-none-any.whl" -upload-time = 2025-07-21 09:56:46.848289+00:00 -url = "https://files.pythonhosted.org/packages/9b/bf/7595e817906a29453ba4d99394e781b6fabe55d21f3c15d240f85dd06bb1/py_serializable-2.1.0-py3-none-any.whl" -size = 23045 - -[packages.wheels.hashes] -sha256 = "b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304" - [[packages]] name = "pycparser" version = "3.12" @@ -2151,35 +1890,6 @@ size = 11248 [packages.wheels.hashes] sha256 = "1b82aa4b37137248bd687055e109723debeb07b489de1565297dc9589c5f2d6b" -[[packages]] -name = "python-dateutil" -version = "3.0.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -dependencies = [ - { name = "six" }, - { name = "tzdata" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "python_dateutil-3.0.0.tar.gz" -upload-time = 2026-10-21 23:41:45.075347+00:00 -url = "https://files.pythonhosted.org/packages/0e/df/f15f80383f152452d8985b36d95cb8751f901a3e0ff4cd27f7e118f168e2/python_dateutil-3.0.0.tar.gz" -size = 271066 - -[packages.sdist.hashes] -sha256 = "7ebcb3c29a92f3ceafe79a7a48a5c45677e9e004bbeec5742ac0bbb00560b7a2" - -[[packages.wheels]] -name = "python_dateutil-3.0.0-py2.py3-none-any.whl" -upload-time = 2026-10-21 23:41:43.431814+00:00 -url = "https://files.pythonhosted.org/packages/5a/17/cebee01d1c0c1441174cf0456ddc91b280d79acb65340c05e41d77866ee8/python_dateutil-3.0.0-py2.py3-none-any.whl" -size = 94659 - -[packages.wheels.hashes] -sha256 = "3c79630202e4152c6add678446984236efe4f57e84d8e892aab0637bab82e9fd" - [[packages]] name = "python-discovery" version = "1.9.0" @@ -2282,35 +1992,6 @@ size = 14239 [packages.wheels.hashes] sha256 = "d0dae1f74bb273b534770cb4cccb6bb78735540afdb03c2146f4e19dcd412560" -[[packages]] -name = "referencing" -version = "0.37.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.10" -dependencies = [ - { name = "attrs" }, - { name = "rpds-py" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "referencing-0.37.0.tar.gz" -upload-time = 2025-10-13 15:30:48.871335+00:00 -url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz" -size = 78036 - -[packages.sdist.hashes] -sha256 = "44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8" - -[[packages.wheels]] -name = "referencing-0.37.0-py3-none-any.whl" -upload-time = 2025-10-13 15:30:47.625937+00:00 -url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl" -size = 26766 - -[packages.wheels.hashes] -sha256 = "381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231" - [[packages]] name = "release-creator" version = "4.4.0" @@ -2403,34 +2084,6 @@ size = 54481 [packages.wheels.hashes] sha256 = "cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06" -[[packages]] -name = "rfc3339-validator" -version = "0.1.4" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -dependencies = [ - { name = "six" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "rfc3339_validator-0.1.4.tar.gz" -upload-time = 2021-05-12 16:37:54.178122+00:00 -url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz" -size = 5513 - -[packages.sdist.hashes] -sha256 = "138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b" - -[[packages.wheels]] -name = "rfc3339_validator-0.1.4-py2.py3-none-any.whl" -upload-time = 2021-05-12 16:37:52.536885+00:00 -url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl" -size = 3490 - -[packages.wheels.hashes] -sha256 = "24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa" - [[packages]] name = "rfc3986" version = "2.0.0" @@ -2456,59 +2109,6 @@ size = 31326 [packages.wheels.hashes] sha256 = "50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd" -[[packages]] -name = "rfc3986-validator" -version = "0.1.1" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "rfc3986_validator-0.1.1.tar.gz" -upload-time = 2019-10-28 16:00:19.144483+00:00 -url = "https://files.pythonhosted.org/packages/da/88/f270de456dd7d11dcc808abfa291ecdd3f45ff44e3b549ffa01b126464d0/rfc3986_validator-0.1.1.tar.gz" -size = 6760 - -[packages.sdist.hashes] -sha256 = "3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055" - -[[packages.wheels]] -name = "rfc3986_validator-0.1.1-py2.py3-none-any.whl" -upload-time = 2019-10-28 16:00:13.976195+00:00 -url = "https://files.pythonhosted.org/packages/9e/51/17023c0f8f1869d8806b979a2bffa3f861f26a3f1a66b094288323fba52f/rfc3986_validator-0.1.1-py2.py3-none-any.whl" -size = 4242 - -[packages.wheels.hashes] -sha256 = "2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9" - -[[packages]] -name = "rfc3987-syntax" -version = "1.1.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.9" -dependencies = [ - { name = "lark" }, -] -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "rfc3987_syntax-1.1.0.tar.gz" -upload-time = 2025-07-18 01:05:05.015375+00:00 -url = "https://files.pythonhosted.org/packages/2c/06/37c1a5557acf449e8e406a830a05bf885ac47d33270aec454ef78675008d/rfc3987_syntax-1.1.0.tar.gz" -size = 14239 - -[packages.sdist.hashes] -sha256 = "717a62cbf33cffdd16dfa3a497d81ce48a660ea691b1ddd7be710c22f00b4a0d" - -[[packages.wheels]] -name = "rfc3987_syntax-1.1.0-py3-none-any.whl" -upload-time = 2025-07-18 01:05:03.843605+00:00 -url = "https://files.pythonhosted.org/packages/7e/71/44ce230e1b7fadd372515a97e32a83011f906ddded8d03e3c6aafbdedbb7/rfc3987_syntax-1.1.0-py3-none-any.whl" -size = 8046 - -[packages.wheels.hashes] -sha256 = "6c3d97604e4c5ce9f714898e05401a0445a641cfa276432b0a648c80856f6a3f" - [[packages]] name = "rich" version = "15.0.0" @@ -2563,31 +2163,6 @@ size = 7676 [packages.wheels.hashes] sha256 = "647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7" -[[packages]] -name = "rpds-py" -version = "2026.9.1" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.11" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "rpds_py-2026.9.1.tar.gz" -upload-time = 2026-10-04 16:32:36.469775+00:00 -url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz" -size = 63948 - -[packages.sdist.hashes] -sha256 = "4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12" - -[[packages.wheels]] -name = "rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" -upload-time = 2026-10-04 16:30:58.295504+00:00 -url = "https://files.pythonhosted.org/packages/57/ca/e2e9a0a46a74ed51a0498ba1fe10f4ea6b2d9a155f372a2f91e51f18cf10/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" -size = 372022 - -[packages.wheels.hashes] -sha256 = "2c16ab111bc27c646ba8aa005d0527754edc538ebb636f0b1bf8e244b48d1945" - [[packages]] name = "secretstorage" version = "3.5.0" @@ -2697,31 +2272,6 @@ size = 33366 [packages.wheels.hashes] sha256 = "82f34c3e3084fc2b57d397200637cc13f2338004052d2fee47baa5f0e902464e" -[[packages]] -name = "six" -version = "1.17.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "six-1.17.0.tar.gz" -upload-time = 2024-12-04 17:35:28.174320+00:00 -url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz" -size = 34031 - -[packages.sdist.hashes] -sha256 = "ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" - -[[packages.wheels]] -name = "six-1.17.0-py2.py3-none-any.whl" -upload-time = 2024-12-04 17:35:26.475808+00:00 -url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl" -size = 11050 - -[packages.wheels.hashes] -sha256 = "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274" - [[packages]] name = "snowballstemmer" version = "3.1.1" @@ -2747,30 +2297,6 @@ size = 104164 [packages.wheels.hashes] sha256 = "7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752" -[[packages]] -name = "sortedcontainers" -version = "2.4.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "sortedcontainers-2.4.0.tar.gz" -upload-time = 2021-05-16 22:03:42.897871+00:00 -url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz" -size = 30594 - -[packages.sdist.hashes] -sha256 = "25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88" - -[[packages.wheels]] -name = "sortedcontainers-2.4.0-py2.py3-none-any.whl" -upload-time = 2021-05-16 22:03:41.177807+00:00 -url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl" -size = 29575 - -[packages.wheels.hashes] -sha256 = "a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" - [[packages]] name = "soupsieve" version = "3.0.4" @@ -3331,56 +2857,6 @@ size = 14750 [packages.wheels.hashes] sha256 = "65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147" -[[packages]] -name = "tzdata" -version = "2026.6" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=2" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "tzdata-2026.6.tar.gz" -upload-time = 2026-10-28 09:57:09.137703+00:00 -url = "https://files.pythonhosted.org/packages/7a/29/6a1391d31cd00ff9e927215a224a880e1e84e826b64ba80a499e51c074d8/tzdata-2026.6.tar.gz" -size = 201017 - -[packages.sdist.hashes] -sha256 = "85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d" - -[[packages.wheels]] -name = "tzdata-2026.6-py2.py3-none-any.whl" -upload-time = 2026-10-28 09:57:07.276552+00:00 -url = "https://files.pythonhosted.org/packages/0a/66/9a06c2227fedfd4919f0eaaa8b94377d9730882a7e4f95823c9a331ab5e1/tzdata-2026.6-py2.py3-none-any.whl" -size = 348210 - -[packages.wheels.hashes] -sha256 = "0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec" - -[[packages]] -name = "uri-template" -version = "1.3.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.7" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "uri-template-1.3.0.tar.gz" -upload-time = 2023-06-21 01:49:05.374755+00:00 -url = "https://files.pythonhosted.org/packages/31/c7/0336f2bd0bcbada6ccef7aaa25e443c118a704f828a0620c6fa0207c1b64/uri-template-1.3.0.tar.gz" -size = 21678 - -[packages.sdist.hashes] -sha256 = "0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7" - -[[packages.wheels]] -name = "uri_template-1.3.0-py3-none-any.whl" -upload-time = 2023-06-21 01:49:03.467606+00:00 -url = "https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl" -size = 11140 - -[packages.wheels.hashes] -sha256 = "a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363" - [[packages]] name = "uritools" version = "6.1.3" @@ -3550,31 +3026,6 @@ size = 301806 [packages.wheels.hashes] sha256 = "901eace6d2dd25ebb917d0ce5db18e5c537e1504c905cf32f0d006c12098b05d" -[[packages]] -name = "webcolors" -version = "25.10.0" -marker = "\"bench\" in dependency_groups or \"dev\" in dependency_groups or \"test\" in dependency_groups or \"type\" in dependency_groups or \"validate\" in extras" -requires-python = ">=3.10" -index = "https://pypi.org/simple/" - -[packages.sdist] -name = "webcolors-25.10.0.tar.gz" -upload-time = 2025-10-31 07:51:03.977878+00:00 -url = "https://files.pythonhosted.org/packages/1d/7a/eb316761ec35664ea5174709a68bbd3389de60d4a1ebab8808bfc264ed67/webcolors-25.10.0.tar.gz" -size = 53491 - -[packages.sdist.hashes] -sha256 = "62abae86504f66d0f6364c2a8520de4a0c47b80c03fc3a5f1815fedbef7c19bf" - -[[packages.wheels]] -name = "webcolors-25.10.0-py3-none-any.whl" -upload-time = 2025-10-31 07:51:01.778455+00:00 -url = "https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl" -size = 14905 - -[packages.wheels.hashes] -sha256 = "032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d" - [[packages]] name = "wheel-filename" version = "1.4.2" diff --git a/pyproject.toml b/pyproject.toml index 874e89c..94149dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,6 @@ optional-dependencies.hatch = [ "hatchling>=1.28", ] optional-dependencies.validate = [ - "cyclonedx-python-lib>=11.12", "jsonschema-rs>=0.60.3", "license-expression>=30", "spdx-tools>=0.8.5", @@ -80,7 +79,6 @@ dev = [ ] test = [ "covdefaults>=2.3", - "cyclonedx-python-lib>=11.12", "hatchling>=1.28", "installer>=1.1", "jsonschema-rs>=0.60.3", diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index efa44f9..5f433d3 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -1,13 +1,12 @@ from __future__ import annotations import json -from pathlib import Path +from importlib.resources import files from typing import TYPE_CHECKING, Final, TypeAlias from python_sbom._errors import MissingExtraError try: - from cyclonedx.schema._res import CRYPTOGRAPHY_DEFS, JSF, SPDX_JSON # ruff: ignore[import-private-name] # its bundled schemas from jsonschema_rs import Draft7Validator, Registry from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser @@ -17,6 +16,8 @@ raise MissingExtraError(_MESSAGE) from error if TYPE_CHECKING: + from importlib.resources.abc import Traversable + from jsonschema_rs import ValidationError from python_sbom._formats._spdx import Spdx @@ -46,19 +47,21 @@ def spdx_problems(spdx: Spdx) -> list[str]: return [f"SPDX rendering invalid: {message.validation_message}" for message in validate_full_spdx_document(parsed)] -# the schema and the files it references are the ones cyclonedx's own Draft 7 validator reads, with format assertion on -# as there; offline, so a reference missing from the registry fails instead of reaching the network +# the official CycloneDX schema files, unchanged: bom-1.6 and the jsf-0.82 it references from the specification's 1.6.2 +# release, and the spdx license list from its 1.7.2 release, the newest, as the cyclonedx.org URL every schema version +# references serves the newest; Apache-2.0, see the LICENSE beside them. Format assertion is on, as in CycloneDX's own +# Draft 7 validation, and offline, so a reference missing from the registry fails instead of reaching the network +_SCHEMAS: Final[Traversable] = files(__package__) / "schemas" +_REFERENCED: Final[list[dict[str, _Json]]] = [ + json.loads((_SCHEMAS / name).read_text(encoding="utf-8")) for name in ("spdx.schema.json", "jsf-0.82.schema.json") +] _VALIDATOR: Final[Draft7Validator] = Draft7Validator( - json.loads(Path(SPDX_JSON).with_name("bom-1.6.SNAPSHOT.schema.json").read_text(encoding="utf-8")), + json.loads((_SCHEMAS / "bom-1.6.schema.json").read_text(encoding="utf-8")), validate_formats=True, offline=True, - registry=Registry([ - (f"http://cyclonedx.org/schema/{Path(path).name}", json.loads(Path(path).read_text(encoding="utf-8"))) - for path in (SPDX_JSON, CRYPTOGRAPHY_DEFS, JSF) - ]), + registry=Registry([(str(schema["$id"]), schema) for schema in _REFERENCED]), ) - __all__ = [ "schema_error", "spdx_problems", diff --git a/src/python_sbom/_validate/schemas/LICENSE b/src/python_sbom/_validate/schemas/LICENSE new file mode 100644 index 0000000..0808751 --- /dev/null +++ b/src/python_sbom/_validate/schemas/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright OWASP Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/python_sbom/_validate/schemas/bom-1.6.schema.json b/src/python_sbom/_validate/schemas/bom-1.6.schema.json new file mode 100644 index 0000000..b6c096a --- /dev/null +++ b/src/python_sbom/_validate/schemas/bom-1.6.schema.json @@ -0,0 +1,5699 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "type": "object", + "title": "CycloneDX Bill of Materials Standard", + "$comment" : "CycloneDX JSON schema is published under the terms of the Apache License 2.0.", + "required": [ + "bomFormat", + "specVersion" + ], + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string" + }, + "bomFormat": { + "type": "string", + "title": "BOM Format", + "description": "Specifies the format of the BOM. This helps to identify the file as CycloneDX since BOMs do not have a filename convention, nor does JSON schema support namespaces. This value must be \"CycloneDX\".", + "enum": [ + "CycloneDX" + ] + }, + "specVersion": { + "type": "string", + "title": "CycloneDX Specification Version", + "description": "The version of the CycloneDX specification the BOM conforms to.", + "examples": ["1.6"] + }, + "serialNumber": { + "type": "string", + "title": "BOM Serial Number", + "description": "Every BOM generated SHOULD have a unique serial number, even if the contents of the BOM have not changed over time. If specified, the serial number must conform to [RFC 4122](https://www.ietf.org/rfc/rfc4122.html). Use of serial numbers is recommended.", + "examples": ["urn:uuid:3e671687-395b-41f5-a30f-a58921a69b79"], + "pattern": "^urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + }, + "version": { + "type": "integer", + "title": "BOM Version", + "description": "Whenever an existing BOM is modified, either manually or through automated processes, the version of the BOM SHOULD be incremented by 1. When a system is presented with multiple BOMs with identical serial numbers, the system SHOULD use the most recent version of the BOM. The default version is '1'.", + "minimum": 1, + "default": 1, + "examples": [1] + }, + "metadata": { + "$ref": "#/definitions/metadata", + "title": "BOM Metadata", + "description": "Provides additional information about a BOM." + }, + "components": { + "type": "array", + "items": {"$ref": "#/definitions/component"}, + "uniqueItems": true, + "title": "Components", + "description": "A list of software and hardware components." + }, + "services": { + "type": "array", + "items": {"$ref": "#/definitions/service"}, + "uniqueItems": true, + "title": "Services", + "description": "A list of services. This may include microservices, function-as-a-service, and other types of network or intra-process services." + }, + "externalReferences": { + "type": "array", + "items": {"$ref": "#/definitions/externalReference"}, + "title": "External References", + "description": "External references provide a way to document systems, sites, and information that may be relevant but are not included with the BOM. They may also establish specific relationships within or external to the BOM." + }, + "dependencies": { + "type": "array", + "items": {"$ref": "#/definitions/dependency"}, + "uniqueItems": true, + "title": "Dependencies", + "description": "Provides the ability to document dependency relationships including provided & implemented components." + }, + "compositions": { + "type": "array", + "items": {"$ref": "#/definitions/compositions"}, + "uniqueItems": true, + "title": "Compositions", + "description": "Compositions describe constituent parts (including components, services, and dependency relationships) and their completeness. The completeness of vulnerabilities expressed in a BOM may also be described." + }, + "vulnerabilities": { + "type": "array", + "items": {"$ref": "#/definitions/vulnerability"}, + "uniqueItems": true, + "title": "Vulnerabilities", + "description": "Vulnerabilities identified in components or services." + }, + "annotations": { + "type": "array", + "items": {"$ref": "#/definitions/annotations"}, + "uniqueItems": true, + "title": "Annotations", + "description": "Comments made by people, organizations, or tools about any object with a bom-ref, such as components, services, vulnerabilities, or the BOM itself. Unlike inventory information, annotations may contain opinions or commentary from various stakeholders. Annotations may be inline (with inventory) or externalized via BOM-Link and may optionally be signed." + }, + "formulation": { + "type": "array", + "items": {"$ref": "#/definitions/formula"}, + "uniqueItems": true, + "title": "Formulation", + "description": "Describes how a component or service was manufactured or deployed. This is achieved through the use of formulas, workflows, tasks, and steps, which declare the precise steps to reproduce along with the observed formulas describing the steps which transpired in the manufacturing process." + }, + "declarations": { + "type": "object", + "title": "Declarations", + "description": "The list of declarations which describe the conformance to standards. Each declaration may include attestations, claims, and evidence.", + "additionalProperties": false, + "properties": { + "assessors": { + "type": "array", + "title": "Assessors", + "description": "The list of assessors evaluating claims and determining conformance to requirements and confidence in that assessment.", + "items": { + "type": "object", + "title": "Assessor", + "description": "The assessor who evaluates claims and determines conformance to requirements and confidence in that assessment.", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the object elsewhere in the BOM. Every bom-ref must be unique within the BOM." + }, + "thirdParty": { + "type": "boolean", + "title": "Third Party", + "description": "The boolean indicating if the assessor is outside the organization generating claims. A value of false indicates a self assessor." + }, + "organization": { + "$ref": "#/definitions/organizationalEntity", + "title": "Organization", + "description": "The entity issuing the assessment." + } + } + } + }, + "attestations": { + "type": "array", + "title": "Attestations", + "description": "The list of attestations asserted by an assessor that maps requirements to claims.", + "items": { + "type": "object", + "title": "Attestation", + "additionalProperties": false, + "properties": { + "summary": { + "type": "string", + "title": "Summary", + "description": "The short description explaining the main points of the attestation." + }, + "assessor": { + "$ref": "#/definitions/refLinkType", + "title": "Assessor", + "description": "The `bom-ref` to the assessor asserting the attestation." + }, + "map": { + "type": "array", + "title": "Map", + "description": "The grouping of requirements to claims and the attestors declared conformance and confidence thereof.", + "items": { + "type": "object", + "title": "Map", + "additionalProperties": false, + "properties": { + "requirement": { + "$ref": "#/definitions/refLinkType", + "title": "Requirement", + "description": "The `bom-ref` to the requirement being attested to." + }, + "claims": { + "type": "array", + "title": "Claims", + "description": "The list of `bom-ref` to the claims being attested to.", + "items": { "$ref": "#/definitions/refLinkType" } + }, + "counterClaims": { + "type": "array", + "title": "Counter Claims", + "description": "The list of `bom-ref` to the counter claims being attested to.", + "items": { "$ref": "#/definitions/refLinkType" } + }, + "conformance": { + "type": "object", + "title": "Conformance", + "description": "The conformance of the claim meeting a requirement.", + "additionalProperties": false, + "properties": { + "score": { + "type": "number", + "minimum": 0, + "maximum": 1, + "title": "Score", + "description": "The conformance of the claim between and inclusive of 0 and 1, where 1 is 100% conformance." + }, + "rationale": { + "type": "string", + "title": "Rationale", + "description": "The rationale for the conformance score." + }, + "mitigationStrategies": { + "type": "array", + "title": "Mitigation Strategies", + "description": "The list of `bom-ref` to the evidence provided describing the mitigation strategies.", + "items": { "$ref": "#/definitions/refLinkType" } + } + } + }, + "confidence": { + "type": "object", + "title": "Confidence", + "description": "The confidence of the claim meeting the requirement.", + "additionalProperties": false, + "properties": { + "score": { + "type": "number", + "minimum": 0, + "maximum": 1, + "title": "Score", + "description": "The confidence of the claim between and inclusive of 0 and 1, where 1 is 100% confidence." + }, + "rationale": { + "type": "string", + "title": "Rationale", + "description": "The rationale for the confidence score." + } + } + } + } + } + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + } + }, + "claims": { + "type": "array", + "title": "Claims", + "description": "The list of claims.", + "items": { + "type": "object", + "title": "Claim", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the object elsewhere in the BOM. Every bom-ref must be unique within the BOM." + }, + "target": { + "$ref": "#/definitions/refLinkType", + "title": "Target", + "description": "The `bom-ref` to a target representing a specific system, application, API, module, team, person, process, business unit, company, etc... that this claim is being applied to." + }, + "predicate": { + "type": "string", + "title": "Predicate", + "description": "The specific statement or assertion about the target." + }, + "mitigationStrategies": { + "type": "array", + "title": "Mitigation Strategies", + "description": "The list of `bom-ref` to the evidence provided describing the mitigation strategies. Each mitigation strategy should include an explanation of how any weaknesses in the evidence will be mitigated.", + "items": { "$ref": "#/definitions/refLinkType" } + }, + "reasoning": { + "type": "string", + "title": "Reasoning", + "description": "The written explanation of why the evidence provided substantiates the claim." + }, + "evidence": { + "type": "array", + "title": "Evidence", + "description": "The list of `bom-ref` to evidence that supports this claim.", + "items": { "$ref": "#/definitions/refLinkType" } + }, + "counterEvidence": { + "type": "array", + "title": "Counter Evidence", + "description": "The list of `bom-ref` to counterEvidence that supports this claim.", + "items": { "$ref": "#/definitions/refLinkType" } + }, + "externalReferences": { + "type": "array", + "items": {"$ref": "#/definitions/externalReference"}, + "title": "External References", + "description": "External references provide a way to document systems, sites, and information that may be relevant but are not included with the BOM. They may also establish specific relationships within or external to the BOM." + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + } + }, + "evidence": { + "type": "array", + "title": "Evidence", + "description": "The list of evidence", + "items": { + "type": "object", + "title": "Evidence", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the object elsewhere in the BOM. Every bom-ref must be unique within the BOM." + }, + "propertyName": { + "type": "string", + "title": "Property Name", + "description": "The reference to the property name as defined in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy/)." + }, + "description": { + "type": "string", + "title": "Description", + "description": "The written description of what this evidence is and how it was created." + }, + "data": { + "type": "array", + "title": "Data", + "description": "The output or analysis that supports claims.", + "items": { + "type": "object", + "title": "Data", + "additionalProperties": false, + "properties": { + "name": { + "title": "Data Name", + "description": "The name of the data.", + "type": "string" + }, + "contents": { + "type": "object", + "title": "Data Contents", + "description": "The contents or references to the contents of the data being described.", + "additionalProperties": false, + "properties": { + "attachment": { + "title": "Data Attachment", + "description": "An optional way to include textual or encoded data.", + "$ref": "#/definitions/attachment" + }, + "url": { + "type": "string", + "title": "Data URL", + "description": "The URL to where the data can be retrieved.", + "format": "iri-reference" + } + } + }, + "classification": { + "$ref": "#/definitions/dataClassification" + }, + "sensitiveData": { + "type": "array", + "title": "Sensitive Data", + "description": "A description of any sensitive data included.", + "items": { + "type": "string" + } + }, + "governance": { + "title": "Data Governance", + "$ref": "#/definitions/dataGovernance" + } + } + } + }, + "created": { + "type": "string", + "format": "date-time", + "title": "Created", + "description": "The date and time (timestamp) when the evidence was created." + }, + "expires": { + "type": "string", + "format": "date-time", + "title": "Expires", + "description": "The optional date and time (timestamp) when the evidence is no longer valid." + }, + "author": { + "$ref": "#/definitions/organizationalContact", + "title": "Author", + "description": "The author of the evidence." + }, + "reviewer": { + "$ref": "#/definitions/organizationalContact", + "title": "Reviewer", + "description": "The reviewer of the evidence." + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + } + }, + "targets": { + "type": "object", + "title": "Targets", + "description": "The list of targets which claims are made against.", + "additionalProperties": false, + "properties": { + "organizations": { + "type": "array", + "title": "Organizations", + "description": "The list of organizations which claims are made against.", + "items": {"$ref": "#/definitions/organizationalEntity"} + }, + "components": { + "type": "array", + "title": "Components", + "description": "The list of components which claims are made against.", + "items": {"$ref": "#/definitions/component"} + }, + "services": { + "type": "array", + "title": "Services", + "description": "The list of services which claims are made against.", + "items": {"$ref": "#/definitions/service"} + } + } + }, + "affirmation": { + "type": "object", + "title": "Affirmation", + "description": "A concise statement affirmed by an individual regarding all declarations, often used for third-party auditor acceptance or recipient acknowledgment. It includes a list of authorized signatories who assert the validity of the document on behalf of the organization.", + "additionalProperties": false, + "properties": { + "statement": { + "type": "string", + "title": "Statement", + "description": "The brief statement affirmed by an individual regarding all declarations.\n*- Notes This could be an affirmation of acceptance by a third-party auditor or receiving individual of a file.", + "examples": [ "I certify, to the best of my knowledge, that all information is correct." ] + }, + "signatories": { + "type": "array", + "title": "Signatories", + "description": "The list of signatories authorized on behalf of an organization to assert validity of this document.", + "items": { + "type": "object", + "title": "Signatory", + "additionalProperties": false, + "oneOf": [ + { + "required": ["signature"] + }, + { + "required": ["externalReference", "organization"] + } + ], + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "The signatory's name." + }, + "role": { + "type": "string", + "title": "Role", + "description": "The signatory's role within an organization." + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + }, + "organization": { + "$ref": "#/definitions/organizationalEntity", + "title": "Organization", + "description": "The signatory's organization." + }, + "externalReference": { + "$ref": "#/definitions/externalReference", + "title": "External Reference", + "description": "External references provide a way to document systems, sites, and information that may be relevant but are not included with the BOM. They may also establish specific relationships within or external to the BOM." + } + } + } + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + }, + "definitions": { + "type": "object", + "title": "Definitions", + "description": "A collection of reusable objects that are defined and may be used elsewhere in the BOM.", + "additionalProperties": false, + "properties": { + "standards": { + "type": "array", + "title": "Standards", + "description": "The list of standards which may consist of regulations, industry or organizational-specific standards, maturity models, best practices, or any other requirements which can be evaluated against or attested to.", + "items": { + "$ref": "#/definitions/standard" + } + } + } + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + }, + "definitions": { + "refType": { + "description": "Identifier for referable and therefore interlinkable elements.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links.", + "type": "string", + "minLength": 1, + "$comment": "TODO (breaking change): add a format constraint that prevents the value from starting with 'urn:cdx:'" + }, + "refLinkType": { + "description": "Descriptor for an element identified by the attribute 'bom-ref' in the same BOM document.\nIn contrast to `bomLinkElementType`.", + "$ref": "#/definitions/refType" + }, + "bomLinkDocumentType": { + "title": "BOM-Link Document", + "description": "Descriptor for another BOM document. See https://cyclonedx.org/capabilities/bomlink/", + "type": "string", + "format": "iri-reference", + "pattern": "^urn:cdx:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/[1-9][0-9]*$", + "$comment": "part of the pattern is based on `bom.serialNumber`'s pattern" + }, + "bomLinkElementType": { + "title": "BOM-Link Element", + "description": "Descriptor for an element in a BOM document. See https://cyclonedx.org/capabilities/bomlink/", + "type": "string", + "format": "iri-reference", + "pattern": "^urn:cdx:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/[1-9][0-9]*#.+$", + "$comment": "part of the pattern is based on `bom.serialNumber`'s pattern" + }, + "bomLink": { + "title": "BOM-Link", + "anyOf": [ + { + "title": "BOM-Link Document", + "$ref": "#/definitions/bomLinkDocumentType" + }, + { + "title": "BOM-Link Element", + "$ref": "#/definitions/bomLinkElementType" + } + ] + }, + "metadata": { + "type": "object", + "title": "BOM Metadata", + "additionalProperties": false, + "properties": { + "timestamp": { + "type": "string", + "format": "date-time", + "title": "Timestamp", + "description": "The date and time (timestamp) when the BOM was created." + }, + "lifecycles": { + "type": "array", + "title": "Lifecycles", + "description": "Lifecycles communicate the stage(s) in which data in the BOM was captured. Different types of data may be available at various phases of a lifecycle, such as the Software Development Lifecycle (SDLC), IT Asset Management (ITAM), and Software Asset Management (SAM). Thus, a BOM may include data specific to or only obtainable in a given lifecycle.", + "items": { + "type": "object", + "title": "Lifecycle", + "description": "The product lifecycle(s) that this BOM represents.", + "oneOf": [ + { + "title": "Pre-Defined Phase", + "required": ["phase"], + "additionalProperties": false, + "properties": { + "phase": { + "type": "string", + "title": "Phase", + "description": "A pre-defined phase in the product lifecycle.", + "enum": [ + "design", + "pre-build", + "build", + "post-build", + "operations", + "discovery", + "decommission" + ], + "meta:enum": { + "design": "BOM produced early in the development lifecycle containing an inventory of components and services that are proposed or planned to be used. The inventory may need to be procured, retrieved, or resourced prior to use.", + "pre-build": "BOM consisting of information obtained prior to a build process and may contain source files and development artifacts and manifests. The inventory may need to be resolved and retrieved prior to use.", + "build": "BOM consisting of information obtained during a build process where component inventory is available for use. The precise versions of resolved components are usually available at this time as well as the provenance of where the components were retrieved from.", + "post-build": "BOM consisting of information obtained after a build process has completed and the resulting components(s) are available for further analysis. Built components may exist as the result of a CI/CD process, may have been installed or deployed to a system or device, and may need to be retrieved or extracted from the system or device.", + "operations": "BOM produced that represents inventory that is running and operational. This may include staging or production environments and will generally encompass multiple SBOMs describing the applications and operating system, along with HBOMs describing the hardware that makes up the system. Operations Bill of Materials (OBOM) can provide full-stack inventory of runtime environments, configurations, and additional dependencies.", + "discovery": "BOM consisting of information observed through network discovery providing point-in-time enumeration of embedded, on-premise, and cloud-native services such as server applications, connected devices, microservices, and serverless functions.", + "decommission": "BOM containing inventory that will be, or has been retired from operations." + } + } + } + }, + { + "title": "Custom Phase", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "The name of the lifecycle phase" + }, + "description": { + "type": "string", + "title": "Description", + "description": "The description of the lifecycle phase" + } + } + } + ] + } + }, + "tools": { + "title": "Tools", + "description": "The tool(s) used in the creation, enrichment, and validation of the BOM.", + "oneOf": [ + { + "type": "object", + "title": "Tools", + "description": "The tool(s) used in the creation, enrichment, and validation of the BOM.", + "additionalProperties": false, + "properties": { + "components": { + "type": "array", + "items": {"$ref": "#/definitions/component"}, + "uniqueItems": true, + "title": "Components", + "description": "A list of software and hardware components used as tools." + }, + "services": { + "type": "array", + "items": {"$ref": "#/definitions/service"}, + "uniqueItems": true, + "title": "Services", + "description": "A list of services used as tools. This may include microservices, function-as-a-service, and other types of network or intra-process services." + } + } + }, + { + "type": "array", + "title": "Tools (legacy)", + "description": "[Deprecated] The tool(s) used in the creation, enrichment, and validation of the BOM.", + "items": {"$ref": "#/definitions/tool"} + } + ] + }, + "manufacturer": { + "title": "BOM Manufacturer", + "description": "The organization that created the BOM.\nManufacturer is common in BOMs created through automated processes. BOMs created through manual means may have `@.authors` instead.", + "$ref": "#/definitions/organizationalEntity" + }, + "authors": { + "type": "array", + "title": "BOM Authors", + "description": "The person(s) who created the BOM.\nAuthors are common in BOMs created through manual processes. BOMs created through automated means may have `@.manufacturer` instead.", + "items": {"$ref": "#/definitions/organizationalContact"} + }, + "component": { + "title": "Component", + "description": "The component that the BOM describes.", + "$ref": "#/definitions/component" + }, + "manufacture": { + "deprecated": true, + "title": "Component Manufacture (legacy)", + "description": "[Deprecated] This will be removed in a future version. Use the `@.component.manufacturer` instead.\nThe organization that manufactured the component that the BOM describes.", + "$ref": "#/definitions/organizationalEntity" + }, + "supplier": { + "title": "Supplier", + "description": " The organization that supplied the component that the BOM describes. The supplier may often be the manufacturer, but may also be a distributor or repackager.", + "$ref": "#/definitions/organizationalEntity" + }, + "licenses": { + "title": "BOM License(s)", + "description": "The license information for the BOM document.\nThis may be different from the license(s) of the component(s) that the BOM describes.", + "$ref": "#/definitions/licenseChoice" + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": {"$ref": "#/definitions/property"} + } + } + }, + "tool": { + "type": "object", + "title": "Tool", + "description": "[Deprecated] This will be removed in a future version. Use component or service instead. Information about the automated or manual tool used", + "additionalProperties": false, + "properties": { + "vendor": { + "type": "string", + "title": "Tool Vendor", + "description": "The name of the vendor who created the tool" + }, + "name": { + "type": "string", + "title": "Tool Name", + "description": "The name of the tool" + }, + "version": { + "$ref": "#/definitions/version", + "title": "Tool Version", + "description": "The version of the tool" + }, + "hashes": { + "type": "array", + "items": {"$ref": "#/definitions/hash"}, + "title": "Hashes", + "description": "The hashes of the tool (if applicable)." + }, + "externalReferences": { + "type": "array", + "items": {"$ref": "#/definitions/externalReference"}, + "title": "External References", + "description": "External references provide a way to document systems, sites, and information that may be relevant, but are not included with the BOM. They may also establish specific relationships within or external to the BOM." + } + } + }, + "organizationalEntity": { + "type": "object", + "title": "Organizational Entity", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the object elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "name": { + "type": "string", + "title": "Organization Name", + "description": "The name of the organization", + "examples": [ + "Example Inc." + ] + }, + "address": { + "$ref": "#/definitions/postalAddress", + "title": "Organization Address", + "description": "The physical address (location) of the organization" + }, + "url": { + "type": "array", + "items": { + "type": "string", + "format": "iri-reference" + }, + "title": "Organization URL(s)", + "description": "The URL of the organization. Multiple URLs are allowed.", + "examples": ["https://example.com"] + }, + "contact": { + "type": "array", + "title": "Organizational Contact", + "description": "A contact at the organization. Multiple contacts are allowed.", + "items": {"$ref": "#/definitions/organizationalContact"} + } + } + }, + "organizationalContact": { + "type": "object", + "title": "Organizational Contact", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the object elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "name": { + "type": "string", + "title": "Name", + "description": "The name of a contact", + "examples": ["Contact name"] + }, + "email": { + "type": "string", + "format": "idn-email", + "title": "Email Address", + "description": "The email address of the contact.", + "examples": ["firstname.lastname@example.com"] + }, + "phone": { + "type": "string", + "title": "Phone", + "description": "The phone number of the contact.", + "examples": ["800-555-1212"] + } + } + }, + "component": { + "type": "object", + "title": "Component", + "required": [ + "type", + "name" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "application", + "framework", + "library", + "container", + "platform", + "operating-system", + "device", + "device-driver", + "firmware", + "file", + "machine-learning-model", + "data", + "cryptographic-asset" + ], + "meta:enum": { + "application": "A software application. Refer to [https://en.wikipedia.org/wiki/Application_software](https://en.wikipedia.org/wiki/Application_software) for information about applications.", + "framework": "A software framework. Refer to [https://en.wikipedia.org/wiki/Software_framework](https://en.wikipedia.org/wiki/Software_framework) for information on how frameworks vary slightly from libraries.", + "library": "A software library. Refer to [https://en.wikipedia.org/wiki/Library_(computing)](https://en.wikipedia.org/wiki/Library_(computing)) for information about libraries. All third-party and open source reusable components will likely be a library. If the library also has key features of a framework, then it should be classified as a framework. If not, or is unknown, then specifying library is recommended.", + "container": "A packaging and/or runtime format, not specific to any particular technology, which isolates software inside the container from software outside of a container through virtualization technology. Refer to [https://en.wikipedia.org/wiki/OS-level_virtualization](https://en.wikipedia.org/wiki/OS-level_virtualization).", + "platform": "A runtime environment which interprets or executes software. This may include runtimes such as those that execute bytecode or low-code/no-code application platforms.", + "operating-system": "A software operating system without regard to deployment model (i.e. installed on physical hardware, virtual machine, image, etc) Refer to [https://en.wikipedia.org/wiki/Operating_system](https://en.wikipedia.org/wiki/Operating_system).", + "device": "A hardware device such as a processor or chip-set. A hardware device containing firmware SHOULD include a component for the physical hardware itself and another component of type 'firmware' or 'operating-system' (whichever is relevant), describing information about the software running on the device. See also the list of [known device properties](https://github.com/CycloneDX/cyclonedx-property-taxonomy/blob/main/cdx/device.md).", + "device-driver": "A special type of software that operates or controls a particular type of device. Refer to [https://en.wikipedia.org/wiki/Device_driver](https://en.wikipedia.org/wiki/Device_driver).", + "firmware": "A special type of software that provides low-level control over a device's hardware. Refer to [https://en.wikipedia.org/wiki/Firmware](https://en.wikipedia.org/wiki/Firmware).", + "file": "A computer file. Refer to [https://en.wikipedia.org/wiki/Computer_file](https://en.wikipedia.org/wiki/Computer_file) for information about files.", + "machine-learning-model": "A model based on training data that can make predictions or decisions without being explicitly programmed to do so.", + "data": "A collection of discrete values that convey information.", + "cryptographic-asset": "A cryptographic asset including algorithms, protocols, certificates, keys, tokens, and secrets." + }, + "title": "Component Type", + "description": "Specifies the type of component. For software components, classify as application if no more specific appropriate classification is available or cannot be determined for the component.", + "examples": ["library"] + }, + "mime-type": { + "type": "string", + "title": "Mime-Type", + "description": "The optional mime-type of the component. When used on file components, the mime-type can provide additional context about the kind of file being represented, such as an image, font, or executable. Some library or framework components may also have an associated mime-type.", + "examples": ["image/jpeg"], + "pattern": "^[-+a-z0-9.]+/[-+a-z0-9.]+$" + }, + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the component elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "supplier": { + "title": "Component Supplier", + "description": " The organization that supplied the component. The supplier may often be the manufacturer, but may also be a distributor or repackager.", + "$ref": "#/definitions/organizationalEntity" + }, + "manufacturer": { + "title": "Component Manufacturer", + "description": "The organization that created the component.\nManufacturer is common in components created through automated processes. Components created through manual means may have `@.authors` instead.", + "$ref": "#/definitions/organizationalEntity" + }, + "authors" :{ + "type": "array", + "title": "Component Authors", + "description": "The person(s) who created the component.\nAuthors are common in components created through manual processes. Components created through automated means may have `@.manufacturer` instead.", + "items": {"$ref": "#/definitions/organizationalContact"} + }, + "author": { + "deprecated": true, + "type": "string", + "title": "Component Author (legacy)", + "description": "[Deprecated] This will be removed in a future version. Use `@.authors` or `@.manufacturer` instead.\nThe person(s) or organization(s) that authored the component", + "examples": ["Acme Inc"] + }, + "publisher": { + "type": "string", + "title": "Component Publisher", + "description": "The person(s) or organization(s) that published the component", + "examples": ["Acme Inc"] + }, + "group": { + "type": "string", + "title": "Component Group", + "description": "The grouping name or identifier. This will often be a shortened, single name of the company or project that produced the component, or the source package or domain name. Whitespace and special characters should be avoided. Examples include: apache, org.apache.commons, and apache.org.", + "examples": ["com.acme"] + }, + "name": { + "type": "string", + "title": "Component Name", + "description": "The name of the component. This will often be a shortened, single name of the component. Examples: commons-lang3 and jquery", + "examples": ["tomcat-catalina"] + }, + "version": { + "$ref": "#/definitions/version", + "title": "Component Version", + "description": "The component version. The version should ideally comply with semantic versioning but is not enforced." + }, + "description": { + "type": "string", + "title": "Component Description", + "description": "Specifies a description for the component" + }, + "scope": { + "type": "string", + "enum": [ + "required", + "optional", + "excluded" + ], + "meta:enum": { + "required": "The component is required for runtime", + "optional": "The component is optional at runtime. Optional components are components that are not capable of being called due to them not being installed or otherwise accessible by any means. Components that are installed but due to configuration or other restrictions are prohibited from being called must be scoped as 'required'.", + "excluded": "Components that are excluded provide the ability to document component usage for test and other non-runtime purposes. Excluded components are not reachable within a call graph at runtime." + }, + "title": "Component Scope", + "description": "Specifies the scope of the component. If scope is not specified, 'required' scope SHOULD be assumed by the consumer of the BOM.", + "default": "required" + }, + "hashes": { + "type": "array", + "title": "Component Hashes", + "description": "The hashes of the component.", + "items": {"$ref": "#/definitions/hash"} + }, + "licenses": { + "$ref": "#/definitions/licenseChoice", + "title": "Component License(s)" + }, + "copyright": { + "type": "string", + "title": "Component Copyright", + "description": "A copyright notice informing users of the underlying claims to copyright ownership in a published work.", + "examples": ["Acme Inc"] + }, + "cpe": { + "type": "string", + "title": "Common Platform Enumeration (CPE)", + "description": "Asserts the identity of the component using CPE. The CPE must conform to the CPE 2.2 or 2.3 specification. See [https://nvd.nist.gov/products/cpe](https://nvd.nist.gov/products/cpe). Refer to `@.evidence.identity` to optionally provide evidence that substantiates the assertion of the component's identity.", + "examples": ["cpe:2.3:a:acme:component_framework:-:*:*:*:*:*:*:*"] + }, + "purl": { + "type": "string", + "title": "Package URL (purl)", + "description": "Asserts the identity of the component using package-url (purl). The purl, if specified, must be valid and conform to the specification defined at: [https://github.com/package-url/purl-spec](https://github.com/package-url/purl-spec). Refer to `@.evidence.identity` to optionally provide evidence that substantiates the assertion of the component's identity.", + "examples": ["pkg:maven/com.acme/tomcat-catalina@9.0.14?packaging=jar"] + }, + "omniborId": { + "type": "array", + "title": "OmniBOR Artifact Identifier (gitoid)", + "description": "Asserts the identity of the component using the OmniBOR Artifact ID. The OmniBOR, if specified, must be valid and conform to the specification defined at: [https://www.iana.org/assignments/uri-schemes/prov/gitoid](https://www.iana.org/assignments/uri-schemes/prov/gitoid). Refer to `@.evidence.identity` to optionally provide evidence that substantiates the assertion of the component's identity.", + "items": { "type": "string" }, + "examples": [ + "gitoid:blob:sha1:a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", + "gitoid:blob:sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + ] + }, + "swhid": { + "type": "array", + "title": "Software Heritage Identifier", + "description": "Asserts the identity of the component using the Software Heritage persistent identifier (SWHID). The SWHID, if specified, must be valid and conform to the specification defined at: [https://docs.softwareheritage.org/devel/swh-model/persistent-identifiers.html](https://docs.softwareheritage.org/devel/swh-model/persistent-identifiers.html). Refer to `@.evidence.identity` to optionally provide evidence that substantiates the assertion of the component's identity.", + "items": { "type": "string" }, + "examples": ["swh:1:cnt:94a9ed024d3859793618152ea559a168bbcbb5e2"] + }, + "swid": { + "$ref": "#/definitions/swid", + "title": "SWID Tag", + "description": "Asserts the identity of the component using [ISO-IEC 19770-2 Software Identification (SWID) Tags](https://www.iso.org/standard/65666.html). Refer to `@.evidence.identity` to optionally provide evidence that substantiates the assertion of the component's identity." + }, + "modified": { + "type": "boolean", + "title": "Component Modified From Original", + "description": "[Deprecated] This will be removed in a future version. Use the pedigree element instead to supply information on exactly how the component was modified. A boolean value indicating if the component has been modified from the original. A value of true indicates the component is a derivative of the original. A value of false indicates the component has not been modified from the original." + }, + "pedigree": { + "type": "object", + "title": "Component Pedigree", + "description": "Component pedigree is a way to document complex supply chain scenarios where components are created, distributed, modified, redistributed, combined with other components, etc. Pedigree supports viewing this complex chain from the beginning, the end, or anywhere in the middle. It also provides a way to document variants where the exact relation may not be known.", + "additionalProperties": false, + "properties": { + "ancestors": { + "type": "array", + "title": "Ancestors", + "description": "Describes zero or more components in which a component is derived from. This is commonly used to describe forks from existing projects where the forked version contains a ancestor node containing the original component it was forked from. For example, Component A is the original component. Component B is the component being used and documented in the BOM. However, Component B contains a pedigree node with a single ancestor documenting Component A - the original component from which Component B is derived from.", + "items": {"$ref": "#/definitions/component"} + }, + "descendants": { + "type": "array", + "title": "Descendants", + "description": "Descendants are the exact opposite of ancestors. This provides a way to document all forks (and their forks) of an original or root component.", + "items": {"$ref": "#/definitions/component"} + }, + "variants": { + "type": "array", + "title": "Variants", + "description": "Variants describe relations where the relationship between the components is not known. For example, if Component A contains nearly identical code to Component B. They are both related, but it is unclear if one is derived from the other, or if they share a common ancestor.", + "items": {"$ref": "#/definitions/component"} + }, + "commits": { + "type": "array", + "title": "Commits", + "description": "A list of zero or more commits which provide a trail describing how the component deviates from an ancestor, descendant, or variant.", + "items": {"$ref": "#/definitions/commit"} + }, + "patches": { + "type": "array", + "title": "Patches", + "description": ">A list of zero or more patches describing how the component deviates from an ancestor, descendant, or variant. Patches may be complementary to commits or may be used in place of commits.", + "items": {"$ref": "#/definitions/patch"} + }, + "notes": { + "type": "string", + "title": "Notes", + "description": "Notes, observations, and other non-structured commentary describing the components pedigree." + } + } + }, + "externalReferences": { + "type": "array", + "items": {"$ref": "#/definitions/externalReference"}, + "title": "External References", + "description": "External references provide a way to document systems, sites, and information that may be relevant but are not included with the BOM. They may also establish specific relationships within or external to the BOM." + }, + "components": { + "type": "array", + "items": {"$ref": "#/definitions/component"}, + "uniqueItems": true, + "title": "Components", + "description": "A list of software and hardware components included in the parent component. This is not a dependency tree. It provides a way to specify a hierarchical representation of component assemblies, similar to system → subsystem → parts assembly in physical supply chains." + }, + "evidence": { + "$ref": "#/definitions/componentEvidence", + "title": "Evidence", + "description": "Provides the ability to document evidence collected through various forms of extraction or analysis." + }, + "releaseNotes": { + "$ref": "#/definitions/releaseNotes", + "title": "Release notes", + "description": "Specifies optional release notes." + }, + "modelCard": { + "$ref": "#/definitions/modelCard", + "title": "AI/ML Model Card" + }, + "data": { + "type": "array", + "items": {"$ref": "#/definitions/componentData"}, + "title": "Data", + "description": "This object SHOULD be specified for any component of type `data` and must not be specified for other component types." + }, + "cryptoProperties": { + "$ref": "#/definitions/cryptoProperties", + "title": "Cryptographic Properties" + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": {"$ref": "#/definitions/property"} + }, + "tags": { + "$ref": "#/definitions/tags", + "title": "Tags" + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + }, + "swid": { + "type": "object", + "title": "SWID Tag", + "description": "Specifies metadata and content for ISO-IEC 19770-2 Software Identification (SWID) Tags.", + "required": [ + "tagId", + "name" + ], + "additionalProperties": false, + "properties": { + "tagId": { + "type": "string", + "title": "Tag ID", + "description": "Maps to the tagId of a SoftwareIdentity." + }, + "name": { + "type": "string", + "title": "Name", + "description": "Maps to the name of a SoftwareIdentity." + }, + "version": { + "type": "string", + "title": "Version", + "default": "0.0", + "description": "Maps to the version of a SoftwareIdentity." + }, + "tagVersion": { + "type": "integer", + "title": "Tag Version", + "default": 0, + "description": "Maps to the tagVersion of a SoftwareIdentity." + }, + "patch": { + "type": "boolean", + "title": "Patch", + "default": false, + "description": "Maps to the patch of a SoftwareIdentity." + }, + "text": { + "title": "Attachment text", + "description": "Specifies the metadata and content of the SWID tag.", + "$ref": "#/definitions/attachment" + }, + "url": { + "type": "string", + "title": "URL", + "description": "The URL to the SWID file.", + "format": "iri-reference" + } + } + }, + "attachment": { + "type": "object", + "title": "Attachment", + "description": "Specifies the metadata and content for an attachment.", + "required": [ + "content" + ], + "additionalProperties": false, + "properties": { + "contentType": { + "type": "string", + "title": "Content-Type", + "description": "Specifies the format and nature of the data being attached, helping systems correctly interpret and process the content. Common content type examples include `application/json` for JSON data and `text/plain` for plain text documents.\n [RFC 2045 section 5.1](https://www.ietf.org/rfc/rfc2045.html#section-5.1) outlines the structure and use of content types. For a comprehensive list of registered content types, refer to the [IANA media types registry](https://www.iana.org/assignments/media-types/media-types.xhtml).", + "default": "text/plain", + "examples": [ + "text/plain", + "application/json", + "image/png" + ] + }, + "encoding": { + "type": "string", + "title": "Encoding", + "description": "Specifies the optional encoding the text is represented in.", + "enum": [ + "base64" + ], + "meta:enum": { + "base64": "Base64 is a binary-to-text encoding scheme that represents binary data in an ASCII string." + } + }, + "content": { + "type": "string", + "title": "Attachment Text", + "description": "The attachment data. Proactive controls such as input validation and sanitization should be employed to prevent misuse of attachment text." + } + } + }, + "hash": { + "type": "object", + "title": "Hash", + "required": [ + "alg", + "content" + ], + "additionalProperties": false, + "properties": { + "alg": { + "$ref": "#/definitions/hash-alg" + }, + "content": { + "$ref": "#/definitions/hash-content" + } + } + }, + "hash-alg": { + "type": "string", + "title": "Hash Algorithm", + "description": "The algorithm that generated the hash value.", + "enum": [ + "MD5", + "SHA-1", + "SHA-256", + "SHA-384", + "SHA-512", + "SHA3-256", + "SHA3-384", + "SHA3-512", + "BLAKE2b-256", + "BLAKE2b-384", + "BLAKE2b-512", + "BLAKE3" + ] + }, + "hash-content": { + "type": "string", + "title": "Hash Value", + "description": "The value of the hash.", + "examples": ["3942447fac867ae5cdb3229b658f4d48"], + "pattern": "^([a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{64}|[a-fA-F0-9]{96}|[a-fA-F0-9]{128})$" + }, + "license": { + "type": "object", + "title": "License", + "description": "Specifies the details and attributes related to a software license. It can either include a valid SPDX license identifier or a named license, along with additional properties such as license acknowledgment, comprehensive commercial licensing information, and the full text of the license.", + "oneOf": [ + { + "required": ["id"] + }, + { + "required": ["name"] + } + ], + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the license elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "id": { + "$ref": "spdx.schema.json", + "title": "License ID (SPDX)", + "description": "A valid SPDX license identifier. If specified, this value must be one of the enumeration of valid SPDX license identifiers defined in the spdx.schema.json (or spdx.xml) subschema which is synchronized with the official SPDX license list.", + "examples": ["Apache-2.0"] + }, + "name": { + "type": "string", + "title": "License Name", + "description": "The name of the license. This may include the name of a commercial or proprietary license or an open source license that may not be defined by SPDX.", + "examples": ["Acme Software License"] + }, + "acknowledgement": { + "$ref": "#/definitions/licenseAcknowledgementEnumeration" + }, + "text": { + "title": "License text", + "description": "An optional way to include the textual content of a license.", + "$ref": "#/definitions/attachment" + }, + "url": { + "type": "string", + "title": "License URL", + "description": "The URL to the license file. If specified, a 'license' externalReference should also be specified for completeness", + "examples": ["https://www.apache.org/licenses/LICENSE-2.0.txt"], + "format": "iri-reference" + }, + "licensing": { + "type": "object", + "title": "Licensing information", + "description": "Licensing details describing the licensor/licensee, license type, renewal and expiration dates, and other important metadata", + "additionalProperties": false, + "properties": { + "altIds": { + "type": "array", + "title": "Alternate License Identifiers", + "description": "License identifiers that may be used to manage licenses and their lifecycle", + "items": { + "type": "string" + } + }, + "licensor": { + "title": "Licensor", + "description": "The individual or organization that grants a license to another individual or organization", + "type": "object", + "additionalProperties": false, + "properties": { + "organization": { + "title": "Licensor (Organization)", + "description": "The organization that granted the license", + "$ref": "#/definitions/organizationalEntity" + }, + "individual": { + "title": "Licensor (Individual)", + "description": "The individual, not associated with an organization, that granted the license", + "$ref": "#/definitions/organizationalContact" + } + }, + "oneOf":[ + { + "required": ["organization"] + }, + { + "required": ["individual"] + } + ] + }, + "licensee": { + "title": "Licensee", + "description": "The individual or organization for which a license was granted to", + "type": "object", + "additionalProperties": false, + "properties": { + "organization": { + "title": "Licensee (Organization)", + "description": "The organization that was granted the license", + "$ref": "#/definitions/organizationalEntity" + }, + "individual": { + "title": "Licensee (Individual)", + "description": "The individual, not associated with an organization, that was granted the license", + "$ref": "#/definitions/organizationalContact" + } + }, + "oneOf":[ + { + "required": ["organization"] + }, + { + "required": ["individual"] + } + ] + }, + "purchaser": { + "title": "Purchaser", + "description": "The individual or organization that purchased the license", + "type": "object", + "additionalProperties": false, + "properties": { + "organization": { + "title": "Purchaser (Organization)", + "description": "The organization that purchased the license", + "$ref": "#/definitions/organizationalEntity" + }, + "individual": { + "title": "Purchaser (Individual)", + "description": "The individual, not associated with an organization, that purchased the license", + "$ref": "#/definitions/organizationalContact" + } + }, + "oneOf":[ + { + "required": ["organization"] + }, + { + "required": ["individual"] + } + ] + }, + "purchaseOrder": { + "type": "string", + "title": "Purchase Order", + "description": "The purchase order identifier the purchaser sent to a supplier or vendor to authorize a purchase" + }, + "licenseTypes": { + "type": "array", + "title": "License Type", + "description": "The type of license(s) that was granted to the licensee.", + "items": { + "type": "string", + "enum": [ + "academic", + "appliance", + "client-access", + "concurrent-user", + "core-points", + "custom-metric", + "device", + "evaluation", + "named-user", + "node-locked", + "oem", + "perpetual", + "processor-points", + "subscription", + "user", + "other" + ], + "meta:enum": { + "academic": "A license that grants use of software solely for the purpose of education or research.", + "appliance": "A license covering use of software embedded in a specific piece of hardware.", + "client-access": "A Client Access License (CAL) allows client computers to access services provided by server software.", + "concurrent-user": "A Concurrent User license (aka floating license) limits the number of licenses for a software application and licenses are shared among a larger number of users.", + "core-points": "A license where the core of a computer's processor is assigned a specific number of points.", + "custom-metric": "A license for which consumption is measured by non-standard metrics.", + "device": "A license that covers a defined number of installations on computers and other types of devices.", + "evaluation": "A license that grants permission to install and use software for trial purposes.", + "named-user": "A license that grants access to the software to one or more pre-defined users.", + "node-locked": "A license that grants access to the software on one or more pre-defined computers or devices.", + "oem": "An Original Equipment Manufacturer license that is delivered with hardware, cannot be transferred to other hardware, and is valid for the life of the hardware.", + "perpetual": "A license where the software is sold on a one-time basis and the licensee can use a copy of the software indefinitely.", + "processor-points": "A license where each installation consumes points per processor.", + "subscription": "A license where the licensee pays a fee to use the software or service.", + "user": "A license that grants access to the software or service by a specified number of users.", + "other": "Another license type." + } + } + }, + "lastRenewal": { + "type": "string", + "format": "date-time", + "title": "Last Renewal", + "description": "The timestamp indicating when the license was last renewed. For new purchases, this is often the purchase or acquisition date. For non-perpetual licenses or subscriptions, this is the timestamp of when the license was last renewed." + }, + "expiration": { + "type": "string", + "format": "date-time", + "title": "Expiration", + "description": "The timestamp indicating when the current license expires (if applicable)." + } + } + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": {"$ref": "#/definitions/property"} + } + } + }, + "licenseAcknowledgementEnumeration": { + "title": "License Acknowledgement", + "description": "Declared licenses and concluded licenses represent two different stages in the licensing process within software development. Declared licenses refer to the initial intention of the software authors regarding the licensing terms under which their code is released. On the other hand, concluded licenses are the result of a comprehensive analysis of the project's codebase to identify and confirm the actual licenses of the components used, which may differ from the initially declared licenses. While declared licenses provide an upfront indication of the licensing intentions, concluded licenses offer a more thorough understanding of the actual licensing within a project, facilitating proper compliance and risk management. Observed licenses are defined in `@.evidence.licenses`. Observed licenses form the evidence necessary to substantiate a concluded license.", + "type": "string", + "enum": [ + "declared", + "concluded" + ], + "meta:enum": { + "declared": "Declared licenses represent the initial intentions of authors regarding the licensing terms of their code.", + "concluded": "Concluded licenses are verified and confirmed." + } + }, + "licenseChoice": { + "title": "License Choice", + "description": "EITHER (list of SPDX licenses and/or named licenses) OR (tuple of one SPDX License Expression)", + "type": "array", + "oneOf": [ + { + "title": "Multiple licenses", + "description": "A list of SPDX licenses and/or named licenses.", + "type": "array", + "items": { + "type": "object", + "title": "License", + "required": ["license"], + "additionalProperties": false, + "properties": { + "license": {"$ref": "#/definitions/license"} + } + } + }, + { + "title": "SPDX License Expression", + "description": "A tuple of exactly one SPDX License Expression.", + "type": "array", + "additionalItems": false, + "minItems": 1, + "maxItems": 1, + "items": [{ + "type": "object", + "additionalProperties": false, + "required": ["expression"], + "properties": { + "expression": { + "type": "string", + "title": "SPDX License Expression", + "description": "A valid SPDX license expression.\nRefer to https://spdx.org/specifications for syntax requirements", + "examples": [ + "Apache-2.0 AND (MIT OR GPL-2.0-only)", + "GPL-3.0-only WITH Classpath-exception-2.0" + ] + }, + "acknowledgement": { + "$ref": "#/definitions/licenseAcknowledgementEnumeration" + }, + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the license elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + } + } + }] + } + ] + }, + "commit": { + "type": "object", + "title": "Commit", + "description": "Specifies an individual commit", + "additionalProperties": false, + "properties": { + "uid": { + "type": "string", + "title": "UID", + "description": "A unique identifier of the commit. This may be version control specific. For example, Subversion uses revision numbers whereas git uses commit hashes." + }, + "url": { + "type": "string", + "title": "URL", + "description": "The URL to the commit. This URL will typically point to a commit in a version control system.", + "format": "iri-reference" + }, + "author": { + "title": "Author", + "description": "The author who created the changes in the commit", + "$ref": "#/definitions/identifiableAction" + }, + "committer": { + "title": "Committer", + "description": "The person who committed or pushed the commit", + "$ref": "#/definitions/identifiableAction" + }, + "message": { + "type": "string", + "title": "Message", + "description": "The text description of the contents of the commit" + } + } + }, + "patch": { + "type": "object", + "title": "Patch", + "description": "Specifies an individual patch", + "required": [ + "type" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "unofficial", + "monkey", + "backport", + "cherry-pick" + ], + "meta:enum": { + "unofficial": "A patch which is not developed by the creators or maintainers of the software being patched. Refer to [https://en.wikipedia.org/wiki/Unofficial_patch](https://en.wikipedia.org/wiki/Unofficial_patch).", + "monkey": "A patch which dynamically modifies runtime behavior. Refer to [https://en.wikipedia.org/wiki/Monkey_patch](https://en.wikipedia.org/wiki/Monkey_patch).", + "backport": "A patch which takes code from a newer version of the software and applies it to older versions of the same software. Refer to [https://en.wikipedia.org/wiki/Backporting](https://en.wikipedia.org/wiki/Backporting).", + "cherry-pick": "A patch created by selectively applying commits from other versions or branches of the same software." + }, + "title": "Patch Type", + "description": "Specifies the purpose for the patch including the resolution of defects, security issues, or new behavior or functionality." + }, + "diff": { + "title": "Diff", + "description": "The patch file (or diff) that shows changes. Refer to [https://en.wikipedia.org/wiki/Diff](https://en.wikipedia.org/wiki/Diff)", + "$ref": "#/definitions/diff" + }, + "resolves": { + "type": "array", + "items": {"$ref": "#/definitions/issue"}, + "title": "Resolves", + "description": "A collection of issues the patch resolves" + } + } + }, + "diff": { + "type": "object", + "title": "Diff", + "description": "The patch file (or diff) that shows changes. Refer to https://en.wikipedia.org/wiki/Diff", + "additionalProperties": false, + "properties": { + "text": { + "title": "Diff text", + "description": "Specifies the optional text of the diff", + "$ref": "#/definitions/attachment" + }, + "url": { + "type": "string", + "title": "URL", + "description": "Specifies the URL to the diff", + "format": "iri-reference" + } + } + }, + "issue": { + "type": "object", + "title": "Issue", + "description": "An individual issue that has been resolved.", + "required": [ + "type" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "defect", + "enhancement", + "security" + ], + "meta:enum": { + "defect": "A fault, flaw, or bug in software.", + "enhancement": "A new feature or behavior in software.", + "security": "A special type of defect which impacts security." + }, + "title": "Issue Type", + "description": "Specifies the type of issue" + }, + "id": { + "type": "string", + "title": "Issue ID", + "description": "The identifier of the issue assigned by the source of the issue" + }, + "name": { + "type": "string", + "title": "Issue Name", + "description": "The name of the issue" + }, + "description": { + "type": "string", + "title": "Issue Description", + "description": "A description of the issue" + }, + "source": { + "type": "object", + "title": "Source", + "description": "The source of the issue where it is documented", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "The name of the source.", + "examples": [ + "National Vulnerability Database", + "NVD", + "Apache" + ] + }, + "url": { + "type": "string", + "title": "URL", + "description": "The url of the issue documentation as provided by the source", + "format": "iri-reference" + } + } + }, + "references": { + "type": "array", + "items": { + "type": "string", + "format": "iri-reference" + }, + "title": "References", + "description": "A collection of URL's for reference. Multiple URLs are allowed.", + "examples": ["https://example.com"] + } + } + }, + "identifiableAction": { + "type": "object", + "title": "Identifiable Action", + "description": "Specifies an individual commit", + "additionalProperties": false, + "properties": { + "timestamp": { + "type": "string", + "format": "date-time", + "title": "Timestamp", + "description": "The timestamp in which the action occurred" + }, + "name": { + "type": "string", + "title": "Name", + "description": "The name of the individual who performed the action" + }, + "email": { + "type": "string", + "format": "idn-email", + "title": "E-mail", + "description": "The email address of the individual who performed the action" + } + } + }, + "externalReference": { + "type": "object", + "title": "External Reference", + "description": "External references provide a way to document systems, sites, and information that may be relevant but are not included with the BOM. They may also establish specific relationships within or external to the BOM.", + "required": [ + "url", + "type" + ], + "additionalProperties": false, + "properties": { + "url": { + "anyOf": [ + { + "title": "URL", + "type": "string", + "format": "iri-reference" + }, + { + "title": "BOM-Link", + "$ref": "#/definitions/bomLink" + } + ], + "title": "URL", + "description": "The URI (URL or URN) to the external reference. External references are URIs and therefore can accept any URL scheme including https ([RFC-7230](https://www.ietf.org/rfc/rfc7230.txt)), mailto ([RFC-2368](https://www.ietf.org/rfc/rfc2368.txt)), tel ([RFC-3966](https://www.ietf.org/rfc/rfc3966.txt)), and dns ([RFC-4501](https://www.ietf.org/rfc/rfc4501.txt)). External references may also include formally registered URNs such as [CycloneDX BOM-Link](https://cyclonedx.org/capabilities/bomlink/) to reference CycloneDX BOMs or any object within a BOM. BOM-Link transforms applicable external references into relationships that can be expressed in a BOM or across BOMs." + }, + "comment": { + "type": "string", + "title": "Comment", + "description": "An optional comment describing the external reference" + }, + "type": { + "type": "string", + "title": "Type", + "description": "Specifies the type of external reference.", + "enum": [ + "vcs", + "issue-tracker", + "website", + "advisories", + "bom", + "mailing-list", + "social", + "chat", + "documentation", + "support", + "source-distribution", + "distribution", + "distribution-intake", + "license", + "build-meta", + "build-system", + "release-notes", + "security-contact", + "model-card", + "log", + "configuration", + "evidence", + "formulation", + "attestation", + "threat-model", + "adversary-model", + "risk-assessment", + "vulnerability-assertion", + "exploitability-statement", + "pentest-report", + "static-analysis-report", + "dynamic-analysis-report", + "runtime-analysis-report", + "component-analysis-report", + "maturity-report", + "certification-report", + "codified-infrastructure", + "quality-metrics", + "poam", + "electronic-signature", + "digital-signature", + "rfc-9116", + "other" + ], + "meta:enum": { + "vcs": "Version Control System", + "issue-tracker": "Issue or defect tracking system, or an Application Lifecycle Management (ALM) system", + "website": "Website", + "advisories": "Security advisories", + "bom": "Bill of Materials (SBOM, OBOM, HBOM, SaaSBOM, etc)", + "mailing-list": "Mailing list or discussion group", + "social": "Social media account", + "chat": "Real-time chat platform", + "documentation": "Documentation, guides, or how-to instructions", + "support": "Community or commercial support", + "source-distribution": "The location where the source code distributable can be obtained. This is often an archive format such as zip or tgz. The source-distribution type complements use of the version control (vcs) type.", + "distribution": "Direct or repository download location", + "distribution-intake": "The location where a component was published to. This is often the same as \"distribution\" but may also include specialized publishing processes that act as an intermediary.", + "license": "The reference to the license file. If a license URL has been defined in the license node, it should also be defined as an external reference for completeness.", + "build-meta": "Build-system specific meta file (i.e. pom.xml, package.json, .nuspec, etc)", + "build-system": "Reference to an automated build system", + "release-notes": "Reference to release notes", + "security-contact": "Specifies a way to contact the maintainer, supplier, or provider in the event of a security incident. Common URIs include links to a disclosure procedure, a mailto (RFC-2368) that specifies an email address, a tel (RFC-3966) that specifies a phone number, or dns (RFC-4501) that specifies the records containing DNS Security TXT.", + "model-card": "A model card describes the intended uses of a machine learning model, potential limitations, biases, ethical considerations, training parameters, datasets used to train the model, performance metrics, and other relevant data useful for ML transparency.", + "log": "A record of events that occurred in a computer system or application, such as problems, errors, or information on current operations.", + "configuration": "Parameters or settings that may be used by other components or services.", + "evidence": "Information used to substantiate a claim.", + "formulation": "Describes how a component or service was manufactured or deployed.", + "attestation": "Human or machine-readable statements containing facts, evidence, or testimony.", + "threat-model": "An enumeration of identified weaknesses, threats, and countermeasures, dataflow diagram (DFD), attack tree, and other supporting documentation in human-readable or machine-readable format.", + "adversary-model": "The defined assumptions, goals, and capabilities of an adversary.", + "risk-assessment": "Identifies and analyzes the potential of future events that may negatively impact individuals, assets, and/or the environment. Risk assessments may also include judgments on the tolerability of each risk.", + "vulnerability-assertion": "A Vulnerability Disclosure Report (VDR) which asserts the known and previously unknown vulnerabilities that affect a component, service, or product including the analysis and findings describing the impact (or lack of impact) that the reported vulnerability has on a component, service, or product.", + "exploitability-statement": "A Vulnerability Exploitability eXchange (VEX) which asserts the known vulnerabilities that do not affect a product, product family, or organization, and optionally the ones that do. The VEX should include the analysis and findings describing the impact (or lack of impact) that the reported vulnerability has on the product, product family, or organization.", + "pentest-report": "Results from an authorized simulated cyberattack on a component or service, otherwise known as a penetration test.", + "static-analysis-report": "SARIF or proprietary machine or human-readable report for which static analysis has identified code quality, security, and other potential issues with the source code.", + "dynamic-analysis-report": "Dynamic analysis report that has identified issues such as vulnerabilities and misconfigurations.", + "runtime-analysis-report": "Report generated by analyzing the call stack of a running application.", + "component-analysis-report": "Report generated by Software Composition Analysis (SCA), container analysis, or other forms of component analysis.", + "maturity-report": "Report containing a formal assessment of an organization, business unit, or team against a maturity model.", + "certification-report": "Industry, regulatory, or other certification from an accredited (if applicable) certification body.", + "codified-infrastructure": "Code or configuration that defines and provisions virtualized infrastructure, commonly referred to as Infrastructure as Code (IaC).", + "quality-metrics": "Report or system in which quality metrics can be obtained.", + "poam": "Plans of Action and Milestones (POA&M) complement an \"attestation\" external reference. POA&M is defined by NIST as a \"document that identifies tasks needing to be accomplished. It details resources required to accomplish the elements of the plan, any milestones in meeting the tasks and scheduled completion dates for the milestones\".", + "electronic-signature": "An e-signature is commonly a scanned representation of a written signature or a stylized script of the person's name.", + "digital-signature": "A signature that leverages cryptography, typically public/private key pairs, which provides strong authenticity verification.", + "rfc-9116": "Document that complies with [RFC 9116](https://www.ietf.org/rfc/rfc9116.html) (A File Format to Aid in Security Vulnerability Disclosure)", + "other": "Use this if no other types accurately describe the purpose of the external reference." + } + }, + "hashes": { + "type": "array", + "items": {"$ref": "#/definitions/hash"}, + "title": "Hashes", + "description": "The hashes of the external reference (if applicable)." + } + } + }, + "dependency": { + "type": "object", + "title": "Dependency", + "description": "Defines the direct dependencies of a component, service, or the components provided/implemented by a given component. Components or services that do not have their own dependencies must be declared as empty elements within the graph. Components or services that are not represented in the dependency graph may have unknown dependencies. It is recommended that implementations assume this to be opaque and not an indicator of an object being dependency-free. It is recommended to leverage compositions to indicate unknown dependency graphs.", + "required": [ + "ref" + ], + "additionalProperties": false, + "properties": { + "ref": { + "$ref": "#/definitions/refLinkType", + "title": "Reference", + "description": "References a component or service by its bom-ref attribute" + }, + "dependsOn": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/refLinkType" + }, + "title": "Depends On", + "description": "The bom-ref identifiers of the components or services that are dependencies of this dependency object." + }, + "provides": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/refLinkType" + }, + "title": "Provides", + "description": "The bom-ref identifiers of the components or services that define a given specification or standard, which are provided or implemented by this dependency object.\nFor example, a cryptographic library which implements a cryptographic algorithm. A component which implements another component does not imply that the implementation is in use." + } + } + }, + "service": { + "type": "object", + "title": "Service", + "required": [ + "name" + ], + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the service elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "provider": { + "title": "Provider", + "description": "The organization that provides the service.", + "$ref": "#/definitions/organizationalEntity" + }, + "group": { + "type": "string", + "title": "Service Group", + "description": "The grouping name, namespace, or identifier. This will often be a shortened, single name of the company or project that produced the service or domain name. Whitespace and special characters should be avoided.", + "examples": ["com.acme"] + }, + "name": { + "type": "string", + "title": "Service Name", + "description": "The name of the service. This will often be a shortened, single name of the service.", + "examples": ["ticker-service"] + }, + "version": { + "$ref": "#/definitions/version", + "title": "Service Version", + "description": "The service version." + }, + "description": { + "type": "string", + "title": "Service Description", + "description": "Specifies a description for the service" + }, + "endpoints": { + "type": "array", + "items": { + "type": "string", + "format": "iri-reference" + }, + "title": "Endpoints", + "description": "The endpoint URIs of the service. Multiple endpoints are allowed.", + "examples": ["https://example.com/api/v1/ticker"] + }, + "authenticated": { + "type": "boolean", + "title": "Authentication Required", + "description": "A boolean value indicating if the service requires authentication. A value of true indicates the service requires authentication prior to use. A value of false indicates the service does not require authentication." + }, + "x-trust-boundary": { + "type": "boolean", + "title": "Crosses Trust Boundary", + "description": "A boolean value indicating if use of the service crosses a trust zone or boundary. A value of true indicates that by using the service, a trust boundary is crossed. A value of false indicates that by using the service, a trust boundary is not crossed." + }, + "trustZone": { + "type": "string", + "title": "Trust Zone", + "description": "The name of the trust zone the service resides in." + }, + "data": { + "type": "array", + "items": {"$ref": "#/definitions/serviceData"}, + "title": "Data", + "description": "Specifies information about the data including the directional flow of data and the data classification." + }, + "licenses": { + "$ref": "#/definitions/licenseChoice", + "title": "Service License(s)" + }, + "externalReferences": { + "type": "array", + "items": {"$ref": "#/definitions/externalReference"}, + "title": "External References", + "description": "External references provide a way to document systems, sites, and information that may be relevant but are not included with the BOM. They may also establish specific relationships within or external to the BOM." + }, + "services": { + "type": "array", + "items": {"$ref": "#/definitions/service"}, + "uniqueItems": true, + "title": "Services", + "description": "A list of services included or deployed behind the parent service. This is not a dependency tree. It provides a way to specify a hierarchical representation of service assemblies." + }, + "releaseNotes": { + "$ref": "#/definitions/releaseNotes", + "title": "Release notes", + "description": "Specifies optional release notes." + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": {"$ref": "#/definitions/property"} + }, + "tags": { + "$ref": "#/definitions/tags", + "title": "Tags" + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + }, + "serviceData": { + "type": "object", + "title": "Hash Objects", + "required": [ + "flow", + "classification" + ], + "additionalProperties": false, + "properties": { + "flow": { + "$ref": "#/definitions/dataFlowDirection", + "title": "Directional Flow", + "description": "Specifies the flow direction of the data. Direction is relative to the service. Inbound flow states that data enters the service. Outbound flow states that data leaves the service. Bi-directional states that data flows both ways and unknown states that the direction is not known." + }, + "classification": { + "$ref": "#/definitions/dataClassification" + }, + "name": { + "type": "string", + "title": "Name", + "description": "Name for the defined data", + "examples": [ + "Credit card reporting" + ] + }, + "description": { + "type": "string", + "title": "Description", + "description": "Short description of the data content and usage", + "examples": [ + "Credit card information being exchanged in between the web app and the database" + ] + }, + "governance": { + "title": "Data Governance", + "$ref": "#/definitions/dataGovernance" + }, + "source": { + "type": "array", + "items": { + "anyOf": [ + { + "title": "URL", + "type": "string", + "format": "iri-reference" + }, + { + "title": "BOM-Link Element", + "$ref": "#/definitions/bomLinkElementType" + } + ] + }, + "title": "Source", + "description": "The URI, URL, or BOM-Link of the components or services the data came in from" + }, + "destination": { + "type": "array", + "items": { + "anyOf": [ + { + "title": "URL", + "type": "string", + "format": "iri-reference" + }, + { + "title": "BOM-Link Element", + "$ref": "#/definitions/bomLinkElementType" + } + ] + }, + "title": "Destination", + "description": "The URI, URL, or BOM-Link of the components or services the data is sent to" + } + } + }, + "dataFlowDirection": { + "type": "string", + "enum": [ + "inbound", + "outbound", + "bi-directional", + "unknown" + ], + "meta:enum": { + "inbound": "Data that enters a service.", + "outbound": "Data that exits a service.", + "bi-directional": "Data flows in and out of the service.", + "unknown": "The directional flow of data is not known." + }, + "title": "Data flow direction", + "description": "Specifies the flow direction of the data. Direction is relative to the service." + }, + "copyright": { + "type": "object", + "title": "Copyright", + "description": "A copyright notice informing users of the underlying claims to copyright ownership in a published work.", + "required": [ + "text" + ], + "additionalProperties": false, + "properties": { + "text": { + "type": "string", + "title": "Copyright Text", + "description": "The textual content of the copyright." + } + } + }, + "componentEvidence": { + "type": "object", + "title": "Evidence", + "description": "Provides the ability to document evidence collected through various forms of extraction or analysis.", + "additionalProperties": false, + "properties": { + "identity": { + "title": "Identity Evidence", + "description": "Evidence that substantiates the identity of a component. The identity may be an object or an array of identity objects. Support for specifying identity as a single object was introduced in CycloneDX v1.5. Arrays were introduced in v1.6. It is recommended that all implementations use arrays, even if only one identity object is specified.", + "oneOf" : [ + { + "type": "array", + "title": "Array of Identity Objects", + "items": { "$ref": "#/definitions/componentIdentityEvidence" } + }, + { + "title": "A Single Identity Object", + "description": "[Deprecated]", + "$ref": "#/definitions/componentIdentityEvidence", + "deprecated": true + } + ] + }, + "occurrences": { + "type": "array", + "title": "Occurrences", + "description": "Evidence of individual instances of a component spread across multiple locations.", + "items": { + "type": "object", + "required": [ "location" ], + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the occurrence elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "location": { + "type": "string", + "title": "Location", + "description": "The location or path to where the component was found." + }, + "line": { + "type": "integer", + "minimum": 0, + "title": "Line Number", + "description": "The line number where the component was found." + }, + "offset": { + "type": "integer", + "minimum": 0, + "title": "Offset", + "description": "The offset where the component was found." + }, + "symbol": { + "type": "string", + "title": "Symbol", + "description": "The symbol name that was found associated with the component." + }, + "additionalContext": { + "type": "string", + "title": "Additional Context", + "description": "Any additional context of the detected component (e.g. a code snippet)." + } + } + } + }, + "callstack": { + "type": "object", + "title": "Call Stack", + "description": "Evidence of the components use through the callstack.", + "additionalProperties": false, + "properties": { + "frames": { + "type": "array", + "title": "Frames", + "description": "Within a call stack, a frame is a discrete unit that encapsulates an execution context, including local variables, parameters, and the return address. As function calls are made, frames are pushed onto the stack, forming an array-like structure that orchestrates the flow of program execution and manages the sequence of function invocations.", + "items": { + "type": "object", + "required": [ + "module" + ], + "additionalProperties": false, + "properties": { + "package": { + "title": "Package", + "description": "A package organizes modules into namespaces, providing a unique namespace for each type it contains.", + "type": "string" + }, + "module": { + "title": "Module", + "description": "A module or class that encloses functions/methods and other code.", + "type": "string" + }, + "function": { + "title": "Function", + "description": "A block of code designed to perform a particular task.", + "type": "string" + }, + "parameters": { + "title": "Parameters", + "description": "Optional arguments that are passed to the module or function.", + "type": "array", + "items": { + "type": "string" + } + }, + "line": { + "title": "Line", + "description": "The line number the code that is called resides on.", + "type": "integer" + }, + "column": { + "title": "Column", + "description": "The column the code that is called resides.", + "type": "integer" + }, + "fullFilename": { + "title": "Full Filename", + "description": "The full path and filename of the module.", + "type": "string" + } + } + } + } + } + }, + "licenses": { + "$ref": "#/definitions/licenseChoice", + "title": "License Evidence" + }, + "copyright": { + "type": "array", + "items": {"$ref": "#/definitions/copyright"}, + "title": "Copyright Evidence", + "description": "Copyright evidence captures intellectual property assertions, providing evidence of possible ownership and legal protection." + } + } + }, + "compositions": { + "type": "object", + "title": "Compositions", + "required": [ + "aggregate" + ], + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the composition elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "aggregate": { + "$ref": "#/definitions/aggregateType", + "title": "Aggregate", + "description": "Specifies an aggregate type that describes how complete a relationship is." + }, + "assemblies": { + "type": "array", + "uniqueItems": true, + "items": { + "anyOf": [ + { + "title": "Ref", + "$ref": "#/definitions/refLinkType" + }, + { + "title": "BOM-Link Element", + "$ref": "#/definitions/bomLinkElementType" + } + ] + }, + "title": "BOM references", + "description": "The bom-ref identifiers of the components or services being described. Assemblies refer to nested relationships whereby a constituent part may include other constituent parts. References do not cascade to child parts. References are explicit for the specified constituent part only." + }, + "dependencies": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "title": "BOM references", + "description": "The bom-ref identifiers of the components or services being described. Dependencies refer to a relationship whereby an independent constituent part requires another independent constituent part. References do not cascade to transitive dependencies. References are explicit for the specified dependency only." + }, + "vulnerabilities": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "title": "BOM references", + "description": "The bom-ref identifiers of the vulnerabilities being described." + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + }, + "aggregateType": { + "type": "string", + "default": "not_specified", + "enum": [ + "complete", + "incomplete", + "incomplete_first_party_only", + "incomplete_first_party_proprietary_only", + "incomplete_first_party_opensource_only", + "incomplete_third_party_only", + "incomplete_third_party_proprietary_only", + "incomplete_third_party_opensource_only", + "unknown", + "not_specified" + ], + "meta:enum": { + "complete": "The relationship is complete. No further relationships including constituent components, services, or dependencies are known to exist.", + "incomplete": "The relationship is incomplete. Additional relationships exist and may include constituent components, services, or dependencies.", + "incomplete_first_party_only": "The relationship is incomplete. Only relationships for first-party components, services, or their dependencies are represented.", + "incomplete_first_party_proprietary_only": "The relationship is incomplete. Only relationships for first-party components, services, or their dependencies are represented, limited specifically to those that are proprietary.", + "incomplete_first_party_opensource_only": "The relationship is incomplete. Only relationships for first-party components, services, or their dependencies are represented, limited specifically to those that are opensource.", + "incomplete_third_party_only": "The relationship is incomplete. Only relationships for third-party components, services, or their dependencies are represented.", + "incomplete_third_party_proprietary_only": "The relationship is incomplete. Only relationships for third-party components, services, or their dependencies are represented, limited specifically to those that are proprietary.", + "incomplete_third_party_opensource_only": "The relationship is incomplete. Only relationships for third-party components, services, or their dependencies are represented, limited specifically to those that are opensource.", + "unknown": "The relationship may be complete or incomplete. This usually signifies a 'best-effort' to obtain constituent components, services, or dependencies but the completeness is inconclusive.", + "not_specified": "The relationship completeness is not specified." + } + }, + "property": { + "type": "object", + "title": "Lightweight name-value pair", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "The name of the property. Duplicate names are allowed, each potentially having a different value." + }, + "value": { + "type": "string", + "title": "Value", + "description": "The value of the property." + } + }, + "additionalProperties": false + }, + "localeType": { + "type": "string", + "pattern": "^([a-z]{2})(-[A-Z]{2})?$", + "title": "Locale", + "description": "Defines a syntax for representing two character language code (ISO-639) followed by an optional two character country code. The language code must be lower case. If the country code is specified, the country code must be upper case. The language code and country code must be separated by a minus sign. Examples: en, en-US, fr, fr-CA" + }, + "releaseType": { + "type": "string", + "examples": [ + "major", + "minor", + "patch", + "pre-release", + "internal" + ], + "description": "The software versioning type. It is recommended that the release type use one of 'major', 'minor', 'patch', 'pre-release', or 'internal'. Representing all possible software release types is not practical, so standardizing on the recommended values, whenever possible, is strongly encouraged.\n\n* __major__ = A major release may contain significant changes or may introduce breaking changes.\n* __minor__ = A minor release, also known as an update, may contain a smaller number of changes than major releases.\n* __patch__ = Patch releases are typically unplanned and may resolve defects or important security issues.\n* __pre-release__ = A pre-release may include alpha, beta, or release candidates and typically have limited support. They provide the ability to preview a release prior to its general availability.\n* __internal__ = Internal releases are not for public consumption and are intended to be used exclusively by the project or manufacturer that produced it." + }, + "note": { + "type": "object", + "title": "Note", + "description": "A note containing the locale and content.", + "required": [ + "text" + ], + "additionalProperties": false, + "properties": { + "locale": { + "$ref": "#/definitions/localeType", + "title": "Locale", + "description": "The ISO-639 (or higher) language code and optional ISO-3166 (or higher) country code. Examples include: \"en\", \"en-US\", \"fr\" and \"fr-CA\"" + }, + "text": { + "title": "Release note content", + "description": "Specifies the full content of the release note.", + "$ref": "#/definitions/attachment" + } + } + }, + "releaseNotes": { + "type": "object", + "title": "Release notes", + "required": [ + "type" + ], + "additionalProperties": false, + "properties": { + "type": { + "$ref": "#/definitions/releaseType", + "title": "Type", + "description": "The software versioning type the release note describes." + }, + "title": { + "type": "string", + "title": "Title", + "description": "The title of the release." + }, + "featuredImage": { + "type": "string", + "format": "iri-reference", + "title": "Featured image", + "description": "The URL to an image that may be prominently displayed with the release note." + }, + "socialImage": { + "type": "string", + "format": "iri-reference", + "title": "Social image", + "description": "The URL to an image that may be used in messaging on social media platforms." + }, + "description": { + "type": "string", + "title": "Description", + "description": "A short description of the release." + }, + "timestamp": { + "type": "string", + "format": "date-time", + "title": "Timestamp", + "description": "The date and time (timestamp) when the release note was created." + }, + "aliases": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Aliases", + "description": "One or more alternate names the release may be referred to. This may include unofficial terms used by development and marketing teams (e.g. code names)." + }, + "tags": { + "$ref": "#/definitions/tags", + "title": "Tags" + }, + "resolves": { + "type": "array", + "items": {"$ref": "#/definitions/issue"}, + "title": "Resolves", + "description": "A collection of issues that have been resolved." + }, + "notes": { + "type": "array", + "items": {"$ref": "#/definitions/note"}, + "title": "Notes", + "description": "Zero or more release notes containing the locale and content. Multiple note objects may be specified to support release notes in a wide variety of languages." + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": {"$ref": "#/definitions/property"} + } + } + }, + "advisory": { + "type": "object", + "title": "Advisory", + "description": "Title and location where advisory information can be obtained. An advisory is a notification of a threat to a component, service, or system.", + "required": ["url"], + "additionalProperties": false, + "properties": { + "title": { + "type": "string", + "title": "Title", + "description": "An optional name of the advisory." + }, + "url": { + "type": "string", + "title": "URL", + "format": "iri-reference", + "description": "Location where the advisory can be obtained." + } + } + }, + "cwe": { + "type": "integer", + "minimum": 1, + "title": "CWE", + "description": "Integer representation of a Common Weaknesses Enumerations (CWE). For example 399 (of https://cwe.mitre.org/data/definitions/399.html)" + }, + "severity": { + "type": "string", + "title": "Severity", + "description": "Textual representation of the severity of the vulnerability adopted by the analysis method. If the analysis method uses values other than what is provided, the user is expected to translate appropriately.", + "enum": [ + "critical", + "high", + "medium", + "low", + "info", + "none", + "unknown" + ], + "meta:enum": { + "critical": "Critical severity", + "high": "High severity", + "medium": "Medium severity", + "low": "Low severity", + "info": "Informational warning.", + "none": "None", + "unknown": "The severity is not known" + } + }, + "scoreMethod": { + "type": "string", + "title": "Method", + "description": "Specifies the severity or risk scoring methodology or standard used.", + "enum": [ + "CVSSv2", + "CVSSv3", + "CVSSv31", + "CVSSv4", + "OWASP", + "SSVC", + "other" + ], + "meta:enum": { + "CVSSv2": "Common Vulnerability Scoring System v2.0", + "CVSSv3": "Common Vulnerability Scoring System v3.0", + "CVSSv31": "Common Vulnerability Scoring System v3.1", + "CVSSv4": "Common Vulnerability Scoring System v4.0", + "OWASP": "OWASP Risk Rating Methodology", + "SSVC": "Stakeholder Specific Vulnerability Categorization", + "other": "Another severity or risk scoring methodology" + } + }, + "impactAnalysisState": { + "type": "string", + "title": "Impact Analysis State", + "description": "Declares the current state of an occurrence of a vulnerability, after automated or manual analysis.", + "enum": [ + "resolved", + "resolved_with_pedigree", + "exploitable", + "in_triage", + "false_positive", + "not_affected" + ], + "meta:enum": { + "resolved": "The vulnerability has been remediated.", + "resolved_with_pedigree": "The vulnerability has been remediated and evidence of the changes are provided in the affected components pedigree containing verifiable commit history and/or diff(s).", + "exploitable": "The vulnerability may be directly or indirectly exploitable.", + "in_triage": "The vulnerability is being investigated.", + "false_positive": "The vulnerability is not specific to the component or service and was falsely identified or associated.", + "not_affected": "The component or service is not affected by the vulnerability. Justification should be specified for all not_affected cases." + } + }, + "impactAnalysisJustification": { + "type": "string", + "title": "Impact Analysis Justification", + "description": "The rationale of why the impact analysis state was asserted.", + "enum": [ + "code_not_present", + "code_not_reachable", + "requires_configuration", + "requires_dependency", + "requires_environment", + "protected_by_compiler", + "protected_at_runtime", + "protected_at_perimeter", + "protected_by_mitigating_control" + ], + "meta:enum": { + "code_not_present": "The code has been removed or tree-shaked.", + "code_not_reachable": "The vulnerable code is not invoked at runtime.", + "requires_configuration": "Exploitability requires a configurable option to be set/unset.", + "requires_dependency": "Exploitability requires a dependency that is not present.", + "requires_environment": "Exploitability requires a certain environment which is not present.", + "protected_by_compiler": "Exploitability requires a compiler flag to be set/unset.", + "protected_at_runtime": "Exploits are prevented at runtime.", + "protected_at_perimeter": "Attacks are blocked at physical, logical, or network perimeter.", + "protected_by_mitigating_control": "Preventative measures have been implemented that reduce the likelihood and/or impact of the vulnerability." + } + }, + "rating": { + "type": "object", + "title": "Rating", + "description": "Defines the severity or risk ratings of a vulnerability.", + "additionalProperties": false, + "properties": { + "source": { + "$ref": "#/definitions/vulnerabilitySource", + "description": "The source that calculated the severity or risk rating of the vulnerability." + }, + "score": { + "type": "number", + "title": "Score", + "description": "The numerical score of the rating." + }, + "severity": { + "$ref": "#/definitions/severity", + "description": "Textual representation of the severity that corresponds to the numerical score of the rating." + }, + "method": { + "$ref": "#/definitions/scoreMethod" + }, + "vector": { + "type": "string", + "title": "Vector", + "description": "Textual representation of the metric values used to score the vulnerability" + }, + "justification": { + "type": "string", + "title": "Justification", + "description": "An optional reason for rating the vulnerability as it was" + } + } + }, + "vulnerabilitySource": { + "type": "object", + "title": "Source", + "description": "The source of vulnerability information. This is often the organization that published the vulnerability.", + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "title": "URL", + "description": "The url of the vulnerability documentation as provided by the source.", + "examples": [ + "https://nvd.nist.gov/vuln/detail/CVE-2021-39182" + ] + }, + "name": { + "type": "string", + "title": "Name", + "description": "The name of the source.", + "examples": [ + "NVD", + "National Vulnerability Database", + "OSS Index", + "VulnDB", + "GitHub Advisories" + ] + } + } + }, + "vulnerability": { + "type": "object", + "title": "Vulnerability", + "description": "Defines a weakness in a component or service that could be exploited or triggered by a threat source.", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the vulnerability elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "id": { + "type": "string", + "title": "ID", + "description": "The identifier that uniquely identifies the vulnerability.", + "examples": [ + "CVE-2021-39182", + "GHSA-35m5-8cvj-8783", + "SNYK-PYTHON-ENROCRYPT-1912876" + ] + }, + "source": { + "$ref": "#/definitions/vulnerabilitySource", + "description": "The source that published the vulnerability." + }, + "references": { + "type": "array", + "title": "References", + "description": "Zero or more pointers to vulnerabilities that are the equivalent of the vulnerability specified. Often times, the same vulnerability may exist in multiple sources of vulnerability intelligence, but have different identifiers. References provide a way to correlate vulnerabilities across multiple sources of vulnerability intelligence.", + "items": { + "type": "object", + "required": [ + "id", + "source" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "title": "ID", + "description": "An identifier that uniquely identifies the vulnerability.", + "examples": [ + "CVE-2021-39182", + "GHSA-35m5-8cvj-8783", + "SNYK-PYTHON-ENROCRYPT-1912876" + ] + }, + "source": { + "$ref": "#/definitions/vulnerabilitySource", + "description": "The source that published the vulnerability." + } + } + } + }, + "ratings": { + "type": "array", + "title": "Ratings", + "description": "List of vulnerability ratings. Consumers SHOULD consider ratings in prioritization decisions; source ratings may differ and aid prioritization.", + "items": { + "$ref": "#/definitions/rating" + } + }, + "cwes": { + "type": "array", + "title": "CWEs", + "description": "List of Common Weaknesses Enumerations (CWEs) codes that describes this vulnerability.", + "examples": [399], + "items": { + "$ref": "#/definitions/cwe" + } + }, + "description": { + "type": "string", + "title": "Description", + "description": "A description of the vulnerability as provided by the source." + }, + "detail": { + "type": "string", + "title": "Details", + "description": "If available, an in-depth description of the vulnerability as provided by the source organization. Details often include information useful in understanding root cause." + }, + "recommendation": { + "type": "string", + "title": "Recommendation", + "description": "Recommendations of how the vulnerability can be remediated or mitigated." + }, + "workaround": { + "type": "string", + "title": "Workarounds", + "description": "A bypass, usually temporary, of the vulnerability that reduces its likelihood and/or impact. Workarounds often involve changes to configuration or deployments." + }, + "proofOfConcept": { + "type": "object", + "title": "Proof of Concept", + "description": "Evidence used to reproduce the vulnerability.", + "properties": { + "reproductionSteps": { + "type": "string", + "title": "Steps to Reproduce", + "description": "Precise steps to reproduce the vulnerability." + }, + "environment": { + "type": "string", + "title": "Environment", + "description": "A description of the environment in which reproduction was possible." + }, + "supportingMaterial": { + "type": "array", + "title": "Supporting Material", + "description": "Supporting material that helps in reproducing or understanding how reproduction is possible. This may include screenshots, payloads, and PoC exploit code.", + "items": { "$ref": "#/definitions/attachment" } + } + } + }, + "advisories": { + "type": "array", + "title": "Advisories", + "description": "Published advisories of the vulnerability if provided.", + "items": { + "$ref": "#/definitions/advisory" + } + }, + "created": { + "type": "string", + "format": "date-time", + "title": "Created", + "description": "The date and time (timestamp) when the vulnerability record was created in the vulnerability database." + }, + "published": { + "type": "string", + "format": "date-time", + "title": "Published", + "description": "The date and time (timestamp) when the vulnerability record was first published." + }, + "updated": { + "type": "string", + "format": "date-time", + "title": "Updated", + "description": "The date and time (timestamp) when the vulnerability record was last updated." + }, + "rejected": { + "type": "string", + "format": "date-time", + "title": "Rejected", + "description": "The date and time (timestamp) when the vulnerability record was rejected (if applicable)." + }, + "credits": { + "type": "object", + "title": "Credits", + "description": "Individuals or organizations credited with the discovery of the vulnerability.", + "additionalProperties": false, + "properties": { + "organizations": { + "type": "array", + "title": "Organizations", + "description": "The organizations credited with vulnerability discovery.", + "items": { + "$ref": "#/definitions/organizationalEntity" + } + }, + "individuals": { + "type": "array", + "title": "Individuals", + "description": "The individuals, not associated with organizations, that are credited with vulnerability discovery.", + "items": { + "$ref": "#/definitions/organizationalContact" + } + } + } + }, + "tools": { + "title": "Tools", + "description": "The tool(s) used to identify, confirm, or score the vulnerability.", + "oneOf": [ + { + "type": "object", + "title": "Tools", + "description": "The tool(s) used to identify, confirm, or score the vulnerability.", + "additionalProperties": false, + "properties": { + "components": { + "type": "array", + "items": {"$ref": "#/definitions/component"}, + "uniqueItems": true, + "title": "Components", + "description": "A list of software and hardware components used as tools." + }, + "services": { + "type": "array", + "items": {"$ref": "#/definitions/service"}, + "uniqueItems": true, + "title": "Services", + "description": "A list of services used as tools. This may include microservices, function-as-a-service, and other types of network or intra-process services." + } + } + }, + { + "type": "array", + "title": "Tools (legacy)", + "description": "[Deprecated] The tool(s) used to identify, confirm, or score the vulnerability.", + "items": {"$ref": "#/definitions/tool"} + } + ] + }, + "analysis": { + "type": "object", + "title": "Impact Analysis", + "description": "An assessment of the impact and exploitability of the vulnerability.", + "additionalProperties": false, + "properties": { + "state": { + "$ref": "#/definitions/impactAnalysisState" + }, + "justification": { + "$ref": "#/definitions/impactAnalysisJustification" + }, + "response": { + "type": "array", + "title": "Response", + "description": "A response to the vulnerability by the manufacturer, supplier, or project responsible for the affected component or service. More than one response is allowed. Responses are strongly encouraged for vulnerabilities where the analysis state is exploitable.", + "items": { + "type": "string", + "enum": [ + "can_not_fix", + "will_not_fix", + "update", + "rollback", + "workaround_available" + ], + "meta:enum": { + "can_not_fix": "Can not fix", + "will_not_fix": "Will not fix", + "update": "Update to a different revision or release", + "rollback": "Revert to a previous revision or release", + "workaround_available": "There is a workaround available" + } + } + }, + "detail": { + "type": "string", + "title": "Detail", + "description": "Detailed description of the impact including methods used during assessment. If a vulnerability is not exploitable, this field should include specific details on why the component or service is not impacted by this vulnerability." + }, + "firstIssued": { + "type": "string", + "format": "date-time", + "title": "First Issued", + "description": "The date and time (timestamp) when the analysis was first issued." + }, + "lastUpdated": { + "type": "string", + "format": "date-time", + "title": "Last Updated", + "description": "The date and time (timestamp) when the analysis was last updated." + } + } + }, + "affects": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "object", + "required": [ + "ref" + ], + "additionalProperties": false, + "properties": { + "ref": { + "anyOf": [ + { + "title": "Ref", + "$ref": "#/definitions/refLinkType" + }, + { + "title": "BOM-Link Element", + "$ref": "#/definitions/bomLinkElementType" + } + ], + "title": "Reference", + "description": "References a component or service by the objects bom-ref" + }, + "versions": { + "type": "array", + "title": "Versions", + "description": "Zero or more individual versions or range of versions.", + "items": { + "type": "object", + "oneOf": [ + { + "required": ["version"] + }, + { + "required": ["range"] + } + ], + "additionalProperties": false, + "properties": { + "version": { + "title": "Version", + "description": "A single version of a component or service.", + "$ref": "#/definitions/version" + }, + "range": { + "title": "Version Range", + "description": "A version range specified in Package URL Version Range syntax (vers) which is defined at https://github.com/package-url/vers-spec", + "$ref": "#/definitions/versionRange" + }, + "status": { + "title": "Status", + "description": "The vulnerability status for the version or range of versions.", + "$ref": "#/definitions/affectedStatus", + "default": "affected" + } + } + } + } + } + }, + "title": "Affects", + "description": "The components or services that are affected by the vulnerability." + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "affectedStatus": { + "description": "The vulnerability status of a given version or range of versions of a product. The statuses 'affected' and 'unaffected' indicate that the version is affected or unaffected by the vulnerability. The status 'unknown' indicates that it is unknown or unspecified whether the given version is affected. There can be many reasons for an 'unknown' status, including that an investigation has not been undertaken or that a vendor has not disclosed the status.", + "type": "string", + "enum": [ + "affected", + "unaffected", + "unknown" + ], + "meta:enum": { + "affected": "The version is affected by the vulnerability.", + "unaffected": "The version is not affected by the vulnerability.", + "unknown": "It is unknown (or unspecified) whether the given version is affected." + } + }, + "version": { + "description": "A single disjunctive version identifier, for a component or service.", + "type": "string", + "maxLength": 1024, + "examples": [ + "9.0.14", + "v1.33.7", + "7.0.0-M1", + "2.0pre1", + "1.0.0-beta1", + "0.8.15" + ] + }, + "versionRange": { + "description": "A version range specified in Package URL Version Range syntax (vers) which is defined at https://github.com/package-url/vers-spec", + "type": "string", + "minLength": 1, + "maxLength": 4096, + "examples": [ + "vers:cargo/9.0.14", + "vers:npm/1.2.3|>=2.0.0|<5.0.0", + "vers:pypi/0.0.0|0.0.1|0.0.2|0.0.3|1.0|2.0pre1", + "vers:tomee/>=1.0.0-beta1|<=1.7.5|>=7.0.0-M1|<=7.0.7|>=7.1.0|<=7.1.2|>=8.0.0-M1|<=8.0.1", + "vers:gem/>=2.2.0|!= 2.2.1|<2.3.0" + ] + }, + "range": { + "deprecated": true, + "description": "Deprecated definition. use definition `versionRange` instead.", + "$ref": "#/definitions/versionRange" + }, + "annotations": { + "type": "object", + "title": "Annotations", + "description": "A comment, note, explanation, or similar textual content which provides additional context to the object(s) being annotated.", + "required": [ + "subjects", + "annotator", + "timestamp", + "text" + ], + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the annotation elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "subjects": { + "type": "array", + "uniqueItems": true, + "items": { + "anyOf": [ + { + "title": "Ref", + "$ref": "#/definitions/refLinkType" + }, + { + "title": "BOM-Link Element", + "$ref": "#/definitions/bomLinkElementType" + } + ] + }, + "title": "Subjects", + "description": "The object in the BOM identified by its bom-ref. This is often a component or service, but may be any object type supporting bom-refs." + }, + "annotator": { + "type": "object", + "title": "Annotator", + "description": "The organization, person, component, or service which created the textual content of the annotation.", + "oneOf": [ + { + "required": [ + "organization" + ] + }, + { + "required": [ + "individual" + ] + }, + { + "required": [ + "component" + ] + }, + { + "required": [ + "service" + ] + } + ], + "additionalProperties": false, + "properties": { + "organization": { + "description": "The organization that created the annotation", + "$ref": "#/definitions/organizationalEntity" + }, + "individual": { + "description": "The person that created the annotation", + "$ref": "#/definitions/organizationalContact" + }, + "component": { + "description": "The tool or component that created the annotation", + "$ref": "#/definitions/component" + }, + "service": { + "description": "The service that created the annotation", + "$ref": "#/definitions/service" + } + } + }, + "timestamp": { + "type": "string", + "format": "date-time", + "title": "Timestamp", + "description": "The date and time (timestamp) when the annotation was created." + }, + "text": { + "type": "string", + "title": "Text", + "description": "The textual content of the annotation." + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + }, + "modelCard": { + "$comment": "Model card support in CycloneDX is derived from TensorFlow Model Card Toolkit released under the Apache 2.0 license and available from https://github.com/tensorflow/model-card-toolkit/blob/main/model_card_toolkit/schema/v0.0.2/model_card.schema.json. In addition, CycloneDX model card support includes portions of VerifyML, also released under the Apache 2.0 license and available from https://github.com/cylynx/verifyml/blob/main/verifyml/model_card_toolkit/schema/v0.0.4/model_card.schema.json.", + "type": "object", + "title": "Model Card", + "description": "A model card describes the intended uses of a machine learning model and potential limitations, including biases and ethical considerations. Model cards typically contain the training parameters, which datasets were used to train the model, performance metrics, and other relevant data useful for ML transparency. This object SHOULD be specified for any component of type `machine-learning-model` and must not be specified for other component types.", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the model card elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "modelParameters": { + "type": "object", + "title": "Model Parameters", + "description": "Hyper-parameters for construction of the model.", + "additionalProperties": false, + "properties": { + "approach": { + "type": "object", + "title": "Approach", + "description": "The overall approach to learning used by the model for problem solving.", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "title": "Learning Type", + "description": "Learning types describing the learning problem or hybrid learning problem.", + "enum": [ + "supervised", + "unsupervised", + "reinforcement-learning", + "semi-supervised", + "self-supervised" + ], + "meta:enum": { + "supervised": "Supervised machine learning involves training an algorithm on labeled data to predict or classify new data based on the patterns learned from the labeled examples.", + "unsupervised": "Unsupervised machine learning involves training algorithms on unlabeled data to discover patterns, structures, or relationships without explicit guidance, allowing the model to identify inherent structures or clusters within the data.", + "reinforcement-learning": "Reinforcement learning is a type of machine learning where an agent learns to make decisions by interacting with an environment to maximize cumulative rewards, through trial and error.", + "semi-supervised": "Semi-supervised machine learning utilizes a combination of labeled and unlabeled data during training to improve model performance, leveraging the benefits of both supervised and unsupervised learning techniques.", + "self-supervised": "Self-supervised machine learning involves training models to predict parts of the input data from other parts of the same data, without requiring external labels, enabling learning from large amounts of unlabeled data." + } + } + } + }, + "task": { + "type": "string", + "title": "Task", + "description": "Directly influences the input and/or output. Examples include classification, regression, clustering, etc." + }, + "architectureFamily": { + "type": "string", + "title": "Architecture Family", + "description": "The model architecture family such as transformer network, convolutional neural network, residual neural network, LSTM neural network, etc." + }, + "modelArchitecture": { + "type": "string", + "title": "Model Architecture", + "description": "The specific architecture of the model such as GPT-1, ResNet-50, YOLOv3, etc." + }, + "datasets": { + "type": "array", + "title": "Datasets", + "description": "The datasets used to train and evaluate the model.", + "items" : { + "oneOf" : [ + { + "title": "Inline Data Information", + "$ref": "#/definitions/componentData" + }, + { + "type": "object", + "title": "Data Reference", + "additionalProperties": false, + "properties": { + "ref": { + "anyOf": [ + { + "title": "Ref", + "$ref": "#/definitions/refLinkType" + }, + { + "title": "BOM-Link Element", + "$ref": "#/definitions/bomLinkElementType" + } + ], + "title": "Reference", + "type": "string", + "description": "References a data component by the components bom-ref attribute" + } + } + } + ] + } + }, + "inputs": { + "type": "array", + "title": "Inputs", + "description": "The input format(s) of the model", + "items": { "$ref": "#/definitions/inputOutputMLParameters" } + }, + "outputs": { + "type": "array", + "title": "Outputs", + "description": "The output format(s) from the model", + "items": { "$ref": "#/definitions/inputOutputMLParameters" } + } + } + }, + "quantitativeAnalysis": { + "type": "object", + "title": "Quantitative Analysis", + "description": "A quantitative analysis of the model", + "additionalProperties": false, + "properties": { + "performanceMetrics": { + "type": "array", + "title": "Performance Metrics", + "description": "The model performance metrics being reported. Examples may include accuracy, F1 score, precision, top-3 error rates, MSC, etc.", + "items": { "$ref": "#/definitions/performanceMetric" } + }, + "graphics": { "$ref": "#/definitions/graphicsCollection" } + } + }, + "considerations": { + "type": "object", + "title": "Considerations", + "description": "What considerations should be taken into account regarding the model's construction, training, and application?", + "additionalProperties": false, + "properties": { + "users": { + "type": "array", + "title": "Users", + "description": "Who are the intended users of the model?", + "items": { + "type": "string" + } + }, + "useCases": { + "type": "array", + "title": "Use Cases", + "description": "What are the intended use cases of the model?", + "items": { + "type": "string" + } + }, + "technicalLimitations": { + "type": "array", + "title": "Technical Limitations", + "description": "What are the known technical limitations of the model? E.g. What kind(s) of data should the model be expected not to perform well on? What are the factors that might degrade model performance?", + "items": { + "type": "string" + } + }, + "performanceTradeoffs": { + "type": "array", + "title": "Performance Tradeoffs", + "description": "What are the known tradeoffs in accuracy/performance of the model?", + "items": { + "type": "string" + } + }, + "ethicalConsiderations": { + "type": "array", + "title": "Ethical Considerations", + "description": "What are the ethical risks involved in the application of this model?", + "items": { "$ref": "#/definitions/risk" } + }, + "environmentalConsiderations":{ + "$ref": "#/definitions/environmentalConsiderations", + "title": "Environmental Considerations", + "description": "What are the various environmental impacts the corresponding machine learning model has exhibited across its lifecycle?" + }, + "fairnessAssessments": { + "type": "array", + "title": "Fairness Assessments", + "description": "How does the model affect groups at risk of being systematically disadvantaged? What are the harms and benefits to the various affected groups?", + "items": { + "$ref": "#/definitions/fairnessAssessment" + } + } + } + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": {"$ref": "#/definitions/property"} + } + } + }, + "inputOutputMLParameters": { + "type": "object", + "title": "Input and Output Parameters", + "additionalProperties": false, + "properties": { + "format": { + "title": "Input/Output Format", + "description": "The data format for input/output to the model.", + "type": "string", + "examples": [ "string", "image", "time-series"] + } + } + }, + "componentData": { + "type": "object", + "additionalProperties": false, + "required": [ + "type" + ], + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the dataset elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links." + }, + "type": { + "type": "string", + "title": "Type of Data", + "description": "The general theme or subject matter of the data being specified.", + "enum": [ + "source-code", + "configuration", + "dataset", + "definition", + "other" + ], + "meta:enum": { + "source-code": "Any type of code, code snippet, or data-as-code.", + "configuration": "Parameters or settings that may be used by other components.", + "dataset": "A collection of data.", + "definition": "Data that can be used to create new instances of what the definition defines.", + "other": "Any other type of data that does not fit into existing definitions." + } + }, + "name": { + "title": "Dataset Name", + "description": "The name of the dataset.", + "type": "string" + }, + "contents": { + "type": "object", + "title": "Data Contents", + "description": "The contents or references to the contents of the data being described.", + "additionalProperties": false, + "properties": { + "attachment": { + "title": "Data Attachment", + "description": "An optional way to include textual or encoded data.", + "$ref": "#/definitions/attachment" + }, + "url": { + "type": "string", + "title": "Data URL", + "description": "The URL to where the data can be retrieved.", + "format": "iri-reference" + }, + "properties": { + "type": "array", + "title": "Configuration Properties", + "description": "Provides the ability to document name-value parameters used for configuration.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "classification": { + "$ref": "#/definitions/dataClassification" + }, + "sensitiveData": { + "type": "array", + "title": "Sensitive Data", + "description": "A description of any sensitive data in a dataset.", + "items": { + "type": "string" + } + }, + "graphics": { "$ref": "#/definitions/graphicsCollection" }, + "description": { + "title": "Dataset Description", + "description": "A description of the dataset. Can describe size of dataset, whether it's used for source code, training, testing, or validation, etc.", + "type": "string" + }, + "governance": { + "title": "Data Governance", + "$ref": "#/definitions/dataGovernance" + } + } + }, + "dataGovernance": { + "type": "object", + "title": "Data Governance", + "description": "Data governance captures information regarding data ownership, stewardship, and custodianship, providing insights into the individuals or entities responsible for managing, overseeing, and safeguarding the data throughout its lifecycle.", + "additionalProperties": false, + "properties": { + "custodians": { + "type": "array", + "title": "Data Custodians", + "description": "Data custodians are responsible for the safe custody, transport, and storage of data.", + "items": { "$ref": "#/definitions/dataGovernanceResponsibleParty" } + }, + "stewards": { + "type": "array", + "title": "Data Stewards", + "description": "Data stewards are responsible for data content, context, and associated business rules.", + "items": { "$ref": "#/definitions/dataGovernanceResponsibleParty" } + }, + "owners": { + "type": "array", + "title": "Data Owners", + "description": "Data owners are concerned with risk and appropriate access to data.", + "items": { "$ref": "#/definitions/dataGovernanceResponsibleParty" } + } + } + }, + "dataGovernanceResponsibleParty": { + "type": "object", + "additionalProperties": false, + "properties": { + "organization": { + "title": "Organization", + "description": "The organization that is responsible for specific data governance role(s).", + "$ref": "#/definitions/organizationalEntity" + }, + "contact": { + "title": "Individual", + "description": "The individual that is responsible for specific data governance role(s).", + "$ref": "#/definitions/organizationalContact" + } + }, + "oneOf":[ + { + "required": ["organization"] + }, + { + "required": ["contact"] + } + ] + }, + "graphicsCollection": { + "type": "object", + "title": "Graphics Collection", + "description": "A collection of graphics that represent various measurements.", + "additionalProperties": false, + "properties": { + "description": { + "title": "Description", + "description": "A description of this collection of graphics.", + "type": "string" + }, + "collection": { + "title": "Collection", + "description": "A collection of graphics.", + "type": "array", + "items": { "$ref": "#/definitions/graphic" } + } + } + }, + "graphic": { + "type": "object", + "title": "Graphic", + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "description": "The name of the graphic.", + "type": "string" + }, + "image": { + "title": "Graphic Image", + "description": "The graphic (vector or raster). Base64 encoding must be specified for binary images.", + "$ref": "#/definitions/attachment" + } + } + }, + "performanceMetric": { + "type": "object", + "title": "Performance Metric", + "additionalProperties": false, + "properties": { + "type": { + "title": "Type", + "description": "The type of performance metric.", + "type": "string" + }, + "value": { + "title": "Value", + "description": "The value of the performance metric.", + "type": "string" + }, + "slice": { + "title": "Slice", + "description": "The name of the slice this metric was computed on. By default, assume this metric is not sliced.", + "type": "string" + }, + "confidenceInterval": { + "title": "Confidence Interval", + "description": "The confidence interval of the metric.", + "type": "object", + "additionalProperties": false, + "properties": { + "lowerBound": { + "title": "Lower Bound", + "description": "The lower bound of the confidence interval.", + "type": "string" + }, + "upperBound": { + "title": "Upper Bound", + "description": "The upper bound of the confidence interval.", + "type": "string" + } + } + } + } + }, + "risk": { + "type": "object", + "title": "Risk", + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "description": "The name of the risk.", + "type": "string" + }, + "mitigationStrategy": { + "title": "Mitigation Strategy", + "description": "Strategy used to address this risk.", + "type": "string" + } + } + }, + "fairnessAssessment": { + "type": "object", + "title": "Fairness Assessment", + "description": "Information about the benefits and harms of the model to an identified at risk group.", + "additionalProperties": false, + "properties": { + "groupAtRisk": { + "type": "string", + "title": "Group at Risk", + "description": "The groups or individuals at risk of being systematically disadvantaged by the model." + }, + "benefits": { + "type": "string", + "title": "Benefits", + "description": "Expected benefits to the identified groups." + }, + "harms": { + "type": "string", + "title": "Harms", + "description": "Expected harms to the identified groups." + }, + "mitigationStrategy": { + "type": "string", + "title": "Mitigation Strategy", + "description": "With respect to the benefits and harms outlined, please describe any mitigation strategy implemented." + } + } + }, + "dataClassification": { + "type": "string", + "title": "Data Classification", + "description": "Data classification tags data according to its type, sensitivity, and value if altered, stolen, or destroyed." + }, + "environmentalConsiderations": { + "type": "object", + "title": "Environmental Considerations", + "description": "Describes various environmental impact metrics.", + "additionalProperties": false, + "properties": { + "energyConsumptions": { + "title": "Energy Consumptions", + "description": "Describes energy consumption information incurred for one or more component lifecycle activities.", + "type": "array", + "items": { + "$ref": "#/definitions/energyConsumption" + } + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "energyConsumption": { + "title": "Energy consumption", + "description": "Describes energy consumption information incurred for the specified lifecycle activity.", + "type": "object", + "required": [ + "activity", + "energyProviders", + "activityEnergyCost" + ], + "additionalProperties": false, + "properties": { + "activity": { + "type": "string", + "title": "Activity", + "description": "The type of activity that is part of a machine learning model development or operational lifecycle.", + "enum": [ + "design", + "data-collection", + "data-preparation", + "training", + "fine-tuning", + "validation", + "deployment", + "inference", + "other" + ], + "meta:enum": { + "design": "A model design including problem framing, goal definition and algorithm selection.", + "data-collection": "Model data acquisition including search, selection and transfer.", + "data-preparation": "Model data preparation including data cleaning, labeling and conversion.", + "training": "Model building, training and generalized tuning.", + "fine-tuning": "Refining a trained model to produce desired outputs for a given problem space.", + "validation": "Model validation including model output evaluation and testing.", + "deployment": "Explicit model deployment to a target hosting infrastructure.", + "inference": "Generating an output response from a hosted model from a set of inputs.", + "other": "A lifecycle activity type whose description does not match currently defined values." + } + }, + "energyProviders": { + "title": "Energy Providers", + "description": "The provider(s) of the energy consumed by the associated model development lifecycle activity.", + "type": "array", + "items": { "$ref": "#/definitions/energyProvider" } + }, + "activityEnergyCost": { + "title": "Activity Energy Cost", + "description": "The total energy cost associated with the model lifecycle activity.", + "$ref": "#/definitions/energyMeasure" + }, + "co2CostEquivalent": { + "title": "CO2 Equivalent Cost", + "description": "The CO2 cost (debit) equivalent to the total energy cost.", + "$ref": "#/definitions/co2Measure" + }, + "co2CostOffset": { + "title": "CO2 Cost Offset", + "description": "The CO2 offset (credit) for the CO2 equivalent cost.", + "$ref": "#/definitions/co2Measure" + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "energyMeasure": { + "type": "object", + "title": "Energy Measure", + "description": "A measure of energy.", + "required": [ + "value", + "unit" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": "number", + "title": "Value", + "description": "Quantity of energy." + }, + "unit": { + "type": "string", + "enum": [ "kWh" ], + "title": "Unit", + "description": "Unit of energy.", + "meta:enum": { + "kWh": "Kilowatt-hour (kWh) is the energy delivered by one kilowatt (kW) of power for one hour (h)." + } + } + } + }, + "co2Measure": { + "type": "object", + "title": "CO2 Measure", + "description": "A measure of carbon dioxide (CO2).", + "required": [ + "value", + "unit" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": "number", + "title": "Value", + "description": "Quantity of carbon dioxide (CO2)." + }, + "unit": { + "type": "string", + "enum": [ "tCO2eq" ], + "title": "Unit", + "description": "Unit of carbon dioxide (CO2).", + "meta:enum": { + "tCO2eq": "Tonnes (t) of carbon dioxide (CO2) equivalent (eq)." + } + } + } + }, + "energyProvider": { + "type": "object", + "title": "Energy Provider", + "description": "Describes the physical provider of energy used for model development or operations.", + "required": [ + "organization", + "energySource", + "energyProvided" + ], + "additionalProperties": false, + "properties": { + "bom-ref": { + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the energy provider elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links.", + "$ref": "#/definitions/refType" + }, + "description": { + "type": "string", + "title": "Description", + "description": "A description of the energy provider." + }, + "organization": { + "type": "object", + "title": "Organization", + "description": "The organization that provides energy.", + "$ref": "#/definitions/organizationalEntity" + }, + "energySource": { + "type": "string", + "enum": [ + "coal", + "oil", + "natural-gas", + "nuclear", + "wind", + "solar", + "geothermal", + "hydropower", + "biofuel", + "unknown", + "other" + ], + "meta:enum": { + "coal": "Energy produced by types of coal.", + "oil": "Petroleum products (primarily crude oil and its derivative fuel oils).", + "natural-gas": "Hydrocarbon gas liquids (HGL) that occur as gases at atmospheric pressure and as liquids under higher pressures including Natural gas (C5H12 and heavier), Ethane (C2H6), Propane (C3H8), etc.", + "nuclear": "Energy produced from the cores of atoms (i.e., through nuclear fission or fusion).", + "wind": "Energy produced from moving air.", + "solar": "Energy produced from the sun (i.e., solar radiation).", + "geothermal": "Energy produced from heat within the earth.", + "hydropower": "Energy produced from flowing water.", + "biofuel": "Liquid fuels produced from biomass feedstocks (i.e., organic materials such as plants or animals).", + "unknown": "The energy source is unknown.", + "other": "An energy source that is not listed." + }, + "title": "Energy Source", + "description": "The energy source for the energy provider." + }, + "energyProvided": { + "$ref": "#/definitions/energyMeasure", + "title": "Energy Provided", + "description": "The energy provided by the energy source for an associated activity." + }, + "externalReferences": { + "type": "array", + "items": {"$ref": "#/definitions/externalReference"}, + "title": "External References", + "description": "External references provide a way to document systems, sites, and information that may be relevant but are not included with the BOM. They may also establish specific relationships within or external to the BOM." + } + } + }, + "postalAddress": { + "type": "object", + "title": "Postal address", + "description": "An address used to identify a contactable location.", + "additionalProperties": false, + "properties": { + "bom-ref": { + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the address elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links.", + "$ref": "#/definitions/refType" + }, + "country": { + "type": "string", + "title": "Country", + "description": "The country name or the two-letter ISO 3166-1 country code." + }, + "region": { + "type": "string", + "title": "Region", + "description": "The region or state in the country.", + "examples": [ "Texas" ] + }, + "locality": { + "type": "string", + "title": "Locality", + "description": "The locality or city within the country.", + "examples": [ "Austin" ] + }, + "postOfficeBoxNumber": { + "type": "string", + "title": "Post Office Box Number", + "description": "The post office box number.", + "examples": [ "901" ] + }, + "postalCode": { + "type": "string", + "title": "Postal Code", + "description": "The postal code.", + "examples": [ "78758" ] + }, + "streetAddress": { + "type": "string", + "title": "Street Address", + "description": "The street address.", + "examples": [ "100 Main Street" ] + } + } + }, + "formula": { + "title": "Formula", + "description": "Describes workflows and resources that captures rules and other aspects of how the associated BOM component or service was formed.", + "type": "object", + "additionalProperties": false, + "properties": { + "bom-ref": { + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the formula elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links.", + "$ref": "#/definitions/refType" + }, + "components": { + "title": "Components", + "description": "Transient components that are used in tasks that constitute one or more of this formula's workflows", + "type": "array", + "items": { + "$ref": "#/definitions/component" + }, + "uniqueItems": true + }, + "services": { + "title": "Services", + "description": "Transient services that are used in tasks that constitute one or more of this formula's workflows", + "type": "array", + "items": { + "$ref": "#/definitions/service" + }, + "uniqueItems": true + }, + "workflows": { + "title": "Workflows", + "description": "List of workflows that can be declared to accomplish specific orchestrated goals and independently triggered.", + "$comment": "Different workflows can be designed to work together to perform end-to-end CI/CD builds and deployments.", + "type": "array", + "items": { + "$ref": "#/definitions/workflow" + }, + "uniqueItems": true + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "workflow": { + "title": "Workflow", + "description": "A specialized orchestration task.", + "$comment": "Workflow are as task themselves and can trigger other workflow tasks. These relationships can be modeled in the taskDependencies graph.", + "type": "object", + "required": [ + "bom-ref", + "uid", + "taskTypes" + ], + "additionalProperties": false, + "properties": { + "bom-ref": { + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the workflow elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links.", + "$ref": "#/definitions/refType" + }, + "uid": { + "title": "Unique Identifier (UID)", + "description": "The unique identifier for the resource instance within its deployment context.", + "type": "string" + }, + "name": { + "title": "Name", + "description": "The name of the resource instance.", + "type": "string" + }, + "description": { + "title": "Description", + "description": "A description of the resource instance.", + "type": "string" + }, + "resourceReferences": { + "title": "Resource references", + "description": "References to component or service resources that are used to realize the resource instance.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/resourceReferenceChoice" + } + }, + "tasks": { + "title": "Tasks", + "description": "The tasks that comprise the workflow.", + "$comment": "Note that tasks can appear more than once as different instances (by name or UID).", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/task" + } + }, + "taskDependencies": { + "title": "Task dependency graph", + "description": "The graph of dependencies between tasks within the workflow.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/dependency" + } + }, + "taskTypes": { + "title": "Task types", + "description": "Indicates the types of activities performed by the set of workflow tasks.", + "$comment": "Currently, these types reflect common CI/CD actions.", + "type": "array", + "items": { + "$ref": "#/definitions/taskType" + } + }, + "trigger": { + "title": "Trigger", + "description": "The trigger that initiated the task.", + "$ref": "#/definitions/trigger" + }, + "steps": { + "title": "Steps", + "description": "The sequence of steps for the task.", + "type": "array", + "items": { + "$ref": "#/definitions/step" + }, + "uniqueItems": true + }, + "inputs": { + "title": "Inputs", + "description": "Represents resources and data brought into a task at runtime by executor or task commands", + "examples": ["a `configuration` file which was declared as a local `component` or `externalReference`"], + "type": "array", + "items": { + "$ref": "#/definitions/inputType" + }, + "uniqueItems": true + }, + "outputs": { + "title": "Outputs", + "description": "Represents resources and data output from a task at runtime by executor or task commands", + "examples": ["a log file or metrics data produced by the task"], + "type": "array", + "items": { + "$ref": "#/definitions/outputType" + }, + "uniqueItems": true + }, + "timeStart": { + "title": "Time start", + "description": "The date and time (timestamp) when the task started.", + "type": "string", + "format": "date-time" + }, + "timeEnd": { + "title": "Time end", + "description": "The date and time (timestamp) when the task ended.", + "type": "string", + "format": "date-time" + }, + "workspaces": { + "title": "Workspaces", + "description": "A set of named filesystem or data resource shareable by workflow tasks.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/workspace" + } + }, + "runtimeTopology": { + "title": "Runtime topology", + "description": "A graph of the component runtime topology for workflow's instance.", + "$comment": "A description of the runtime component and service topology. This can describe a partial or complete topology used to host and execute the task (e.g., hardware, operating systems, configurations, etc.),", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/dependency" + } + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "task": { + "title": "Task", + "description": "Describes the inputs, sequence of steps and resources used to accomplish a task and its output.", + "$comment": "Tasks are building blocks for constructing assemble CI/CD workflows or pipelines.", + "type": "object", + "required": [ + "bom-ref", + "uid", + "taskTypes" + ], + "additionalProperties": false, + "properties": { + "bom-ref": { + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the task elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links.", + "$ref": "#/definitions/refType" + }, + "uid": { + "title": "Unique Identifier (UID)", + "description": "The unique identifier for the resource instance within its deployment context.", + "type": "string" + }, + "name": { + "title": "Name", + "description": "The name of the resource instance.", + "type": "string" + }, + "description": { + "title": "Description", + "description": "A description of the resource instance.", + "type": "string" + }, + "resourceReferences": { + "title": "Resource references", + "description": "References to component or service resources that are used to realize the resource instance.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/resourceReferenceChoice" + } + }, + "taskTypes": { + "title": "Task types", + "description": "Indicates the types of activities performed by the set of workflow tasks.", + "$comment": "Currently, these types reflect common CI/CD actions.", + "type": "array", + "items": { + "$ref": "#/definitions/taskType" + } + }, + "trigger": { + "title": "Trigger", + "description": "The trigger that initiated the task.", + "$ref": "#/definitions/trigger" + }, + "steps": { + "title": "Steps", + "description": "The sequence of steps for the task.", + "type": "array", + "items": { + "$ref": "#/definitions/step" + }, + "uniqueItems": true + }, + "inputs": { + "title": "Inputs", + "description": "Represents resources and data brought into a task at runtime by executor or task commands", + "examples": ["a `configuration` file which was declared as a local `component` or `externalReference`"], + "type": "array", + "items": { + "$ref": "#/definitions/inputType" + }, + "uniqueItems": true + }, + "outputs": { + "title": "Outputs", + "description": "Represents resources and data output from a task at runtime by executor or task commands", + "examples": ["a log file or metrics data produced by the task"], + "type": "array", + "items": { + "$ref": "#/definitions/outputType" + }, + "uniqueItems": true + }, + "timeStart": { + "title": "Time start", + "description": "The date and time (timestamp) when the task started.", + "type": "string", + "format": "date-time" + }, + "timeEnd": { + "title": "Time end", + "description": "The date and time (timestamp) when the task ended.", + "type": "string", + "format": "date-time" + }, + "workspaces": { + "title": "Workspaces", + "description": "A set of named filesystem or data resource shareable by workflow tasks.", + "type": "array", + "items": { + "$ref": "#/definitions/workspace" + }, + "uniqueItems": true + }, + "runtimeTopology": { + "title": "Runtime topology", + "description": "A graph of the component runtime topology for task's instance.", + "$comment": "A description of the runtime component and service topology. This can describe a partial or complete topology used to host and execute the task (e.g., hardware, operating systems, configurations, etc.),", + "type": "array", + "items": { + "$ref": "#/definitions/dependency" + }, + "uniqueItems": true + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "step": { + "type": "object", + "description": "Executes specific commands or tools in order to accomplish its owning task as part of a sequence.", + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "description": "A name for the step.", + "type": "string" + }, + "description": { + "title": "Description", + "description": "A description of the step.", + "type": "string" + }, + "commands": { + "title": "Commands", + "description": "Ordered list of commands or directives for the step", + "type": "array", + "items": { + "$ref": "#/definitions/command" + } + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "command": { + "type": "object", + "additionalProperties": false, + "properties": { + "executed": { + "title": "Executed", + "description": "A text representation of the executed command.", + "type": "string" + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "workspace": { + "title": "Workspace", + "description": "A named filesystem or data resource shareable by workflow tasks.", + "type": "object", + "required": [ + "bom-ref", + "uid" + ], + "additionalProperties": false, + "properties": { + "bom-ref": { + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the workspace elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links.", + "$ref": "#/definitions/refType" + }, + "uid": { + "title": "Unique Identifier (UID)", + "description": "The unique identifier for the resource instance within its deployment context.", + "type": "string" + }, + "name": { + "title": "Name", + "description": "The name of the resource instance.", + "type": "string" + }, + "aliases": { + "title": "Aliases", + "description": "The names for the workspace as referenced by other workflow tasks. Effectively, a name mapping so other tasks can use their own local name in their steps.", + "type": "array", + "items": {"type": "string"} + }, + "description": { + "title": "Description", + "description": "A description of the resource instance.", + "type": "string" + }, + "resourceReferences": { + "title": "Resource references", + "description": "References to component or service resources that are used to realize the resource instance.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/resourceReferenceChoice" + } + }, + "accessMode": { + "title": "Access mode", + "description": "Describes the read-write access control for the workspace relative to the owning resource instance.", + "type": "string", + "enum": [ + "read-only", + "read-write", + "read-write-once", + "write-once", + "write-only" + ] + }, + "mountPath": { + "title": "Mount path", + "description": "A path to a location on disk where the workspace will be available to the associated task's steps.", + "type": "string" + }, + "managedDataType": { + "title": "Managed data type", + "description": "The name of a domain-specific data type the workspace represents.", + "$comment": "This property is for CI/CD frameworks that are able to provide access to structured, managed data at a more granular level than a filesystem.", + "examples": ["ConfigMap","Secret"], + "type": "string" + }, + "volumeRequest": { + "title": "Volume request", + "description": "Identifies the reference to the request for a specific volume type and parameters.", + "examples": ["a kubernetes Persistent Volume Claim (PVC) name"], + "type": "string" + }, + "volume": { + "title": "Volume", + "description": "Information about the actual volume instance allocated to the workspace.", + "$comment": "The actual volume allocated may be different than the request.", + "examples": ["see https://kubernetes.io/docs/concepts/storage/persistent-volumes/"], + "$ref": "#/definitions/volume" + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "volume": { + "title": "Volume", + "description": "An identifiable, logical unit of data storage tied to a physical device.", + "type": "object", + "additionalProperties": false, + "properties": { + "uid": { + "title": "Unique Identifier (UID)", + "description": "The unique identifier for the volume instance within its deployment context.", + "type": "string" + }, + "name": { + "title": "Name", + "description": "The name of the volume instance", + "type": "string" + }, + "mode": { + "title": "Mode", + "description": "The mode for the volume instance.", + "type": "string", + "enum": [ + "filesystem", "block" + ], + "default": "filesystem" + }, + "path": { + "title": "Path", + "description": "The underlying path created from the actual volume.", + "type": "string" + }, + "sizeAllocated": { + "title": "Size allocated", + "description": "The allocated size of the volume accessible to the associated workspace. This should include the scalar size as well as IEC standard unit in either decimal or binary form.", + "examples": ["10GB", "2Ti", "1Pi"], + "type": "string" + }, + "persistent": { + "title": "Persistent", + "description": "Indicates if the volume persists beyond the life of the resource it is associated with.", + "type": "boolean" + }, + "remote": { + "title": "Remote", + "description": "Indicates if the volume is remotely (i.e., network) attached.", + "type": "boolean" + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "trigger": { + "title": "Trigger", + "description": "Represents a resource that can conditionally activate (or fire) tasks based upon associated events and their data.", + "type": "object", + "additionalProperties": false, + "required": [ + "type", + "bom-ref", + "uid" + ], + "properties": { + "bom-ref": { + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the trigger elsewhere in the BOM. Every bom-ref must be unique within the BOM.\nValue SHOULD not start with the BOM-Link intro 'urn:cdx:' to avoid conflicts with BOM-Links.", + "$ref": "#/definitions/refType" + }, + "uid": { + "title": "Unique Identifier (UID)", + "description": "The unique identifier for the resource instance within its deployment context.", + "type": "string" + }, + "name": { + "title": "Name", + "description": "The name of the resource instance.", + "type": "string" + }, + "description": { + "title": "Description", + "description": "A description of the resource instance.", + "type": "string" + }, + "resourceReferences": { + "title": "Resource references", + "description": "References to component or service resources that are used to realize the resource instance.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/resourceReferenceChoice" + } + }, + "type": { + "title": "Type", + "description": "The source type of event which caused the trigger to fire.", + "type": "string", + "enum": [ + "manual", + "api", + "webhook", + "scheduled" + ] + }, + "event": { + "title": "Event", + "description": "The event data that caused the associated trigger to activate.", + "$ref": "#/definitions/event" + }, + "conditions": { + "type": "array", + "title": "Conditions", + "description": "A list of conditions used to determine if a trigger should be activated.", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/condition" + } + }, + "timeActivated": { + "title": "Time activated", + "description": "The date and time (timestamp) when the trigger was activated.", + "type": "string", + "format": "date-time" + }, + "inputs": { + "title": "Inputs", + "description": "Represents resources and data brought into a task at runtime by executor or task commands", + "examples": ["a `configuration` file which was declared as a local `component` or `externalReference`"], + "type": "array", + "items": { + "$ref": "#/definitions/inputType" + }, + "uniqueItems": true + }, + "outputs": { + "title": "Outputs", + "description": "Represents resources and data output from a task at runtime by executor or task commands", + "examples": ["a log file or metrics data produced by the task"], + "type": "array", + "items": { + "$ref": "#/definitions/outputType" + }, + "uniqueItems": true + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "event": { + "title": "Event", + "description": "Represents something that happened that may trigger a response.", + "type": "object", + "additionalProperties": false, + "properties": { + "uid": { + "title": "Unique Identifier (UID)", + "description": "The unique identifier of the event.", + "type": "string" + }, + "description": { + "title": "Description", + "description": "A description of the event.", + "type": "string" + }, + "timeReceived": { + "title": "Time Received", + "description": "The date and time (timestamp) when the event was received.", + "type": "string", + "format": "date-time" + }, + "data": { + "title": "Data", + "description": "Encoding of the raw event data.", + "$ref": "#/definitions/attachment" + }, + "source": { + "title": "Source", + "description": "References the component or service that was the source of the event", + "$ref": "#/definitions/resourceReferenceChoice" + }, + "target": { + "title": "Target", + "description": "References the component or service that was the target of the event", + "$ref": "#/definitions/resourceReferenceChoice" + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "inputType": { + "title": "Input type", + "description": "Type that represents various input data types and formats.", + "type": "object", + "oneOf": [ + { + "required": [ + "resource" + ] + }, + { + "required": [ + "parameters" + ] + }, + { + "required": [ + "environmentVars" + ] + }, + { + "required": [ + "data" + ] + } + ], + "additionalProperties": false, + "properties": { + "source": { + "title": "Source", + "description": "A reference to the component or service that provided the input to the task (e.g., reference to a service with data flow value of `inbound`)", + "examples": [ + "source code repository", + "database" + ], + "$ref": "#/definitions/resourceReferenceChoice" + }, + "target": { + "title": "Target", + "description": "A reference to the component or service that received or stored the input if not the task itself (e.g., a local, named storage workspace)", + "examples": [ + "workspace", + "directory" + ], + "$ref": "#/definitions/resourceReferenceChoice" + }, + "resource": { + "title": "Resource", + "description": "A reference to an independent resource provided as an input to a task by the workflow runtime.", + "examples": [ + "a reference to a configuration file in a repository (i.e., a bom-ref)", + "a reference to a scanning service used in a task (i.e., a bom-ref)" + ], + "$ref": "#/definitions/resourceReferenceChoice" + }, + "parameters": { + "title": "Parameters", + "description": "Inputs that have the form of parameters with names and values.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/definitions/parameter" + } + }, + "environmentVars": { + "title": "Environment variables", + "description": "Inputs that have the form of parameters with names and values.", + "type": "array", + "uniqueItems": true, + "items": { + "oneOf": [ + { + "$ref": "#/definitions/property" + }, + { + "type": "string" + } + ] + } + }, + "data": { + "title": "Data", + "description": "Inputs that have the form of data.", + "$ref": "#/definitions/attachment" + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "outputType": { + "type": "object", + "oneOf": [ + { + "required": [ + "resource" + ] + }, + { + "required": [ + "environmentVars" + ] + }, + { + "required": [ + "data" + ] + } + ], + "additionalProperties": false, + "properties": { + "type": { + "title": "Type", + "description": "Describes the type of data output.", + "type": "string", + "enum": [ + "artifact", + "attestation", + "log", + "evidence", + "metrics", + "other" + ] + }, + "source": { + "title": "Source", + "description": "Component or service that generated or provided the output from the task (e.g., a build tool)", + "$ref": "#/definitions/resourceReferenceChoice" + }, + "target": { + "title": "Target", + "description": "Component or service that received the output from the task (e.g., reference to an artifactory service with data flow value of `outbound`)", + "examples": ["a log file described as an `externalReference` within its target domain."], + "$ref": "#/definitions/resourceReferenceChoice" + }, + "resource": { + "title": "Resource", + "description": "A reference to an independent resource generated as output by the task.", + "examples": [ + "configuration file", + "source code", + "scanning service" + ], + "$ref": "#/definitions/resourceReferenceChoice" + }, + "data": { + "title": "Data", + "description": "Outputs that have the form of data.", + "$ref": "#/definitions/attachment" + }, + "environmentVars": { + "title": "Environment variables", + "description": "Outputs that have the form of environment variables.", + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/definitions/property" + }, + { + "type": "string" + } + ] + }, + "uniqueItems": true + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "resourceReferenceChoice": { + "title": "Resource reference choice", + "description": "A reference to a locally defined resource (e.g., a bom-ref) or an externally accessible resource.", + "$comment": "Enables reference to a resource that participates in a workflow; using either internal (bom-ref) or external (externalReference) types.", + "type": "object", + "additionalProperties": false, + "properties": { + "ref": { + "title": "BOM Reference", + "description": "References an object by its bom-ref attribute", + "anyOf": [ + { + "title": "Ref", + "$ref": "#/definitions/refLinkType" + }, + { + "title": "BOM-Link Element", + "$ref": "#/definitions/bomLinkElementType" + } + ] + }, + "externalReference": { + "title": "External reference", + "description": "Reference to an externally accessible resource.", + "$ref": "#/definitions/externalReference" + } + }, + "oneOf": [ + { + "required": [ + "ref" + ] + }, + { + "required": [ + "externalReference" + ] + } + ] + }, + "condition": { + "title": "Condition", + "description": "A condition that was used to determine a trigger should be activated.", + "type": "object", + "additionalProperties": false, + "properties": { + "description": { + "title": "Description", + "description": "Describes the set of conditions which cause the trigger to activate.", + "type": "string" + }, + "expression": { + "title": "Expression", + "description": "The logical expression that was evaluated that determined the trigger should be fired.", + "type": "string" + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + } + } + }, + "taskType": { + "type": "string", + "enum": [ + "copy", + "clone", + "lint", + "scan", + "merge", + "build", + "test", + "deliver", + "deploy", + "release", + "clean", + "other" + ], + "meta:enum": { + "copy": "A task that copies software or data used to accomplish other tasks in the workflow.", + "clone": "A task that clones a software repository into the workflow in order to retrieve its source code or data for use in a build step.", + "lint": "A task that checks source code for programmatic and stylistic errors.", + "scan": "A task that performs a scan against source code, or built or deployed components and services. Scans are typically run to gather or test for security vulnerabilities or policy compliance.", + "merge": "A task that merges changes or fixes into source code prior to a build step in the workflow.", + "build": "A task that builds the source code, dependencies and/or data into an artifact that can be deployed to and executed on target systems.", + "test": "A task that verifies the functionality of a component or service.", + "deliver": "A task that delivers a built artifact to one or more target repositories or storage systems.", + "deploy": "A task that deploys a built artifact for execution on one or more target systems.", + "release": "A task that releases a built, versioned artifact to a target repository or distribution system.", + "clean": "A task that cleans unnecessary tools, build artifacts and/or data from workflow storage.", + "other": "A workflow task that does not match current task type definitions." + } + }, + "parameter": { + "title": "Parameter", + "description": "A representation of a functional parameter.", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "title": "Name", + "description": "The name of the parameter.", + "type": "string" + }, + "value": { + "title": "Value", + "description": "The value of the parameter.", + "type": "string" + }, + "dataType": { + "title": "Data type", + "description": "The data type of the parameter.", + "type": "string" + } + } + }, + "componentIdentityEvidence": { + "type": "object", + "title": "Identity Evidence", + "description": "Evidence that substantiates the identity of a component.", + "required": [ "field" ], + "additionalProperties": false, + "properties": { + "field": { + "type": "string", + "enum": [ + "group", "name", "version", "purl", "cpe", "omniborId", "swhid", "swid", "hash" + ], + "title": "Field", + "description": "The identity field of the component which the evidence describes." + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1, + "title": "Confidence", + "description": "The overall confidence of the evidence from 0 - 1, where 1 is 100% confidence." + }, + "concludedValue": { + "type": "string", + "title": "Concluded Value", + "description": "The value of the field (cpe, purl, etc) that has been concluded based on the aggregate of all methods (if available)." + }, + "methods": { + "type": "array", + "title": "Methods", + "description": "The methods used to extract and/or analyze the evidence.", + "items": { + "type": "object", + "required": [ + "technique" , + "confidence" + ], + "additionalProperties": false, + "properties": { + "technique": { + "title": "Technique", + "description": "The technique used in this method of analysis.", + "type": "string", + "enum": [ + "source-code-analysis", + "binary-analysis", + "manifest-analysis", + "ast-fingerprint", + "hash-comparison", + "instrumentation", + "dynamic-analysis", + "filename", + "attestation", + "other" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1, + "title": "Confidence", + "description": "The confidence of the evidence from 0 - 1, where 1 is 100% confidence. Confidence is specific to the technique used. Each technique of analysis can have independent confidence." + }, + "value": { + "type": "string", + "title": "Value", + "description": "The value or contents of the evidence." + } + } + } + }, + "tools": { + "type": "array", + "uniqueItems": true, + "items": { + "anyOf": [ + { + "title": "Ref", + "$ref": "#/definitions/refLinkType" + }, + { + "title": "BOM-Link Element", + "$ref": "#/definitions/bomLinkElementType" + } + ] + }, + "title": "BOM References", + "description": "The object in the BOM identified by its bom-ref. This is often a component or service but may be any object type supporting bom-refs. Tools used for analysis should already be defined in the BOM, either in the metadata/tools, components, or formulation." + } + } + }, + "standard": { + "type": "object", + "title": "Standard", + "description": "A standard may consist of regulations, industry or organizational-specific standards, maturity models, best practices, or any other requirements which can be evaluated against or attested to.", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the object elsewhere in the BOM. Every bom-ref must be unique within the BOM." + }, + "name": { + "type": "string", + "title": "Name", + "description": "The name of the standard. This will often be a shortened, single name of the standard." + }, + "version": { + "type": "string", + "title": "Version", + "description": "The version of the standard." + }, + "description": { + "type": "string", + "title": "Description", + "description": "The description of the standard." + }, + "owner": { + "type": "string", + "title": "Owner", + "description": "The owner of the standard, often the entity responsible for its release." + }, + "requirements": { + "type": "array", + "title": "Requirements", + "description": "The list of requirements comprising the standard.", + "items": { + "type": "object", + "title": "Requirement", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the object elsewhere in the BOM. Every bom-ref must be unique within the BOM." + }, + "identifier": { + "type": "string", + "title": "Identifier", + "description": "The unique identifier used in the standard to identify a specific requirement. This should match what is in the standard and should not be the requirements bom-ref." + }, + "title": { + "type": "string", + "title": "Title", + "description": "The title of the requirement." + }, + "text": { + "type": "string", + "title": "Text", + "description": "The textual content of the requirement." + }, + "descriptions": { + "type": "array", + "title": "Descriptions", + "description": "The supplemental text that provides additional guidance or context to the requirement, but is not directly part of the requirement.", + "items": { "type": "string" } + }, + "openCre": { + "type": "array", + "title": "OWASP OpenCRE Identifier(s)", + "description": "The Common Requirements Enumeration (CRE) identifier(s). CRE is a structured and standardized framework for uniting security standards and guidelines. CRE links each section of a resource to a shared topic identifier (a Common Requirement). Through this shared topic link, all resources map to each other. Use of CRE promotes clear and unambiguous communication among stakeholders.", + "items": { + "type": "string", + "pattern": "^CRE:[0-9]+-[0-9]+$", + "examples": [ "CRE:764-507" ] + } + }, + "parent": { + "$ref": "#/definitions/refLinkType", + "title": "Parent BOM Reference", + "description": "The optional `bom-ref` to a parent requirement. This establishes a hierarchy of requirements. Top-level requirements must not define a parent. Only child requirements should define parents." + }, + "properties": { + "type": "array", + "title": "Properties", + "description": "Provides the ability to document properties in a name-value store. This provides flexibility to include data not officially supported in the standard without having to use additional namespaces or create extensions. Unlike key-value stores, properties support duplicate names, each potentially having different values. Property names of interest to the general public are encouraged to be registered in the [CycloneDX Property Taxonomy](https://github.com/CycloneDX/cyclonedx-property-taxonomy). Formal registration is optional.", + "items": { + "$ref": "#/definitions/property" + } + }, + "externalReferences": { + "type": "array", + "items": {"$ref": "#/definitions/externalReference"}, + "title": "External References", + "description": "External references provide a way to document systems, sites, and information that may be relevant, but are not included with the BOM. They may also establish specific relationships within or external to the BOM." + } + } + } + }, + "levels": { + "type": "array", + "title": "Levels", + "description": "The list of levels associated with the standard. Some standards have different levels of compliance.", + "items": { + "type": "object", + "title": "Level", + "additionalProperties": false, + "properties": { + "bom-ref": { + "$ref": "#/definitions/refType", + "title": "BOM Reference", + "description": "An optional identifier which can be used to reference the object elsewhere in the BOM. Every bom-ref must be unique within the BOM." + }, + "identifier": { + "type": "string", + "title": "Identifier", + "description": "The identifier used in the standard to identify a specific level." + }, + "title": { + "type": "string", + "title": "Title", + "description": "The title of the level." + }, + "description": { + "type": "string", + "title": "Description", + "description": "The description of the level." + }, + "requirements": { + "type": "array", + "title": "Requirements", + "description": "The list of requirement `bom-ref`s that comprise the level.", + "items": { "$ref": "#/definitions/refLinkType" } + } + } + } + }, + "externalReferences": { + "type": "array", + "items": {"$ref": "#/definitions/externalReference"}, + "title": "External References", + "description": "External references provide a way to document systems, sites, and information that may be relevant but are not included with the BOM. They may also establish specific relationships within or external to the BOM." + }, + "signature": { + "$ref": "#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + } + } + }, + "signature": { + "$ref": "jsf-0.82.schema.json#/definitions/signature", + "title": "Signature", + "description": "Enveloped signature in [JSON Signature Format (JSF)](https://cyberphone.github.io/doc/security/jsf.html)." + }, + "cryptoProperties": { + "type": "object", + "title": "Cryptographic Properties", + "description": "Cryptographic assets have properties that uniquely define them and that make them actionable for further reasoning. As an example, it makes a difference if one knows the algorithm family (e.g. AES) or the specific variant or instantiation (e.g. AES-128-GCM). This is because the security level and the algorithm primitive (authenticated encryption) are only defined by the definition of the algorithm variant. The presence of a weak cryptographic algorithm like SHA1 vs. HMAC-SHA1 also makes a difference.", + "additionalProperties": false, + "required": [ + "assetType" + ], + "properties": { + "assetType": { + "type": "string", + "title": "Asset Type", + "description": "Cryptographic assets occur in several forms. Algorithms and protocols are most commonly implemented in specialized cryptographic libraries. They may, however, also be 'hardcoded' in software components. Certificates and related cryptographic material like keys, tokens, secrets or passwords are other cryptographic assets to be modelled.", + "enum": [ + "algorithm", + "certificate", + "protocol", + "related-crypto-material" + ], + "meta:enum": { + "algorithm": "Mathematical function commonly used for data encryption, authentication, and digital signatures.", + "certificate": "An electronic document that is used to provide the identity or validate a public key.", + "protocol": "A set of rules and guidelines that govern the behavior and communication with each other.", + "related-crypto-material": "Other cryptographic assets related to algorithms, certificates, and protocols such as keys and tokens." + } + }, + "algorithmProperties": { + "type": "object", + "title": "Algorithm Properties", + "description": "Additional properties specific to a cryptographic algorithm.", + "additionalProperties": false, + "properties": { + "primitive": { + "type": "string", + "title": "primitive", + "description": "Cryptographic building blocks used in higher-level cryptographic systems and protocols. Primitives represent different cryptographic routines: deterministic random bit generators (drbg, e.g. CTR_DRBG from NIST SP800-90A-r1), message authentication codes (mac, e.g. HMAC-SHA-256), blockciphers (e.g. AES), streamciphers (e.g. Salsa20), signatures (e.g. ECDSA), hash functions (e.g. SHA-256), public-key encryption schemes (pke, e.g. RSA), extended output functions (xof, e.g. SHAKE256), key derivation functions (e.g. pbkdf2), key agreement algorithms (e.g. ECDH), key encapsulation mechanisms (e.g. ML-KEM), authenticated encryption (ae, e.g. AES-GCM) and the combination of multiple algorithms (combiner, e.g. SP800-56Cr2).", + "enum": [ + "drbg", + "mac", + "block-cipher", + "stream-cipher", + "signature", + "hash", + "pke", + "xof", + "kdf", + "key-agree", + "kem", + "ae", + "combiner", + "other", + "unknown" + ], + "meta:enum": { + "drbg": "Deterministic Random Bit Generator (DRBG) is a type of pseudorandom number generator designed to produce a sequence of bits from an initial seed value. DRBGs are commonly used in cryptographic applications where reproducibility of random values is important.", + "mac": "In cryptography, a Message Authentication Code (MAC) is information used for authenticating and integrity-checking a message.", + "block-cipher": "A block cipher is a symmetric key algorithm that operates on fixed-size blocks of data. It encrypts or decrypts the data in block units, providing confidentiality. Block ciphers are widely used in various cryptographic modes and protocols for secure data transmission.", + "stream-cipher": "A stream cipher is a symmetric key cipher where plaintext digits are combined with a pseudorandom cipher digit stream (keystream).", + "signature": "In cryptography, a signature is a digital representation of a message or data that proves its origin, identity, and integrity. Digital signatures are generated using cryptographic algorithms and are widely used for authentication and verification in secure communication.", + "hash": "A hash function is a mathematical algorithm that takes an input (or 'message') and produces a fixed-size string of characters, which is typically a hash value. Hash functions are commonly used in various cryptographic applications, including data integrity verification and password hashing.", + "pke": "Public Key Encryption (PKE) is a type of encryption that uses a pair of public and private keys for secure communication. The public key is used for encryption, while the private key is used for decryption. PKE is a fundamental component of public-key cryptography.", + "xof": "An XOF is an extendable output function that can take arbitrary input and creates a stream of output, up to a limit determined by the size of the internal state of the hash function that underlies the XOF.", + "kdf": "A Key Derivation Function (KDF) derives key material from another source of entropy while preserving the entropy of the input.", + "key-agree": "In cryptography, a key-agreement is a protocol whereby two or more parties agree on a cryptographic key in such a way that both influence the outcome.", + "kem": "A Key Encapsulation Mechanism (KEM) algorithm is a mechanism for transporting random keying material to a recipient using the recipient's public key.", + "ae": "Authenticated Encryption (AE) is a cryptographic process that provides both confidentiality and data integrity. It ensures that the encrypted data has not been tampered with and comes from a legitimate source. AE is commonly used in secure communication protocols.", + "combiner": "A combiner aggregates many candidates for a cryptographic primitive and generates a new candidate for the same primitive.", + "other": "Another primitive type.", + "unknown": "The primitive is not known." + } + }, + "parameterSetIdentifier": { + "type": "string", + "title": "Parameter Set Identifier", + "description": "An identifier for the parameter set of the cryptographic algorithm. Examples: in AES128, '128' identifies the key length in bits, in SHA256, '256' identifies the digest length, '128' in SHAKE128 identifies its maximum security level in bits, and 'SHA2-128s' identifies a parameter set used in SLH-DSA (FIPS205)." + }, + "curve": { + "type": "string", + "title": "Elliptic Curve", + "description": "The specific underlying Elliptic Curve (EC) definition employed which is an indicator of the level of security strength, performance and complexity. Absent an authoritative source of curve names, CycloneDX recommends using curve names as defined at [https://neuromancer.sk/std/](https://neuromancer.sk/std/), the source of which can be found at [https://github.com/J08nY/std-curves](https://github.com/J08nY/std-curves)." + }, + "executionEnvironment": { + "type": "string", + "title": "Execution Environment", + "description": "The target and execution environment in which the algorithm is implemented in.", + "enum": [ + "software-plain-ram", + "software-encrypted-ram", + "software-tee", + "hardware", + "other", + "unknown" + ], + "meta:enum": { + "software-plain-ram": "A software implementation running in plain unencrypted RAM.", + "software-encrypted-ram": "A software implementation running in encrypted RAM.", + "software-tee": "A software implementation running in a trusted execution environment.", + "hardware": "A hardware implementation.", + "other": "Another implementation environment.", + "unknown": "The execution environment is not known." + } + }, + "implementationPlatform": { + "type": "string", + "title": "Implementation platform", + "description": "The target platform for which the algorithm is implemented. The implementation can be 'generic', running on any platform or for a specific platform.", + "enum": [ + "generic", + "x86_32", + "x86_64", + "armv7-a", + "armv7-m", + "armv8-a", + "armv8-m", + "armv9-a", + "armv9-m", + "s390x", + "ppc64", + "ppc64le", + "other", + "unknown" + ] + }, + "certificationLevel": { + "type": "array", + "title": "Certification Level", + "description": "The certification that the implementation of the cryptographic algorithm has received, if any. Certifications include revisions and levels of FIPS 140 or Common Criteria of different Extended Assurance Levels (CC-EAL).", + "items": { + "type": "string", + "enum": [ + "none", + "fips140-1-l1", + "fips140-1-l2", + "fips140-1-l3", + "fips140-1-l4", + "fips140-2-l1", + "fips140-2-l2", + "fips140-2-l3", + "fips140-2-l4", + "fips140-3-l1", + "fips140-3-l2", + "fips140-3-l3", + "fips140-3-l4", + "cc-eal1", + "cc-eal1+", + "cc-eal2", + "cc-eal2+", + "cc-eal3", + "cc-eal3+", + "cc-eal4", + "cc-eal4+", + "cc-eal5", + "cc-eal5+", + "cc-eal6", + "cc-eal6+", + "cc-eal7", + "cc-eal7+", + "other", + "unknown" + ], + "meta:enum": { + "none": "No certification obtained", + "fips140-1-l1": "FIPS 140-1 Level 1", + "fips140-1-l2": "FIPS 140-1 Level 2", + "fips140-1-l3": "FIPS 140-1 Level 3", + "fips140-1-l4": "FIPS 140-1 Level 4", + "fips140-2-l1": "FIPS 140-2 Level 1", + "fips140-2-l2": "FIPS 140-2 Level 2", + "fips140-2-l3": "FIPS 140-2 Level 3", + "fips140-2-l4": "FIPS 140-2 Level 4", + "fips140-3-l1": "FIPS 140-3 Level 1", + "fips140-3-l2": "FIPS 140-3 Level 2", + "fips140-3-l3": "FIPS 140-3 Level 3", + "fips140-3-l4": "FIPS 140-3 Level 4", + "cc-eal1": "Common Criteria - Evaluation Assurance Level 1", + "cc-eal1+": "Common Criteria - Evaluation Assurance Level 1 (Augmented)", + "cc-eal2": "Common Criteria - Evaluation Assurance Level 2", + "cc-eal2+": "Common Criteria - Evaluation Assurance Level 2 (Augmented)", + "cc-eal3": "Common Criteria - Evaluation Assurance Level 3", + "cc-eal3+": "Common Criteria - Evaluation Assurance Level 3 (Augmented)", + "cc-eal4": "Common Criteria - Evaluation Assurance Level 4", + "cc-eal4+": "Common Criteria - Evaluation Assurance Level 4 (Augmented)", + "cc-eal5": "Common Criteria - Evaluation Assurance Level 5", + "cc-eal5+": "Common Criteria - Evaluation Assurance Level 5 (Augmented)", + "cc-eal6": "Common Criteria - Evaluation Assurance Level 6", + "cc-eal6+": "Common Criteria - Evaluation Assurance Level 6 (Augmented)", + "cc-eal7": "Common Criteria - Evaluation Assurance Level 7", + "cc-eal7+": "Common Criteria - Evaluation Assurance Level 7 (Augmented)", + "other": "Another certification", + "unknown": "The certification level is not known" + } + } + }, + "mode": { + "type": "string", + "title": "Mode", + "description": "The mode of operation in which the cryptographic algorithm (block cipher) is used.", + "enum": [ + "cbc", + "ecb", + "ccm", + "gcm", + "cfb", + "ofb", + "ctr", + "other", + "unknown" + ], + "meta:enum": { + "cbc": "Cipher block chaining", + "ecb": "Electronic codebook", + "ccm": "Counter with cipher block chaining message authentication code", + "gcm": "Galois/counter", + "cfb": "Cipher feedback", + "ofb": "Output feedback", + "ctr": "Counter", + "other": "Another mode of operation", + "unknown": "The mode of operation is not known" + } + }, + "padding": { + "type": "string", + "title": "Padding", + "description": "The padding scheme that is used for the cryptographic algorithm.", + "enum": [ + "pkcs5", + "pkcs7", + "pkcs1v15", + "oaep", + "raw", + "other", + "unknown" + ], + "meta:enum": { + "pkcs5": "Public Key Cryptography Standard: Password-Based Cryptography", + "pkcs7": "Public Key Cryptography Standard: Cryptographic Message Syntax", + "pkcs1v15": "Public Key Cryptography Standard: RSA Cryptography v1.5", + "oaep": "Optimal asymmetric encryption padding", + "raw": "Raw", + "other": "Another padding scheme", + "unknown": "The padding scheme is not known" + } + }, + "cryptoFunctions": { + "type": "array", + "title": "Cryptographic functions", + "description": "The cryptographic functions implemented by the cryptographic algorithm.", + "items": { + "type": "string", + "enum": [ + "generate", + "keygen", + "encrypt", + "decrypt", + "digest", + "tag", + "keyderive", + "sign", + "verify", + "encapsulate", + "decapsulate", + "other", + "unknown" + ] + } + }, + "classicalSecurityLevel": { + "type": "integer", + "title": "classical security level", + "description": "The classical security level that a cryptographic algorithm provides (in bits).", + "minimum": 0 + }, + "nistQuantumSecurityLevel": { + "type": "integer", + "title": "NIST security strength category", + "description": "The NIST security strength category as defined in https://csrc.nist.gov/projects/post-quantum-cryptography/post-quantum-cryptography-standardization/evaluation-criteria/security-(evaluation-criteria). A value of 0 indicates that none of the categories are met.", + "minimum": 0, + "maximum": 6 + } + } + }, + "certificateProperties": { + "type": "object", + "title": "Certificate Properties", + "description": "Properties for cryptographic assets of asset type 'certificate'", + "additionalProperties": false, + "properties": { + "subjectName": { + "type": "string", + "title": "Subject Name", + "description": "The subject name for the certificate" + }, + "issuerName": { + "type": "string", + "title": "Issuer Name", + "description": "The issuer name for the certificate" + }, + "notValidBefore": { + "type": "string", + "format": "date-time", + "title": "Not Valid Before", + "description": "The date and time according to ISO-8601 standard from which the certificate is valid" + }, + "notValidAfter": { + "type": "string", + "format": "date-time", + "title": "Not Valid After", + "description": "The date and time according to ISO-8601 standard from which the certificate is not valid anymore" + }, + "signatureAlgorithmRef": { + "$ref": "#/definitions/refType", + "title": "Algorithm Reference", + "description": "The bom-ref to signature algorithm used by the certificate" + }, + "subjectPublicKeyRef": { + "$ref": "#/definitions/refType", + "title": "Key reference", + "description": "The bom-ref to the public key of the subject" + }, + "certificateFormat": { + "type": "string", + "title": "Certificate Format", + "description": "The format of the certificate", + "examples": [ + "X.509", + "PEM", + "DER", + "CVC" + ] + }, + "certificateExtension": { + "type": "string", + "title": "Certificate File Extension", + "description": "The file extension of the certificate", + "examples": [ + "crt", + "pem", + "cer", + "der", + "p12" + ] + } + } + }, + "relatedCryptoMaterialProperties": { + "type": "object", + "title": "Related Cryptographic Material Properties", + "description": "Properties for cryptographic assets of asset type: `related-crypto-material`", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "title": "relatedCryptoMaterialType", + "description": "The type for the related cryptographic material", + "enum": [ + "private-key", + "public-key", + "secret-key", + "key", + "ciphertext", + "signature", + "digest", + "initialization-vector", + "nonce", + "seed", + "salt", + "shared-secret", + "tag", + "additional-data", + "password", + "credential", + "token", + "other", + "unknown" + ], + "meta:enum": { + "private-key": "The confidential key of a key pair used in asymmetric cryptography.", + "public-key": "The non-confidential key of a key pair used in asymmetric cryptography.", + "secret-key": "A key used to encrypt and decrypt messages in symmetric cryptography.", + "key": "A piece of information, usually an octet string, which, when processed through a cryptographic algorithm, processes cryptographic data.", + "ciphertext": "The result of encryption performed on plaintext using an algorithm (or cipher).", + "signature": "A cryptographic value that is calculated from the data and a key known only by the signer.", + "digest": "The output of the hash function.", + "initialization-vector": "A fixed-size random or pseudo-random value used as an input parameter for cryptographic algorithms.", + "nonce": "A random or pseudo-random number that can only be used once in a cryptographic communication.", + "seed": "The input to a pseudo-random number generator. Different seeds generate different pseudo-random sequences.", + "salt": "A value used in a cryptographic process, usually to ensure that the results of computations for one instance cannot be reused by an attacker.", + "shared-secret": "A piece of data known only to the parties involved, in a secure communication.", + "tag": "A message authentication code (MAC), sometimes known as an authentication tag, is a short piece of information used for authenticating and integrity-checking a message.", + "additional-data": "An unspecified collection of data with relevance to cryptographic activity.", + "password": "A secret word, phrase, or sequence of characters used during authentication or authorization.", + "credential": "Establishes the identity of a party to communication, usually in the form of cryptographic keys or passwords.", + "token": "An object encapsulating a security identity.", + "other": "Another type of cryptographic asset.", + "unknown": "The type of cryptographic asset is not known." + } + }, + "id": { + "type": "string", + "title": "ID", + "description": "The optional unique identifier for the related cryptographic material." + }, + "state": { + "type": "string", + "title": "State", + "description": "The key state as defined by NIST SP 800-57.", + "enum": [ + "pre-activation", + "active", + "suspended", + "deactivated", + "compromised", + "destroyed" + ] + }, + "algorithmRef": { + "$ref": "#/definitions/refType", + "title": "Algorithm Reference", + "description": "The bom-ref to the algorithm used to generate the related cryptographic material." + }, + "creationDate": { + "type": "string", + "format": "date-time", + "title": "Creation Date", + "description": "The date and time (timestamp) when the related cryptographic material was created." + }, + "activationDate": { + "type": "string", + "format": "date-time", + "title": "Activation Date", + "description": "The date and time (timestamp) when the related cryptographic material was activated." + }, + "updateDate": { + "type": "string", + "format": "date-time", + "title": "Update Date", + "description": "The date and time (timestamp) when the related cryptographic material was updated." + }, + "expirationDate": { + "type": "string", + "format": "date-time", + "title": "Expiration Date", + "description": "The date and time (timestamp) when the related cryptographic material expires." + }, + "value": { + "type": "string", + "title": "Value", + "description": "The associated value of the cryptographic material." + }, + "size": { + "type": "integer", + "title": "Size", + "description": "The size of the cryptographic asset (in bits)." + }, + "format": { + "type": "string", + "title": "Format", + "description": "The format of the related cryptographic material (e.g. P8, PEM, DER)." + }, + "securedBy": { + "$ref": "#/definitions/securedBy", + "title": "Secured By", + "description": "The mechanism by which the cryptographic asset is secured by." + } + } + }, + "protocolProperties": { + "type": "object", + "title": "Protocol Properties", + "description": "Properties specific to cryptographic assets of type: `protocol`.", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "title": "Type", + "description": "The concrete protocol type.", + "enum": [ + "tls", + "ssh", + "ipsec", + "ike", + "sstp", + "wpa", + "other", + "unknown" + ], + "meta:enum": { + "tls": "Transport Layer Security", + "ssh": "Secure Shell", + "ipsec": "Internet Protocol Security", + "ike": "Internet Key Exchange", + "sstp": "Secure Socket Tunneling Protocol", + "wpa": "Wi-Fi Protected Access", + "other": "Another protocol type", + "unknown": "The protocol type is not known" + } + }, + "version": { + "type": "string", + "title": "Protocol Version", + "description": "The version of the protocol.", + "examples": [ + "1.0", + "1.2", + "1.99" + ] + }, + "cipherSuites": { + "type": "array", + "title": "Cipher Suites", + "description": "A list of cipher suites related to the protocol.", + "items": { + "$ref": "#/definitions/cipherSuite", + "title": "Cipher Suite" + } + }, + "ikev2TransformTypes": { + "type": "object", + "title": "IKEv2 Transform Types", + "description": "The IKEv2 transform types supported (types 1-4), defined in [RFC 7296 section 3.3.2](https://www.ietf.org/rfc/rfc7296.html#section-3.3.2), and additional properties.", + "additionalProperties": false, + "properties": { + "encr": { + "$ref": "#/definitions/cryptoRefArray", + "title": "Encryption Algorithm (ENCR)", + "description": "Transform Type 1: encryption algorithms" + }, + "prf": { + "$ref": "#/definitions/cryptoRefArray", + "title": "Pseudorandom Function (PRF)", + "description": "Transform Type 2: pseudorandom functions" + }, + "integ": { + "$ref": "#/definitions/cryptoRefArray", + "title": "Integrity Algorithm (INTEG)", + "description": "Transform Type 3: integrity algorithms" + }, + "ke": { + "$ref": "#/definitions/cryptoRefArray", + "title": "Key Exchange Method (KE)", + "description": "Transform Type 4: Key Exchange Method (KE) per [RFC 9370](https://www.ietf.org/rfc/rfc9370.html), formerly called Diffie-Hellman Group (D-H)." + }, + "esn": { + "type": "boolean", + "title": "Extended Sequence Numbers (ESN)", + "description": "Specifies if an Extended Sequence Number (ESN) is used." + }, + "auth": { + "$ref": "#/definitions/cryptoRefArray", + "title": "IKEv2 Authentication method", + "description": "IKEv2 Authentication method" + } + } + }, + "cryptoRefArray": { + "$ref": "#/definitions/cryptoRefArray", + "title": "Cryptographic References", + "description": "A list of protocol-related cryptographic assets" + } + } + }, + "oid": { + "type": "string", + "title": "OID", + "description": "The object identifier (OID) of the cryptographic asset." + } + } + }, + "cipherSuite": { + "type": "object", + "title": "Cipher Suite", + "description": "Object representing a cipher suite", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "title": "Common Name", + "description": "A common name for the cipher suite.", + "examples": [ + "TLS_DHE_RSA_WITH_AES_128_CCM" + ] + }, + "algorithms": { + "type": "array", + "title": "Related Algorithms", + "description": "A list of algorithms related to the cipher suite.", + "items": { + "$ref": "#/definitions/refType", + "title": "Algorithm reference", + "description": "The bom-ref to algorithm cryptographic asset." + } + }, + "identifiers": { + "type": "array", + "title": "Cipher Suite Identifiers", + "description": "A list of common identifiers for the cipher suite.", + "items": { + "type": "string", + "title": "identifier", + "description": "Cipher suite identifier", + "examples": [ + "0xC0", + "0x9E" + ] + } + } + } + }, + "cryptoRefArray" : { + "type": "array", + "items": { + "$ref": "#/definitions/refType" + } + }, + "securedBy": { + "type": "object", + "title": "Secured By", + "description": "Specifies the mechanism by which the cryptographic asset is secured by", + "additionalProperties": false, + "properties": { + "mechanism": { + "type": "string", + "title": "Mechanism", + "description": "Specifies the mechanism by which the cryptographic asset is secured by.", + "examples": [ + "HSM", + "TPM", + "SGX", + "Software", + "None" + ] + }, + "algorithmRef": { + "$ref": "#/definitions/refType", + "title": "Algorithm Reference", + "description": "The bom-ref to the algorithm." + } + } + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Tags", + "description": "Textual strings that aid in discovery, search, and retrieval of the associated object. Tags often serve as a way to group or categorize similar or related objects by various attributes.", + "examples": [ + "json-parser", + "object-persistence", + "text-to-image", + "translation", + "object-detection" + ] + } + } +} diff --git a/src/python_sbom/_validate/schemas/jsf-0.82.schema.json b/src/python_sbom/_validate/schemas/jsf-0.82.schema.json new file mode 100644 index 0000000..f46bfb1 --- /dev/null +++ b/src/python_sbom/_validate/schemas/jsf-0.82.schema.json @@ -0,0 +1,240 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://cyclonedx.org/schema/jsf-0.82.schema.json", + "type": "object", + "title": "JSON Signature Format (JSF) standard", + "$comment" : "JSON Signature Format schema is published under the terms of the Apache License 2.0. JSF was developed by Anders Rundgren (anders.rundgren.net@gmail.com) as a part of the OpenKeyStore project. This schema supports the entirely of the JSF standard excluding 'extensions'.", + "definitions": { + "signature": { + "type": "object", + "title": "Signature", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "signers": { + "type": "array", + "title": "Signature", + "description": "Unique top level property for Multiple Signatures. (multisignature)", + "items": {"$ref": "#/definitions/signer"} + } + } + }, + { + "additionalProperties": false, + "properties": { + "chain": { + "type": "array", + "title": "Signature", + "description": "Unique top level property for Signature Chains. (signaturechain)", + "items": {"$ref": "#/definitions/signer"} + } + } + }, + { + "title": "Signature", + "description": "Unique top level property for simple signatures. (signaturecore)", + "$ref": "#/definitions/signer" + } + ] + }, + "signer": { + "type": "object", + "title": "Signature", + "required": [ + "algorithm", + "value" + ], + "additionalProperties": false, + "properties": { + "algorithm": { + "oneOf": [ + { + "type": "string", + "title": "Algorithm", + "description": "Signature algorithm. The currently recognized JWA [RFC7518] and RFC8037 [RFC8037] asymmetric key algorithms. Note: Unlike RFC8037 [RFC8037] JSF requires explicit Ed* algorithm names instead of \"EdDSA\".", + "enum": [ + "RS256", + "RS384", + "RS512", + "PS256", + "PS384", + "PS512", + "ES256", + "ES384", + "ES512", + "Ed25519", + "Ed448", + "HS256", + "HS384", + "HS512" + ] + }, + { + "type": "string", + "title": "Algorithm", + "description": "Signature algorithm. Note: If proprietary signature algorithms are added, they must be expressed as URIs.", + "format": "uri" + } + ] + }, + "keyId": { + "type": "string", + "title": "Key ID", + "description": "Optional. Application specific string identifying the signature key." + }, + "publicKey": { + "title": "Public key", + "description": "Optional. Public key object.", + "$ref": "#/definitions/publicKey" + }, + "certificatePath": { + "type": "array", + "title": "Certificate path", + "description": "Optional. Sorted array of X.509 [RFC5280] certificates, where the first element must contain the signature certificate. The certificate path must be contiguous but is not required to be complete.", + "items": { + "type": "string" + } + }, + "excludes": { + "type": "array", + "title": "Excludes", + "description": "Optional. Array holding the names of one or more application level properties that must be excluded from the signature process. Note that the \"excludes\" property itself, must also be excluded from the signature process. Since both the \"excludes\" property and the associated data it points to are unsigned, a conforming JSF implementation must provide options for specifying which properties to accept.", + "items": { + "type": "string" + } + }, + "value": { + "type": "string", + "title": "Signature", + "description": "The signature data. Note that the binary representation must follow the JWA [RFC7518] specifications." + } + } + }, + "keyType": { + "type": "string", + "title": "Key type", + "description": "Key type indicator.", + "enum": [ + "EC", + "OKP", + "RSA" + ] + }, + "publicKey": { + "title": "Public key", + "description": "Optional. Public key object.", + "type": "object", + "required": [ + "kty" + ], + "additionalProperties": true, + "properties": { + "kty": { + "$ref": "#/definitions/keyType" + } + }, + "allOf": [ + { + "if": { + "properties": { "kty": { "const": "EC" } } + }, + "then": { + "required": [ + "kty", + "crv", + "x", + "y" + ], + "additionalProperties": false, + "properties": { + "kty": { + "$ref": "#/definitions/keyType" + }, + "crv": { + "type": "string", + "title": "Curve name", + "description": "EC curve name.", + "enum": [ + "P-256", + "P-384", + "P-521" + ] + }, + "x": { + "type": "string", + "title": "Coordinate", + "description": "EC curve point X. The length of this field must be the full size of a coordinate for the curve specified in the \"crv\" parameter. For example, if the value of \"crv\" is \"P-521\", the decoded argument must be 66 bytes." + }, + "y": { + "type": "string", + "title": "Coordinate", + "description": "EC curve point Y. The length of this field must be the full size of a coordinate for the curve specified in the \"crv\" parameter. For example, if the value of \"crv\" is \"P-256\", the decoded argument must be 32 bytes." + } + } + } + }, + { + "if": { + "properties": { "kty": { "const": "OKP" } } + }, + "then": { + "required": [ + "kty", + "crv", + "x" + ], + "additionalProperties": false, + "properties": { + "kty": { + "$ref": "#/definitions/keyType" + }, + "crv": { + "type": "string", + "title": "Curve name", + "description": "EdDSA curve name.", + "enum": [ + "Ed25519", + "Ed448" + ] + }, + "x": { + "type": "string", + "title": "Coordinate", + "description": "EdDSA curve point X. The length of this field must be the full size of a coordinate for the curve specified in the \"crv\" parameter. For example, if the value of \"crv\" is \"Ed25519\", the decoded argument must be 32 bytes." + } + } + } + }, + { + "if": { + "properties": { "kty": { "const": "RSA" } } + }, + "then": { + "required": [ + "kty", + "n", + "e" + ], + "additionalProperties": false, + "properties": { + "kty": { + "$ref": "#/definitions/keyType" + }, + "n": { + "type": "string", + "title": "Modulus", + "description": "RSA modulus." + }, + "e": { + "type": "string", + "title": "Exponent", + "description": "RSA exponent." + } + } + } + } + ] + } + } +} diff --git a/src/python_sbom/_validate/schemas/spdx.schema.json b/src/python_sbom/_validate/schemas/spdx.schema.json new file mode 100644 index 0000000..566c4a3 --- /dev/null +++ b/src/python_sbom/_validate/schemas/spdx.schema.json @@ -0,0 +1,1662 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://cyclonedx.org/schema/spdx.schema.json", + "$comment": "v1.1-3.29.0", + "type": "string", + "enum": [ + "0BSD", + "3D-Slicer-1.0", + "AAL", + "Abstyles", + "AdaCore-doc", + "Adobe-2006", + "Adobe-Display-PostScript", + "Adobe-Glyph", + "Adobe-Utopia", + "ADSL", + "Advanced-Cryptics-Dictionary", + "AFL-1.1", + "AFL-1.2", + "AFL-2.0", + "AFL-2.1", + "AFL-3.0", + "Afmparse", + "AGPL-1.0", + "AGPL-1.0-only", + "AGPL-1.0-or-later", + "AGPL-3.0", + "AGPL-3.0-only", + "AGPL-3.0-or-later", + "Aladdin", + "ALGLIB-Documentation", + "AMD-newlib", + "AMDPLPA", + "AML", + "AML-glslang", + "AMPAS", + "ANTLR-PD", + "ANTLR-PD-fallback", + "any-OSI", + "any-OSI-perl-modules", + "Apache-1.0", + "Apache-1.1", + "Apache-2.0", + "APAFML", + "APL-1.0", + "App-s2p", + "APSL-1.0", + "APSL-1.1", + "APSL-1.2", + "APSL-2.0", + "Arphic-1999", + "Artistic-1.0", + "Artistic-1.0-cl8", + "Artistic-1.0-Perl", + "Artistic-2.0", + "Artistic-dist", + "Aspell-RU", + "ASWF-Digital-Assets-1.0", + "ASWF-Digital-Assets-1.1", + "atc-game", + "Baekmuk", + "Bahyph", + "Barr", + "bcrypt-Solar-Designer", + "Beerware", + "Bitstream-Charter", + "Bitstream-Vera", + "BitTorrent-1.0", + "BitTorrent-1.1", + "blessing", + "BlueOak-1.0.0", + "Boehm-GC", + "Boehm-GC-without-fee", + "BOLA-1.1", + "Borceux", + "Brian-Gladman-2-Clause", + "Brian-Gladman-3-Clause", + "Brian-Gladman-3-Clause-no-conversion", + "BSD-1-Clause", + "BSD-2-Clause", + "BSD-2-Clause-Darwin", + "BSD-2-Clause-first-lines", + "BSD-2-Clause-FreeBSD", + "BSD-2-Clause-NetBSD", + "BSD-2-Clause-Patent", + "BSD-2-Clause-pkgconf-disclaimer", + "BSD-2-Clause-pos-unchanged", + "BSD-2-Clause-Views", + "BSD-3-Clause", + "BSD-3-Clause-acpica", + "BSD-3-Clause-Attribution", + "BSD-3-Clause-Clear", + "BSD-3-Clause-flex", + "BSD-3-Clause-HP", + "BSD-3-Clause-LBNL", + "BSD-3-Clause-Modification", + "BSD-3-Clause-No-Military-License", + "BSD-3-Clause-No-Nuclear-License", + "BSD-3-Clause-No-Nuclear-License-2014", + "BSD-3-Clause-No-Nuclear-Warranty", + "BSD-3-Clause-Open-MPI", + "BSD-3-Clause-OpenWebUI", + "BSD-3-Clause-Sun", + "BSD-3-Clause-Tso", + "BSD-4-Clause", + "BSD-4-Clause-Shortened", + "BSD-4-Clause-UC", + "BSD-4.3RENO", + "BSD-4.3TAHOE", + "BSD-Advertising-Acknowledgement", + "BSD-ask-to-endorse", + "BSD-Attribution-HPND-disclaimer", + "BSD-Inferno-Nettverk", + "BSD-Mark-Modifications", + "BSD-Protection", + "BSD-Source-alt-GPL", + "BSD-Source-beginning-file", + "BSD-Source-Code", + "BSD-Source-Code-no-disclaimer", + "BSD-Systemics", + "BSD-Systemics-W3Works", + "BSL-1.0", + "Buddy", + "Bugroff", + "BUSL-1.1", + "bzip2-1.0.5", + "bzip2-1.0.6", + "C-UDA-1.0", + "CAL-1.0", + "CAL-1.0-Combined-Work-Exception", + "Caldera", + "Caldera-no-preamble", + "CAPEC-tou", + "Catharon", + "CATOSL-1.1", + "CC-BY-1.0", + "CC-BY-2.0", + "CC-BY-2.5", + "CC-BY-2.5-AU", + "CC-BY-3.0", + "CC-BY-3.0-AT", + "CC-BY-3.0-AU", + "CC-BY-3.0-DE", + "CC-BY-3.0-IGO", + "CC-BY-3.0-NL", + "CC-BY-3.0-US", + "CC-BY-4.0", + "CC-BY-NC-1.0", + "CC-BY-NC-2.0", + "CC-BY-NC-2.5", + "CC-BY-NC-3.0", + "CC-BY-NC-3.0-DE", + "CC-BY-NC-3.0-IGO", + "CC-BY-NC-4.0", + "CC-BY-NC-ND-1.0", + "CC-BY-NC-ND-2.0", + "CC-BY-NC-ND-2.5", + "CC-BY-NC-ND-3.0", + "CC-BY-NC-ND-3.0-DE", + "CC-BY-NC-ND-3.0-IGO", + "CC-BY-NC-ND-4.0", + "CC-BY-NC-SA-1.0", + "CC-BY-NC-SA-2.0", + "CC-BY-NC-SA-2.0-DE", + "CC-BY-NC-SA-2.0-FR", + "CC-BY-NC-SA-2.0-UK", + "CC-BY-NC-SA-2.5", + "CC-BY-NC-SA-3.0", + "CC-BY-NC-SA-3.0-DE", + "CC-BY-NC-SA-3.0-IGO", + "CC-BY-NC-SA-4.0", + "CC-BY-ND-1.0", + "CC-BY-ND-2.0", + "CC-BY-ND-2.5", + "CC-BY-ND-3.0", + "CC-BY-ND-3.0-DE", + "CC-BY-ND-4.0", + "CC-BY-SA-1.0", + "CC-BY-SA-2.0", + "CC-BY-SA-2.0-UK", + "CC-BY-SA-2.1-JP", + "CC-BY-SA-2.5", + "CC-BY-SA-3.0", + "CC-BY-SA-3.0-AT", + "CC-BY-SA-3.0-DE", + "CC-BY-SA-3.0-IGO", + "CC-BY-SA-4.0", + "CC-PDDC", + "CC-PDM-1.0", + "CC-SA-1.0", + "CC0-1.0", + "CDDL-1.0", + "CDDL-1.1", + "CDL-1.0", + "CDLA-Permissive-1.0", + "CDLA-Permissive-2.0", + "CDLA-Sharing-1.0", + "CECILL-1.0", + "CECILL-1.1", + "CECILL-2.0", + "CECILL-2.1", + "CECILL-B", + "CECILL-C", + "CERN-OHL-1.1", + "CERN-OHL-1.2", + "CERN-OHL-P-2.0", + "CERN-OHL-S-2.0", + "CERN-OHL-W-2.0", + "CFITSIO", + "check-cvs", + "checkmk", + "ClArtistic", + "Clips", + "CMU-Mach", + "CMU-Mach-nodoc", + "CNRI-Jython", + "CNRI-Python", + "CNRI-Python-GPL-Compatible", + "COIL-1.0", + "Community-Spec-1.0", + "Condor-1.1", + "copyleft-next-0.3.0", + "copyleft-next-0.3.1", + "Cornell-Lossless-JPEG", + "CPAL-1.0", + "CPL-1.0", + "CPOL-1.02", + "Cronyx", + "Crossword", + "CryptoSwift", + "CrystalStacker", + "CUA-OPL-1.0", + "Cube", + "curl", + "cve-tou", + "D-FSL-1.0", + "DEC-3-Clause", + "diffmark", + "DL-DE-BY-2.0", + "DL-DE-ZERO-2.0", + "DOC", + "DocBook-DTD", + "DocBook-Schema", + "DocBook-Stylesheet", + "DocBook-XML", + "Dotseqn", + "DRL-1.0", + "DRL-1.1", + "DSDP", + "dtoa", + "dvipdfm", + "ECL-1.0", + "ECL-2.0", + "eCos-2.0", + "EFL-1.0", + "EFL-2.0", + "eGenix", + "Elastic-2.0", + "Entessa", + "EPICS", + "EPL-1.0", + "EPL-2.0", + "ErlPL-1.1", + "ESA-PL-permissive-2.4", + "ESA-PL-strong-copyleft-2.4", + "ESA-PL-weak-copyleft-2.4", + "etalab-2.0", + "EUDatagrid", + "EUPL-1.0", + "EUPL-1.1", + "EUPL-1.2", + "Eurosym", + "Fair", + "FBM", + "FDK-AAC", + "FDK-MPEG-H", + "Ferguson-Twofish", + "Frameworx-1.0", + "FreeBSD-DOC", + "FreeImage", + "FSFAP", + "FSFAP-no-warranty-disclaimer", + "FSFUL", + "FSFULLR", + "FSFULLRSD", + "FSFULLRWD", + "FSL-1.1-ALv2", + "FSL-1.1-MIT", + "FTL", + "Furuseth", + "fwlw", + "Game-Programming-Gems", + "GCR-docs", + "GD", + "generic-xts", + "GFDL-1.1", + "GFDL-1.1-invariants-only", + "GFDL-1.1-invariants-or-later", + "GFDL-1.1-no-invariants-only", + "GFDL-1.1-no-invariants-or-later", + "GFDL-1.1-only", + "GFDL-1.1-or-later", + "GFDL-1.2", + "GFDL-1.2-invariants-only", + "GFDL-1.2-invariants-or-later", + "GFDL-1.2-no-invariants-only", + "GFDL-1.2-no-invariants-or-later", + "GFDL-1.2-only", + "GFDL-1.2-or-later", + "GFDL-1.3", + "GFDL-1.3-invariants-only", + "GFDL-1.3-invariants-or-later", + "GFDL-1.3-no-invariants-only", + "GFDL-1.3-no-invariants-or-later", + "GFDL-1.3-only", + "GFDL-1.3-or-later", + "Giftware", + "GL2PS", + "Glide", + "Glulxe", + "GLWTPL", + "gnuplot", + "GPL-1.0", + "GPL-1.0+", + "GPL-1.0-only", + "GPL-1.0-or-later", + "GPL-2.0", + "GPL-2.0+", + "GPL-2.0-only", + "GPL-2.0-or-later", + "GPL-2.0-with-autoconf-exception", + "GPL-2.0-with-bison-exception", + "GPL-2.0-with-classpath-exception", + "GPL-2.0-with-font-exception", + "GPL-2.0-with-GCC-exception", + "GPL-3.0", + "GPL-3.0+", + "GPL-3.0-only", + "GPL-3.0-or-later", + "GPL-3.0-with-autoconf-exception", + "GPL-3.0-with-GCC-exception", + "Graphics-Gems", + "gSOAP-1.3b", + "gtkbook", + "Gutmann", + "HaskellReport", + "HDF5", + "hdparm", + "HIDAPI", + "Hippocratic-2.1", + "Hippocratic-3.0-core", + "HP-1986", + "HP-1989", + "HPND", + "HPND-DEC", + "HPND-doc", + "HPND-doc-sell", + "HPND-export-US", + "HPND-export-US-acknowledgement", + "HPND-export-US-modify", + "HPND-export2-US", + "HPND-Fenneberg-Livingston", + "HPND-INRIA-IMAG", + "HPND-Intel", + "HPND-Kevlin-Henney", + "HPND-Markus-Kuhn", + "HPND-merchantability-variant", + "HPND-MIT-disclaimer", + "HPND-Netrek", + "HPND-Pbmplus", + "HPND-sell-MIT-disclaimer-xserver", + "HPND-sell-regexpr", + "HPND-sell-variant", + "HPND-sell-variant-critical-systems", + "HPND-sell-variant-MIT-disclaimer", + "HPND-sell-variant-MIT-disclaimer-rev", + "HPND-SMC", + "HPND-UC", + "HPND-UC-export-US", + "HTMLTIDY", + "hyphen-bulgarian", + "IBM-pibs", + "ICU", + "IEC-Code-Components-EULA", + "IJG", + "IJG-short", + "ImageMagick", + "iMatix", + "Imlib2", + "Info-ZIP", + "Informatica", + "Inner-Net-2.0", + "InnoSetup", + "Intel", + "Intel-ACPI", + "Interbase-1.0", + "IPA", + "IPL-1.0", + "ISC", + "ISC-Veillard", + "ISO-permission", + "Jam", + "JasPer-2.0", + "jove", + "JPL-image", + "JPNIC", + "JSON", + "Kastrup", + "Kazlib", + "Knuth-CTAN", + "LAL-1.2", + "LAL-1.3", + "Latex2e", + "Latex2e-translated-notice", + "Leptonica", + "LGPL-2.0", + "LGPL-2.0+", + "LGPL-2.0-only", + "LGPL-2.0-or-later", + "LGPL-2.1", + "LGPL-2.1+", + "LGPL-2.1-only", + "LGPL-2.1-or-later", + "LGPL-3.0", + "LGPL-3.0+", + "LGPL-3.0-only", + "LGPL-3.0-or-later", + "LGPLLR", + "Libpng", + "libpng-1.6.35", + "libpng-2.0", + "libselinux-1.0", + "libtiff", + "libutil-David-Nugent", + "LiLiQ-P-1.1", + "LiLiQ-R-1.1", + "LiLiQ-Rplus-1.1", + "Linux-man-pages-1-para", + "Linux-man-pages-copyleft", + "Linux-man-pages-copyleft-2-para", + "Linux-man-pages-copyleft-var", + "Linux-OpenIB", + "LOOP", + "LPD-document", + "LPL-1.0", + "LPL-1.02", + "LPPL-1.0", + "LPPL-1.1", + "LPPL-1.2", + "LPPL-1.3a", + "LPPL-1.3c", + "lsof", + "Lucida-Bitmap-Fonts", + "LZMA-SDK-9.11-to-9.20", + "LZMA-SDK-9.22", + "Mackerras-3-Clause", + "Mackerras-3-Clause-acknowledgment", + "magaz", + "mailprio", + "MakeIndex", + "man2html", + "Martin-Birgmeier", + "McPhee-slideshow", + "metamail", + "Minpack", + "MIPS", + "MirOS", + "MIT", + "MIT-0", + "MIT-advertising", + "MIT-Click", + "MIT-CMU", + "MIT-enna", + "MIT-feh", + "MIT-Festival", + "MIT-Khronos-old", + "MIT-Modern-Variant", + "MIT-open-group", + "MIT-STK", + "MIT-testregex", + "MIT-Wu", + "MITNFA", + "MMIXware", + "MMPL-1.0.1", + "Motosoto", + "MPEG-SSG", + "mpi-permissive", + "mpich2", + "MPL-1.0", + "MPL-1.1", + "MPL-2.0", + "MPL-2.0-no-copyleft-exception", + "mplus", + "MS-LPL", + "MS-PL", + "MS-RL", + "MTLL", + "MulanPSL-1.0", + "MulanPSL-2.0", + "Multics", + "Mup", + "MVT-1.1", + "NAIST-2003", + "NASA-1.3", + "Naumen", + "NBPL-1.0", + "NCBI-PD", + "NCGL-UK-2.0", + "NCL", + "NCSA", + "Net-SNMP", + "NetCDF", + "Newsletr", + "NGPL", + "ngrep", + "NICTA-1.0", + "NIST-PD", + "NIST-PD-fallback", + "NIST-PD-TNT", + "NIST-Software", + "NLOD-1.0", + "NLOD-2.0", + "NLPL", + "Nokia", + "NOSL", + "Noweb", + "NPL-1.0", + "NPL-1.1", + "NPOSL-3.0", + "NRL", + "NTIA-PD", + "NTP", + "NTP-0", + "Nunit", + "O-UDA-1.0", + "OAR", + "OCCT-PL", + "OCLC-2.0", + "ODbL-1.0", + "ODC-By-1.0", + "OFFIS", + "OFL-1.0", + "OFL-1.0-no-RFN", + "OFL-1.0-RFN", + "OFL-1.1", + "OFL-1.1-no-RFN", + "OFL-1.1-RFN", + "OGC-1.0", + "OGDL-Taiwan-1.0", + "OGL-Canada-2.0", + "OGL-UK-1.0", + "OGL-UK-2.0", + "OGL-UK-3.0", + "OGTSL", + "OLDAP-1.1", + "OLDAP-1.2", + "OLDAP-1.3", + "OLDAP-1.4", + "OLDAP-2.0", + "OLDAP-2.0.1", + "OLDAP-2.1", + "OLDAP-2.2", + "OLDAP-2.2.1", + "OLDAP-2.2.2", + "OLDAP-2.3", + "OLDAP-2.4", + "OLDAP-2.5", + "OLDAP-2.6", + "OLDAP-2.7", + "OLDAP-2.8", + "OLFL-1.3", + "OML", + "OpenMDW-1.0", + "OpenPBS-2.3", + "OpenSSL", + "OpenSSL-standalone", + "OpenVision", + "OPL-1.0", + "OPL-UK-3.0", + "OPUBL-1.0", + "OSC-1.0", + "OSET-PL-2.1", + "OSL-1.0", + "OSL-1.1", + "OSL-2.0", + "OSL-2.1", + "OSL-3.0", + "OSSP", + "PADL", + "ParaType-Free-Font-1.3", + "Parity-6.0.0", + "Parity-7.0.0", + "PDDL-1.0", + "PHP-3.0", + "PHP-3.01", + "Pixar", + "pkgconf", + "Plexus", + "pnmstitch", + "PolyForm-Noncommercial-1.0.0", + "PolyForm-Small-Business-1.0.0", + "PostgreSQL", + "PPL", + "PSF-2.0", + "psfrag", + "psutils", + "Python-2.0", + "Python-2.0.1", + "python-ldap", + "Qhull", + "QPL-1.0", + "QPL-1.0-INRIA-2004", + "radvd", + "Rdisc", + "RHeCos-1.1", + "RPL-1.1", + "RPL-1.5", + "RPSL-1.0", + "RSA-MD", + "RSCPL", + "Ruby", + "Ruby-pty", + "SAX-PD", + "SAX-PD-2.0", + "Saxpath", + "SCEA", + "SchemeReport", + "Sendmail", + "Sendmail-8.23", + "Sendmail-Open-Source-1.1", + "SGI-B-1.0", + "SGI-B-1.1", + "SGI-B-2.0", + "SGI-OpenGL", + "SGMLUG-PM", + "SGP4", + "SHL-0.5", + "SHL-0.51", + "SimPL-2.0", + "SISSL", + "SISSL-1.2", + "SL", + "Sleepycat", + "SMAIL-GPL", + "SMLNJ", + "SMPPL", + "SNIA", + "snprintf", + "SOFA", + "softSurfer", + "Soundex", + "Spencer-86", + "Spencer-94", + "Spencer-99", + "SPL-1.0", + "ssh-keyscan", + "SSH-OpenSSH", + "SSH-short", + "SSLeay-standalone", + "SSPL-1.0", + "StandardML-NJ", + "SugarCRM-1.1.3", + "SUL-1.0", + "Sun-PPP", + "Sun-PPP-2000", + "SunPro", + "SWL", + "swrule", + "Symlinks", + "TAPR-OHL-1.0", + "TCL", + "TCP-wrappers", + "TekHVC", + "TermReadKey", + "TGPPL-1.0", + "ThirdEye", + "threeparttable", + "TMate", + "TORQUE-1.1", + "TOSL", + "TPDL", + "TPL-1.0", + "TrustedQSL", + "TTWL", + "TTYP0", + "TU-Berlin-1.0", + "TU-Berlin-2.0", + "Ubuntu-font-1.0", + "UCAR", + "UCL-1.0", + "ulem", + "UMich-Merit", + "Unicode-3.0", + "Unicode-DFS-2015", + "Unicode-DFS-2016", + "Unicode-TOU", + "UnixCrypt", + "Unlicense", + "Unlicense-libtelnet", + "Unlicense-libwhirlpool", + "UnRAR", + "UPL-1.0", + "URT-RLE", + "Vim", + "Vixie-Cron", + "VOSTROM", + "VSL-1.0", + "W3C", + "W3C-19980720", + "W3C-20150513", + "w3m", + "Watcom-1.0", + "Widget-Workshop", + "WordNet", + "Wsuipa", + "WTFNMFPL", + "WTFPL", + "wwl", + "wxWindows", + "X11", + "X11-distribute-modifications-variant", + "X11-no-permit-persons", + "X11-swapped", + "Xdebug-1.03", + "Xerox", + "Xfig", + "XFree86-1.1", + "xinetd", + "xkeyboard-config-Zinoviev", + "xlock", + "Xnet", + "xpp", + "XSkat", + "xzoom", + "YPL-1.0", + "YPL-1.1", + "Zed", + "Zeeff", + "Zend-2.0", + "Zimbra-1.3", + "Zimbra-1.4", + "Zlib", + "zlib-acknowledgement", + "ZPL-1.1", + "ZPL-2.0", + "ZPL-2.1", + "389-exception", + "Asterisk-exception", + "Asterisk-linking-protocols-exception", + "Autoconf-exception-2.0", + "Autoconf-exception-3.0", + "Autoconf-exception-generic", + "Autoconf-exception-generic-3.0", + "Autoconf-exception-macro", + "Bison-exception-1.24", + "Bison-exception-2.2", + "Bootloader-exception", + "CGAL-linking-exception", + "Classpath-exception-2.0", + "Classpath-exception-2.0-short", + "CLISP-exception-2.0", + "cryptsetup-OpenSSL-exception", + "Digia-Qt-LGPL-exception-1.1", + "DigiRule-FOSS-exception", + "eCos-exception-2.0", + "erlang-otp-linking-exception", + "Fawkes-Runtime-exception", + "FLTK-exception", + "fmt-exception", + "Font-exception-2.0", + "freertos-exception-2.0", + "GCC-exception-2.0", + "GCC-exception-2.0-note", + "GCC-exception-3.1", + "Gmsh-exception", + "GNAT-exception", + "GNOME-examples-exception", + "GNU-compiler-exception", + "gnu-javamail-exception", + "Google-Patent-WebM", + "GPL-3.0-389-ds-base-exception", + "GPL-3.0-interface-exception", + "GPL-3.0-linking-exception", + "GPL-3.0-linking-source-exception", + "GPL-CC-1.0", + "GStreamer-exception-2005", + "GStreamer-exception-2008", + "harbour-exception", + "i2p-gpl-java-exception", + "Independent-modules-exception", + "KiCad-libraries-exception", + "kvirc-openssl-exception", + "LGPL-3.0-linking-exception", + "libpri-OpenH323-exception", + "Libtool-exception", + "Linux-syscall-note", + "LLGPL", + "LLVM-exception", + "LZMA-exception", + "mif-exception", + "mxml-exception", + "Nokia-Qt-exception-1.1", + "OCaml-LGPL-linking-exception", + "OCCT-exception-1.0", + "OpenJDK-assembly-exception-1.0", + "openvpn-openssl-exception", + "PCRE2-exception", + "polyparse-exception", + "PS-or-PDF-font-exception-20170817", + "QPL-1.0-INRIA-2004-exception", + "Qt-GPL-exception-1.0", + "Qt-LGPL-exception-1.1", + "Qwt-exception-1.0", + "romic-exception", + "RRDtool-FLOSS-exception-2.0", + "rsync-linking-exception", + "SANE-exception", + "SHL-2.0", + "SHL-2.1", + "Simple-Library-Usage-exception", + "Spelling-Provider-LGPL-exception", + "sqlitestudio-OpenSSL-exception", + "stunnel-exception", + "SWI-exception", + "Swift-exception", + "Texinfo-exception", + "u-boot-exception-2.0", + "UBDL-exception", + "Universal-FOSS-exception-1.0", + "vsftpd-openssl-exception", + "WxWindows-exception-3.1", + "x11vnc-openssl-exception" + ], + "meta:enum": { + "0BSD": "BSD Zero Clause License", + "3D-Slicer-1.0": "3D Slicer License v1.0", + "AAL": "Attribution Assurance License", + "Abstyles": "Abstyles License", + "AdaCore-doc": "AdaCore Doc License", + "Adobe-2006": "Adobe Systems Incorporated Source Code License Agreement", + "Adobe-Display-PostScript": "Adobe Display PostScript License", + "Adobe-Glyph": "Adobe Glyph List License", + "Adobe-Utopia": "Adobe Utopia Font License", + "ADSL": "Amazon Digital Services License", + "Advanced-Cryptics-Dictionary": "Advanced Cryptics Dictionary License", + "AFL-1.1": "Academic Free License v1.1", + "AFL-1.2": "Academic Free License v1.2", + "AFL-2.0": "Academic Free License v2.0", + "AFL-2.1": "Academic Free License v2.1", + "AFL-3.0": "Academic Free License v3.0", + "Afmparse": "Afmparse License", + "AGPL-1.0": "Affero General Public License v1.0", + "AGPL-1.0-only": "Affero General Public License v1.0 only", + "AGPL-1.0-or-later": "Affero General Public License v1.0 or later", + "AGPL-3.0": "GNU Affero General Public License v3.0", + "AGPL-3.0-only": "GNU Affero General Public License v3.0 only", + "AGPL-3.0-or-later": "GNU Affero General Public License v3.0 or later", + "Aladdin": "Aladdin Free Public License", + "ALGLIB-Documentation": "ALGLIB Documentation License", + "AMD-newlib": "AMD newlib License", + "AMDPLPA": "AMD's plpa_map.c License", + "AML": "Apple MIT License", + "AML-glslang": "AML glslang variant License", + "AMPAS": "Academy of Motion Picture Arts and Sciences BSD", + "ANTLR-PD": "ANTLR Software Rights Notice", + "ANTLR-PD-fallback": "ANTLR Software Rights Notice with license fallback", + "any-OSI": "Any OSI License", + "any-OSI-perl-modules": "Any OSI License - Perl Modules", + "Apache-1.0": "Apache License 1.0", + "Apache-1.1": "Apache License 1.1", + "Apache-2.0": "Apache License 2.0", + "APAFML": "Adobe Postscript AFM License", + "APL-1.0": "Adaptive Public License 1.0", + "App-s2p": "App::s2p License", + "APSL-1.0": "Apple Public Source License 1.0", + "APSL-1.1": "Apple Public Source License 1.1", + "APSL-1.2": "Apple Public Source License 1.2", + "APSL-2.0": "Apple Public Source License 2.0", + "Arphic-1999": "Arphic Public License", + "Artistic-1.0": "Artistic License 1.0", + "Artistic-1.0-cl8": "Artistic License 1.0 w\/clause 8", + "Artistic-1.0-Perl": "Artistic License 1.0 (Perl)", + "Artistic-2.0": "Artistic License 2.0", + "Artistic-dist": "Artistic License 1.0 (dist)", + "Aspell-RU": "Aspell Russian License", + "ASWF-Digital-Assets-1.0": "ASWF Digital Assets License version 1.0", + "ASWF-Digital-Assets-1.1": "ASWF Digital Assets License 1.1", + "atc-game": "atc Game License", + "Baekmuk": "Baekmuk License", + "Bahyph": "Bahyph License", + "Barr": "Barr License", + "bcrypt-Solar-Designer": "bcrypt Solar Designer License", + "Beerware": "Beerware License", + "Bitstream-Charter": "Bitstream Charter Font License", + "Bitstream-Vera": "Bitstream Vera Font License", + "BitTorrent-1.0": "BitTorrent Open Source License v1.0", + "BitTorrent-1.1": "BitTorrent Open Source License v1.1", + "blessing": "SQLite Blessing", + "BlueOak-1.0.0": "Blue Oak Model License 1.0.0", + "Boehm-GC": "Boehm-Demers-Weiser GC License", + "Boehm-GC-without-fee": "Boehm-Demers-Weiser GC License (without fee)", + "BOLA-1.1": "Buena Onda License Agreement v1.1", + "Borceux": "Borceux license", + "Brian-Gladman-2-Clause": "Brian Gladman 2-Clause License", + "Brian-Gladman-3-Clause": "Brian Gladman 3-Clause License", + "Brian-Gladman-3-Clause-no-conversion": "Brian Gladman 3-Clause License (no conversion clause)", + "BSD-1-Clause": "BSD 1-Clause License", + "BSD-2-Clause": "BSD 2-Clause \"Simplified\" License", + "BSD-2-Clause-Darwin": "BSD 2-Clause - Ian Darwin variant", + "BSD-2-Clause-first-lines": "BSD 2-Clause - first lines requirement", + "BSD-2-Clause-FreeBSD": "BSD 2-Clause FreeBSD License", + "BSD-2-Clause-NetBSD": "BSD 2-Clause NetBSD License", + "BSD-2-Clause-Patent": "BSD-2-Clause Plus Patent License", + "BSD-2-Clause-pkgconf-disclaimer": "BSD 2-Clause pkgconf disclaimer variant", + "BSD-2-Clause-pos-unchanged": "BSD 2-Clause - position unchanged variant", + "BSD-2-Clause-Views": "BSD 2-Clause with views sentence", + "BSD-3-Clause": "BSD 3-Clause \"New\" or \"Revised\" License", + "BSD-3-Clause-acpica": "BSD 3-Clause acpica variant", + "BSD-3-Clause-Attribution": "BSD with attribution", + "BSD-3-Clause-Clear": "BSD 3-Clause Clear License", + "BSD-3-Clause-flex": "BSD 3-Clause Flex variant", + "BSD-3-Clause-HP": "Hewlett-Packard BSD variant license", + "BSD-3-Clause-LBNL": "Lawrence Berkeley National Labs BSD variant license", + "BSD-3-Clause-Modification": "BSD 3-Clause Modification", + "BSD-3-Clause-No-Military-License": "BSD 3-Clause No Military License", + "BSD-3-Clause-No-Nuclear-License": "BSD 3-Clause No Nuclear License", + "BSD-3-Clause-No-Nuclear-License-2014": "BSD 3-Clause No Nuclear License 2014", + "BSD-3-Clause-No-Nuclear-Warranty": "BSD 3-Clause No Nuclear Warranty", + "BSD-3-Clause-Open-MPI": "BSD 3-Clause Open MPI variant", + "BSD-3-Clause-OpenWebUI": "BSD 3-Clause - OpenWebUI variant", + "BSD-3-Clause-Sun": "BSD 3-Clause Sun Microsystems", + "BSD-3-Clause-Tso": "BSD 3-Clause Tso variant", + "BSD-4-Clause": "BSD 4-Clause \"Original\" or \"Old\" License", + "BSD-4-Clause-Shortened": "BSD 4 Clause Shortened", + "BSD-4-Clause-UC": "BSD-4-Clause (University of California-Specific)", + "BSD-4.3RENO": "BSD 4.3 RENO License", + "BSD-4.3TAHOE": "BSD 4.3 TAHOE License", + "BSD-Advertising-Acknowledgement": "BSD Advertising Acknowledgement License", + "BSD-ask-to-endorse": "BSD - ask to endorse", + "BSD-Attribution-HPND-disclaimer": "BSD with Attribution and HPND disclaimer", + "BSD-Inferno-Nettverk": "BSD-Inferno-Nettverk", + "BSD-Mark-Modifications": "BSD Mark Modifications License", + "BSD-Protection": "BSD Protection License", + "BSD-Source-alt-GPL": "BSD Source Code Attribution - GPL alternative", + "BSD-Source-beginning-file": "BSD Source Code Attribution - beginning of file variant", + "BSD-Source-Code": "BSD Source Code Attribution", + "BSD-Source-Code-no-disclaimer": "BSD Source Code Attribution - no disclaimer", + "BSD-Systemics": "Systemics BSD variant license", + "BSD-Systemics-W3Works": "Systemics W3Works BSD variant license", + "BSL-1.0": "Boost Software License 1.0", + "Buddy": "Buddy License", + "Bugroff": "Bugroff License", + "BUSL-1.1": "Business Source License 1.1", + "bzip2-1.0.5": "bzip2 and libbzip2 License v1.0.5", + "bzip2-1.0.6": "bzip2 and libbzip2 License v1.0.6", + "C-UDA-1.0": "Computational Use of Data Agreement v1.0", + "CAL-1.0": "Cryptographic Autonomy License 1.0", + "CAL-1.0-Combined-Work-Exception": "Cryptographic Autonomy License 1.0 (Combined Work Exception)", + "Caldera": "Caldera License", + "Caldera-no-preamble": "Caldera License (without preamble)", + "CAPEC-tou": "Common Attack Pattern Enumeration and Classification License", + "Catharon": "Catharon License", + "CATOSL-1.1": "Computer Associates Trusted Open Source License 1.1", + "CC-BY-1.0": "Creative Commons Attribution 1.0 Generic", + "CC-BY-2.0": "Creative Commons Attribution 2.0 Generic", + "CC-BY-2.5": "Creative Commons Attribution 2.5 Generic", + "CC-BY-2.5-AU": "Creative Commons Attribution 2.5 Australia", + "CC-BY-3.0": "Creative Commons Attribution 3.0 Unported", + "CC-BY-3.0-AT": "Creative Commons Attribution 3.0 Austria", + "CC-BY-3.0-AU": "Creative Commons Attribution 3.0 Australia", + "CC-BY-3.0-DE": "Creative Commons Attribution 3.0 Germany", + "CC-BY-3.0-IGO": "Creative Commons Attribution 3.0 IGO", + "CC-BY-3.0-NL": "Creative Commons Attribution 3.0 Netherlands", + "CC-BY-3.0-US": "Creative Commons Attribution 3.0 United States", + "CC-BY-4.0": "Creative Commons Attribution 4.0 International", + "CC-BY-NC-1.0": "Creative Commons Attribution Non Commercial 1.0 Generic", + "CC-BY-NC-2.0": "Creative Commons Attribution Non Commercial 2.0 Generic", + "CC-BY-NC-2.5": "Creative Commons Attribution Non Commercial 2.5 Generic", + "CC-BY-NC-3.0": "Creative Commons Attribution Non Commercial 3.0 Unported", + "CC-BY-NC-3.0-DE": "Creative Commons Attribution Non Commercial 3.0 Germany", + "CC-BY-NC-3.0-IGO": "Creative Commons Attribution Non Commercial 3.0 IGO", + "CC-BY-NC-4.0": "Creative Commons Attribution Non Commercial 4.0 International", + "CC-BY-NC-ND-1.0": "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic", + "CC-BY-NC-ND-2.0": "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic", + "CC-BY-NC-ND-2.5": "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic", + "CC-BY-NC-ND-3.0": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported", + "CC-BY-NC-ND-3.0-DE": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany", + "CC-BY-NC-ND-3.0-IGO": "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO", + "CC-BY-NC-ND-4.0": "Creative Commons Attribution Non Commercial No Derivatives 4.0 International", + "CC-BY-NC-SA-1.0": "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic", + "CC-BY-NC-SA-2.0": "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic", + "CC-BY-NC-SA-2.0-DE": "Creative Commons Attribution Non Commercial Share Alike 2.0 Germany", + "CC-BY-NC-SA-2.0-FR": "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France", + "CC-BY-NC-SA-2.0-UK": "Creative Commons Attribution Non Commercial Share Alike 2.0 England and Wales", + "CC-BY-NC-SA-2.5": "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic", + "CC-BY-NC-SA-3.0": "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported", + "CC-BY-NC-SA-3.0-DE": "Creative Commons Attribution Non Commercial Share Alike 3.0 Germany", + "CC-BY-NC-SA-3.0-IGO": "Creative Commons Attribution Non Commercial Share Alike 3.0 IGO", + "CC-BY-NC-SA-4.0": "Creative Commons Attribution Non Commercial Share Alike 4.0 International", + "CC-BY-ND-1.0": "Creative Commons Attribution No Derivatives 1.0 Generic", + "CC-BY-ND-2.0": "Creative Commons Attribution No Derivatives 2.0 Generic", + "CC-BY-ND-2.5": "Creative Commons Attribution No Derivatives 2.5 Generic", + "CC-BY-ND-3.0": "Creative Commons Attribution No Derivatives 3.0 Unported", + "CC-BY-ND-3.0-DE": "Creative Commons Attribution No Derivatives 3.0 Germany", + "CC-BY-ND-4.0": "Creative Commons Attribution No Derivatives 4.0 International", + "CC-BY-SA-1.0": "Creative Commons Attribution Share Alike 1.0 Generic", + "CC-BY-SA-2.0": "Creative Commons Attribution Share Alike 2.0 Generic", + "CC-BY-SA-2.0-UK": "Creative Commons Attribution Share Alike 2.0 England and Wales", + "CC-BY-SA-2.1-JP": "Creative Commons Attribution Share Alike 2.1 Japan", + "CC-BY-SA-2.5": "Creative Commons Attribution Share Alike 2.5 Generic", + "CC-BY-SA-3.0": "Creative Commons Attribution Share Alike 3.0 Unported", + "CC-BY-SA-3.0-AT": "Creative Commons Attribution Share Alike 3.0 Austria", + "CC-BY-SA-3.0-DE": "Creative Commons Attribution Share Alike 3.0 Germany", + "CC-BY-SA-3.0-IGO": "Creative Commons Attribution-ShareAlike 3.0 IGO", + "CC-BY-SA-4.0": "Creative Commons Attribution Share Alike 4.0 International", + "CC-PDDC": "Creative Commons Public Domain Dedication and Certification", + "CC-PDM-1.0": "Creative Commons Public Domain Mark 1.0 Universal", + "CC-SA-1.0": "Creative Commons Share Alike 1.0 Generic", + "CC0-1.0": "Creative Commons Zero v1.0 Universal", + "CDDL-1.0": "Common Development and Distribution License 1.0", + "CDDL-1.1": "Common Development and Distribution License 1.1", + "CDL-1.0": "Common Documentation License 1.0", + "CDLA-Permissive-1.0": "Community Data License Agreement Permissive 1.0", + "CDLA-Permissive-2.0": "Community Data License Agreement Permissive 2.0", + "CDLA-Sharing-1.0": "Community Data License Agreement Sharing 1.0", + "CECILL-1.0": "CeCILL Free Software License Agreement v1.0", + "CECILL-1.1": "CeCILL Free Software License Agreement v1.1", + "CECILL-2.0": "CeCILL Free Software License Agreement v2.0", + "CECILL-2.1": "CeCILL Free Software License Agreement v2.1", + "CECILL-B": "CeCILL-B Free Software License Agreement", + "CECILL-C": "CeCILL-C Free Software License Agreement", + "CERN-OHL-1.1": "CERN Open Hardware Licence v1.1", + "CERN-OHL-1.2": "CERN Open Hardware Licence v1.2", + "CERN-OHL-P-2.0": "CERN Open Hardware Licence Version 2 - Permissive", + "CERN-OHL-S-2.0": "CERN Open Hardware Licence Version 2 - Strongly Reciprocal", + "CERN-OHL-W-2.0": "CERN Open Hardware Licence Version 2 - Weakly Reciprocal", + "CFITSIO": "CFITSIO License", + "check-cvs": "check-cvs License", + "checkmk": "Checkmk License", + "ClArtistic": "Clarified Artistic License", + "Clips": "Clips License", + "CMU-Mach": "CMU Mach License", + "CMU-Mach-nodoc": "CMU Mach - no notices-in-documentation variant", + "CNRI-Jython": "CNRI Jython License", + "CNRI-Python": "CNRI Python License", + "CNRI-Python-GPL-Compatible": "CNRI Python Open Source GPL Compatible License Agreement", + "COIL-1.0": "Copyfree Open Innovation License", + "Community-Spec-1.0": "Community Specification License 1.0", + "Condor-1.1": "Condor Public License v1.1", + "copyleft-next-0.3.0": "copyleft-next 0.3.0", + "copyleft-next-0.3.1": "copyleft-next 0.3.1", + "Cornell-Lossless-JPEG": "Cornell Lossless JPEG License", + "CPAL-1.0": "Common Public Attribution License 1.0", + "CPL-1.0": "Common Public License 1.0", + "CPOL-1.02": "Code Project Open License 1.02", + "Cronyx": "Cronyx License", + "Crossword": "Crossword License", + "CryptoSwift": "CryptoSwift License", + "CrystalStacker": "CrystalStacker License", + "CUA-OPL-1.0": "CUA Office Public License v1.0", + "Cube": "Cube License", + "curl": "curl License", + "cve-tou": "Common Vulnerability Enumeration ToU License", + "D-FSL-1.0": "Deutsche Freie Software Lizenz", + "DEC-3-Clause": "DEC 3-Clause License", + "diffmark": "diffmark license", + "DL-DE-BY-2.0": "Data licence Germany \u2013 attribution \u2013 version 2.0", + "DL-DE-ZERO-2.0": "Data licence Germany \u2013 zero \u2013 version 2.0", + "DOC": "DOC License", + "DocBook-DTD": "DocBook DTD License", + "DocBook-Schema": "DocBook Schema License", + "DocBook-Stylesheet": "DocBook Stylesheet License", + "DocBook-XML": "DocBook XML License", + "Dotseqn": "Dotseqn License", + "DRL-1.0": "Detection Rule License 1.0", + "DRL-1.1": "Detection Rule License 1.1", + "DSDP": "DSDP License", + "dtoa": "David M. Gay dtoa License", + "dvipdfm": "dvipdfm License", + "ECL-1.0": "Educational Community License v1.0", + "ECL-2.0": "Educational Community License v2.0", + "eCos-2.0": "eCos license version 2.0", + "EFL-1.0": "Eiffel Forum License v1.0", + "EFL-2.0": "Eiffel Forum License v2.0", + "eGenix": "eGenix.com Public License 1.1.0", + "Elastic-2.0": "Elastic License 2.0", + "Entessa": "Entessa Public License v1.0", + "EPICS": "EPICS Open License", + "EPL-1.0": "Eclipse Public License 1.0", + "EPL-2.0": "Eclipse Public License 2.0", + "ErlPL-1.1": "Erlang Public License v1.1", + "ESA-PL-permissive-2.4": "European Space Agency Public License \u2013 v2.4 \u2013 Permissive (Type 3)", + "ESA-PL-strong-copyleft-2.4": "European Space Agency Public License (ESA-PL) - V2.4 - Strong Copyleft (Type 1)", + "ESA-PL-weak-copyleft-2.4": "European Space Agency Public License \u2013 v2.4 \u2013 Weak Copyleft (Type 2)", + "etalab-2.0": "Etalab Open License 2.0", + "EUDatagrid": "EU DataGrid Software License", + "EUPL-1.0": "European Union Public License 1.0", + "EUPL-1.1": "European Union Public License 1.1", + "EUPL-1.2": "European Union Public License 1.2", + "Eurosym": "Eurosym License", + "Fair": "Fair License", + "FBM": "Fuzzy Bitmap License", + "FDK-AAC": "Fraunhofer FDK AAC Codec Library", + "FDK-MPEG-H": "Fraunhofer FDK MPEG-H Software", + "Ferguson-Twofish": "Ferguson Twofish License", + "Frameworx-1.0": "Frameworx Open License 1.0", + "FreeBSD-DOC": "FreeBSD Documentation License", + "FreeImage": "FreeImage Public License v1.0", + "FSFAP": "FSF All Permissive License", + "FSFAP-no-warranty-disclaimer": "FSF All Permissive License (without Warranty)", + "FSFUL": "FSF Unlimited License", + "FSFULLR": "FSF Unlimited License (with License Retention)", + "FSFULLRSD": "FSF Unlimited License (with License Retention and Short Disclaimer)", + "FSFULLRWD": "FSF Unlimited License (With License Retention and Warranty Disclaimer)", + "FSL-1.1-ALv2": "Functional Source License, Version 1.1, ALv2 Future License", + "FSL-1.1-MIT": "Functional Source License, Version 1.1, MIT Future License", + "FTL": "Freetype Project License", + "Furuseth": "Furuseth License", + "fwlw": "fwlw License", + "Game-Programming-Gems": "Game Programming Gems License", + "GCR-docs": "Gnome GCR Documentation License", + "GD": "GD License", + "generic-xts": "Generic XTS License", + "GFDL-1.1": "GNU Free Documentation License v1.1", + "GFDL-1.1-invariants-only": "GNU Free Documentation License v1.1 only - invariants", + "GFDL-1.1-invariants-or-later": "GNU Free Documentation License v1.1 or later - invariants", + "GFDL-1.1-no-invariants-only": "GNU Free Documentation License v1.1 only - no invariants", + "GFDL-1.1-no-invariants-or-later": "GNU Free Documentation License v1.1 or later - no invariants", + "GFDL-1.1-only": "GNU Free Documentation License v1.1 only", + "GFDL-1.1-or-later": "GNU Free Documentation License v1.1 or later", + "GFDL-1.2": "GNU Free Documentation License v1.2", + "GFDL-1.2-invariants-only": "GNU Free Documentation License v1.2 only - invariants", + "GFDL-1.2-invariants-or-later": "GNU Free Documentation License v1.2 or later - invariants", + "GFDL-1.2-no-invariants-only": "GNU Free Documentation License v1.2 only - no invariants", + "GFDL-1.2-no-invariants-or-later": "GNU Free Documentation License v1.2 or later - no invariants", + "GFDL-1.2-only": "GNU Free Documentation License v1.2 only", + "GFDL-1.2-or-later": "GNU Free Documentation License v1.2 or later", + "GFDL-1.3": "GNU Free Documentation License v1.3", + "GFDL-1.3-invariants-only": "GNU Free Documentation License v1.3 only - invariants", + "GFDL-1.3-invariants-or-later": "GNU Free Documentation License v1.3 or later - invariants", + "GFDL-1.3-no-invariants-only": "GNU Free Documentation License v1.3 only - no invariants", + "GFDL-1.3-no-invariants-or-later": "GNU Free Documentation License v1.3 or later - no invariants", + "GFDL-1.3-only": "GNU Free Documentation License v1.3 only", + "GFDL-1.3-or-later": "GNU Free Documentation License v1.3 or later", + "Giftware": "Giftware License", + "GL2PS": "GL2PS License", + "Glide": "3dfx Glide License", + "Glulxe": "Glulxe License", + "GLWTPL": "Good Luck With That Public License", + "gnuplot": "gnuplot License", + "GPL-1.0": "GNU General Public License v1.0 only", + "GPL-1.0+": "GNU General Public License v1.0 or later", + "GPL-1.0-only": "GNU General Public License v1.0 only", + "GPL-1.0-or-later": "GNU General Public License v1.0 or later", + "GPL-2.0": "GNU General Public License v2.0 only", + "GPL-2.0+": "GNU General Public License v2.0 or later", + "GPL-2.0-only": "GNU General Public License v2.0 only", + "GPL-2.0-or-later": "GNU General Public License v2.0 or later", + "GPL-2.0-with-autoconf-exception": "GNU General Public License v2.0 w\/Autoconf exception", + "GPL-2.0-with-bison-exception": "GNU General Public License v2.0 w\/Bison exception", + "GPL-2.0-with-classpath-exception": "GNU General Public License v2.0 w\/Classpath exception", + "GPL-2.0-with-font-exception": "GNU General Public License v2.0 w\/Font exception", + "GPL-2.0-with-GCC-exception": "GNU General Public License v2.0 w\/GCC Runtime Library exception", + "GPL-3.0": "GNU General Public License v3.0 only", + "GPL-3.0+": "GNU General Public License v3.0 or later", + "GPL-3.0-only": "GNU General Public License v3.0 only", + "GPL-3.0-or-later": "GNU General Public License v3.0 or later", + "GPL-3.0-with-autoconf-exception": "GNU General Public License v3.0 w\/Autoconf exception", + "GPL-3.0-with-GCC-exception": "GNU General Public License v3.0 w\/GCC Runtime Library exception", + "Graphics-Gems": "Graphics Gems License", + "gSOAP-1.3b": "gSOAP Public License v1.3b", + "gtkbook": "gtkbook License", + "Gutmann": "Gutmann License", + "HaskellReport": "Haskell Language Report License", + "HDF5": "HDF5 License", + "hdparm": "hdparm License", + "HIDAPI": "HIDAPI License", + "Hippocratic-2.1": "Hippocratic License 2.1", + "Hippocratic-3.0-core": "Hippocratic License 3.0", + "HP-1986": "Hewlett-Packard 1986 License", + "HP-1989": "Hewlett-Packard 1989 License", + "HPND": "Historical Permission Notice and Disclaimer", + "HPND-DEC": "Historical Permission Notice and Disclaimer - DEC variant", + "HPND-doc": "Historical Permission Notice and Disclaimer - documentation variant", + "HPND-doc-sell": "Historical Permission Notice and Disclaimer - documentation sell variant", + "HPND-export-US": "HPND with US Government export control warning", + "HPND-export-US-acknowledgement": "HPND with US Government export control warning and acknowledgment", + "HPND-export-US-modify": "HPND with US Government export control warning and modification rqmt", + "HPND-export2-US": "HPND with US Government export control and 2 disclaimers", + "HPND-Fenneberg-Livingston": "Historical Permission Notice and Disclaimer - Fenneberg-Livingston variant", + "HPND-INRIA-IMAG": "Historical Permission Notice and Disclaimer - INRIA-IMAG variant", + "HPND-Intel": "Historical Permission Notice and Disclaimer - Intel variant", + "HPND-Kevlin-Henney": "Historical Permission Notice and Disclaimer - Kevlin Henney variant", + "HPND-Markus-Kuhn": "Historical Permission Notice and Disclaimer - Markus Kuhn variant", + "HPND-merchantability-variant": "Historical Permission Notice and Disclaimer - merchantability variant", + "HPND-MIT-disclaimer": "Historical Permission Notice and Disclaimer with MIT disclaimer", + "HPND-Netrek": "Historical Permission Notice and Disclaimer - Netrek variant", + "HPND-Pbmplus": "Historical Permission Notice and Disclaimer - Pbmplus variant", + "HPND-sell-MIT-disclaimer-xserver": "Historical Permission Notice and Disclaimer - sell xserver variant with MIT disclaimer", + "HPND-sell-regexpr": "Historical Permission Notice and Disclaimer - sell regexpr variant", + "HPND-sell-variant": "Historical Permission Notice and Disclaimer - sell variant", + "HPND-sell-variant-critical-systems": "HPND - sell variant with safety critical systems clause", + "HPND-sell-variant-MIT-disclaimer": "HPND sell variant with MIT disclaimer", + "HPND-sell-variant-MIT-disclaimer-rev": "HPND sell variant with MIT disclaimer - reverse", + "HPND-SMC": "Historical Permission Notice and Disclaimer - SMC variant", + "HPND-UC": "Historical Permission Notice and Disclaimer - University of California variant", + "HPND-UC-export-US": "Historical Permission Notice and Disclaimer - University of California, US export warning", + "HTMLTIDY": "HTML Tidy License", + "hyphen-bulgarian": "hyphen-bulgarian License", + "IBM-pibs": "IBM PowerPC Initialization and Boot Software", + "ICU": "ICU License", + "IEC-Code-Components-EULA": "IEC Code Components End-user licence agreement", + "IJG": "Independent JPEG Group License", + "IJG-short": "Independent JPEG Group License - short", + "ImageMagick": "ImageMagick License", + "iMatix": "iMatix Standard Function Library Agreement", + "Imlib2": "Imlib2 License", + "Info-ZIP": "Info-ZIP License", + "Informatica": "Informatica License", + "Inner-Net-2.0": "Inner Net License v2.0", + "InnoSetup": "Inno Setup License", + "Intel": "Intel Open Source License", + "Intel-ACPI": "Intel ACPI Software License Agreement", + "Interbase-1.0": "Interbase Public License v1.0", + "IPA": "IPA Font License", + "IPL-1.0": "IBM Public License v1.0", + "ISC": "ISC License", + "ISC-Veillard": "ISC Veillard variant", + "ISO-permission": "ISO permission notice", + "Jam": "Jam License", + "JasPer-2.0": "JasPer License", + "jove": "Jove License", + "JPL-image": "JPL Image Use Policy", + "JPNIC": "Japan Network Information Center License", + "JSON": "JSON License", + "Kastrup": "Kastrup License", + "Kazlib": "Kazlib License", + "Knuth-CTAN": "Knuth CTAN License", + "LAL-1.2": "Licence Art Libre 1.2", + "LAL-1.3": "Licence Art Libre 1.3", + "Latex2e": "Latex2e License", + "Latex2e-translated-notice": "Latex2e with translated notice permission", + "Leptonica": "Leptonica License", + "LGPL-2.0": "GNU Library General Public License v2 only", + "LGPL-2.0+": "GNU Library General Public License v2 or later", + "LGPL-2.0-only": "GNU Library General Public License v2 only", + "LGPL-2.0-or-later": "GNU Library General Public License v2 or later", + "LGPL-2.1": "GNU Lesser General Public License v2.1 only", + "LGPL-2.1+": "GNU Lesser General Public License v2.1 or later", + "LGPL-2.1-only": "GNU Lesser General Public License v2.1 only", + "LGPL-2.1-or-later": "GNU Lesser General Public License v2.1 or later", + "LGPL-3.0": "GNU Lesser General Public License v3.0 only", + "LGPL-3.0+": "GNU Lesser General Public License v3.0 or later", + "LGPL-3.0-only": "GNU Lesser General Public License v3.0 only", + "LGPL-3.0-or-later": "GNU Lesser General Public License v3.0 or later", + "LGPLLR": "Lesser General Public License For Linguistic Resources", + "Libpng": "libpng License", + "libpng-1.6.35": "PNG Reference Library License v1 (for libpng 0.5 through 1.6.35)", + "libpng-2.0": "PNG Reference Library version 2", + "libselinux-1.0": "libselinux public domain notice", + "libtiff": "libtiff License", + "libutil-David-Nugent": "libutil David Nugent License", + "LiLiQ-P-1.1": "Licence Libre du Qu\u00E9bec \u2013 Permissive version 1.1", + "LiLiQ-R-1.1": "Licence Libre du Qu\u00E9bec \u2013 R\u00E9ciprocit\u00E9 version 1.1", + "LiLiQ-Rplus-1.1": "Licence Libre du Qu\u00E9bec \u2013 R\u00E9ciprocit\u00E9 forte version 1.1", + "Linux-man-pages-1-para": "Linux man-pages - 1 paragraph", + "Linux-man-pages-copyleft": "Linux man-pages Copyleft", + "Linux-man-pages-copyleft-2-para": "Linux man-pages Copyleft - 2 paragraphs", + "Linux-man-pages-copyleft-var": "Linux man-pages Copyleft Variant", + "Linux-OpenIB": "Linux Kernel Variant of OpenIB.org license", + "LOOP": "Common Lisp LOOP License", + "LPD-document": "LPD Documentation License", + "LPL-1.0": "Lucent Public License Version 1.0", + "LPL-1.02": "Lucent Public License v1.02", + "LPPL-1.0": "LaTeX Project Public License v1.0", + "LPPL-1.1": "LaTeX Project Public License v1.1", + "LPPL-1.2": "LaTeX Project Public License v1.2", + "LPPL-1.3a": "LaTeX Project Public License v1.3a", + "LPPL-1.3c": "LaTeX Project Public License v1.3c", + "lsof": "lsof License", + "Lucida-Bitmap-Fonts": "Lucida Bitmap Fonts License", + "LZMA-SDK-9.11-to-9.20": "LZMA SDK License (versions 9.11 to 9.20)", + "LZMA-SDK-9.22": "LZMA SDK License (versions 9.22 and beyond)", + "Mackerras-3-Clause": "Mackerras 3-Clause License", + "Mackerras-3-Clause-acknowledgment": "Mackerras 3-Clause - acknowledgment variant", + "magaz": "magaz License", + "mailprio": "mailprio License", + "MakeIndex": "MakeIndex License", + "man2html": "man2html License", + "Martin-Birgmeier": "Martin Birgmeier License", + "McPhee-slideshow": "McPhee Slideshow License", + "metamail": "metamail License", + "Minpack": "Minpack License", + "MIPS": "MIPS License", + "MirOS": "The MirOS Licence", + "MIT": "MIT License", + "MIT-0": "MIT No Attribution", + "MIT-advertising": "Enlightenment License (e16)", + "MIT-Click": "MIT Click License", + "MIT-CMU": "CMU License", + "MIT-enna": "enna License", + "MIT-feh": "feh License", + "MIT-Festival": "MIT Festival Variant", + "MIT-Khronos-old": "MIT Khronos - old variant", + "MIT-Modern-Variant": "MIT License Modern Variant", + "MIT-open-group": "MIT Open Group variant", + "MIT-STK": "MIT-STK License", + "MIT-testregex": "MIT testregex Variant", + "MIT-Wu": "MIT Tom Wu Variant", + "MITNFA": "MIT +no-false-attribs license", + "MMIXware": "MMIXware License", + "MMPL-1.0.1": "Minecraft Mod Public License v1.0.1", + "Motosoto": "Motosoto License", + "MPEG-SSG": "MPEG Software Simulation", + "mpi-permissive": "mpi Permissive License", + "mpich2": "mpich2 License", + "MPL-1.0": "Mozilla Public License 1.0", + "MPL-1.1": "Mozilla Public License 1.1", + "MPL-2.0": "Mozilla Public License 2.0", + "MPL-2.0-no-copyleft-exception": "Mozilla Public License 2.0 (no copyleft exception)", + "mplus": "mplus Font License", + "MS-LPL": "Microsoft Limited Public License", + "MS-PL": "Microsoft Public License", + "MS-RL": "Microsoft Reciprocal License", + "MTLL": "Matrix Template Library License", + "MulanPSL-1.0": "Mulan Permissive Software License, Version 1", + "MulanPSL-2.0": "Mulan Permissive Software License, Version 2", + "Multics": "Multics License", + "Mup": "Mup License", + "MVT-1.1": "MVT License 1.1", + "NAIST-2003": "Nara Institute of Science and Technology License (2003)", + "NASA-1.3": "NASA Open Source Agreement 1.3", + "Naumen": "Naumen Public License", + "NBPL-1.0": "Net Boolean Public License v1", + "NCBI-PD": "NCBI Public Domain Notice", + "NCGL-UK-2.0": "Non-Commercial Government Licence", + "NCL": "NCL Source Code License", + "NCSA": "University of Illinois\/NCSA Open Source License", + "Net-SNMP": "Net-SNMP License", + "NetCDF": "NetCDF license", + "Newsletr": "Newsletr License", + "NGPL": "Nethack General Public License", + "ngrep": "ngrep License", + "NICTA-1.0": "NICTA Public Software License, Version 1.0", + "NIST-PD": "NIST Public Domain Notice", + "NIST-PD-fallback": "NIST Public Domain Notice with license fallback", + "NIST-PD-TNT": "NIST Public Domain Notice TNT variant", + "NIST-Software": "NIST Software License", + "NLOD-1.0": "Norwegian Licence for Open Government Data (NLOD) 1.0", + "NLOD-2.0": "Norwegian Licence for Open Government Data (NLOD) 2.0", + "NLPL": "No Limit Public License", + "Nokia": "Nokia Open Source License", + "NOSL": "Netizen Open Source License", + "Noweb": "Noweb License", + "NPL-1.0": "Netscape Public License v1.0", + "NPL-1.1": "Netscape Public License v1.1", + "NPOSL-3.0": "Non-Profit Open Software License 3.0", + "NRL": "NRL License", + "NTIA-PD": "NTIA Public Domain Notice", + "NTP": "NTP License", + "NTP-0": "NTP No Attribution", + "Nunit": "Nunit License", + "O-UDA-1.0": "Open Use of Data Agreement v1.0", + "OAR": "OAR License", + "OCCT-PL": "Open CASCADE Technology Public License", + "OCLC-2.0": "OCLC Research Public License 2.0", + "ODbL-1.0": "Open Data Commons Open Database License v1.0", + "ODC-By-1.0": "Open Data Commons Attribution License v1.0", + "OFFIS": "OFFIS License", + "OFL-1.0": "SIL Open Font License 1.0", + "OFL-1.0-no-RFN": "SIL Open Font License 1.0 with no Reserved Font Name", + "OFL-1.0-RFN": "SIL Open Font License 1.0 with Reserved Font Name", + "OFL-1.1": "SIL Open Font License 1.1", + "OFL-1.1-no-RFN": "SIL Open Font License 1.1 with no Reserved Font Name", + "OFL-1.1-RFN": "SIL Open Font License 1.1 with Reserved Font Name", + "OGC-1.0": "OGC Software License, Version 1.0", + "OGDL-Taiwan-1.0": "Taiwan Open Government Data License, version 1.0", + "OGL-Canada-2.0": "Open Government Licence - Canada", + "OGL-UK-1.0": "Open Government Licence v1.0", + "OGL-UK-2.0": "Open Government Licence v2.0", + "OGL-UK-3.0": "Open Government Licence v3.0", + "OGTSL": "Open Group Test Suite License", + "OLDAP-1.1": "Open LDAP Public License v1.1", + "OLDAP-1.2": "Open LDAP Public License v1.2", + "OLDAP-1.3": "Open LDAP Public License v1.3", + "OLDAP-1.4": "Open LDAP Public License v1.4", + "OLDAP-2.0": "Open LDAP Public License v2.0 (or possibly 2.0A and 2.0B)", + "OLDAP-2.0.1": "Open LDAP Public License v2.0.1", + "OLDAP-2.1": "Open LDAP Public License v2.1", + "OLDAP-2.2": "Open LDAP Public License v2.2", + "OLDAP-2.2.1": "Open LDAP Public License v2.2.1", + "OLDAP-2.2.2": "Open LDAP Public License 2.2.2", + "OLDAP-2.3": "Open LDAP Public License v2.3", + "OLDAP-2.4": "Open LDAP Public License v2.4", + "OLDAP-2.5": "Open LDAP Public License v2.5", + "OLDAP-2.6": "Open LDAP Public License v2.6", + "OLDAP-2.7": "Open LDAP Public License v2.7", + "OLDAP-2.8": "Open LDAP Public License v2.8", + "OLFL-1.3": "Open Logistics Foundation License Version 1.3", + "OML": "Open Market License", + "OpenMDW-1.0": "OpenMDW License Agreement v1.0", + "OpenPBS-2.3": "OpenPBS v2.3 Software License", + "OpenSSL": "OpenSSL License", + "OpenSSL-standalone": "OpenSSL License - standalone", + "OpenVision": "OpenVision License", + "OPL-1.0": "Open Public License v1.0", + "OPL-UK-3.0": "United Kingdom Open Parliament Licence v3.0", + "OPUBL-1.0": "Open Publication License v1.0", + "OSC-1.0": "OSC License 1.0", + "OSET-PL-2.1": "OSET Public License version 2.1", + "OSL-1.0": "Open Software License 1.0", + "OSL-1.1": "Open Software License 1.1", + "OSL-2.0": "Open Software License 2.0", + "OSL-2.1": "Open Software License 2.1", + "OSL-3.0": "Open Software License 3.0", + "OSSP": "OSSP License", + "PADL": "PADL License", + "ParaType-Free-Font-1.3": "ParaType Free Font Licensing Agreement v1.3", + "Parity-6.0.0": "The Parity Public License 6.0.0", + "Parity-7.0.0": "The Parity Public License 7.0.0", + "PDDL-1.0": "Open Data Commons Public Domain Dedication & License 1.0", + "PHP-3.0": "PHP License v3.0", + "PHP-3.01": "PHP License v3.01", + "Pixar": "Pixar License", + "pkgconf": "pkgconf License", + "Plexus": "Plexus Classworlds License", + "pnmstitch": "pnmstitch License", + "PolyForm-Noncommercial-1.0.0": "PolyForm Noncommercial License 1.0.0", + "PolyForm-Small-Business-1.0.0": "PolyForm Small Business License 1.0.0", + "PostgreSQL": "PostgreSQL License", + "PPL": "Peer Production License", + "PSF-2.0": "Python Software Foundation License 2.0", + "psfrag": "psfrag License", + "psutils": "psutils License", + "Python-2.0": "Python License 2.0", + "Python-2.0.1": "Python License 2.0.1", + "python-ldap": "Python ldap License", + "Qhull": "Qhull License", + "QPL-1.0": "Q Public License 1.0", + "QPL-1.0-INRIA-2004": "Q Public License 1.0 - INRIA 2004 variant", + "radvd": "radvd License", + "Rdisc": "Rdisc License", + "RHeCos-1.1": "Red Hat eCos Public License v1.1", + "RPL-1.1": "Reciprocal Public License 1.1", + "RPL-1.5": "Reciprocal Public License 1.5", + "RPSL-1.0": "RealNetworks Public Source License v1.0", + "RSA-MD": "RSA Message-Digest License", + "RSCPL": "Ricoh Source Code Public License", + "Ruby": "Ruby License", + "Ruby-pty": "Ruby pty extension license", + "SAX-PD": "Sax Public Domain Notice", + "SAX-PD-2.0": "Sax Public Domain Notice 2.0", + "Saxpath": "Saxpath License", + "SCEA": "SCEA Shared Source License", + "SchemeReport": "Scheme Language Report License", + "Sendmail": "Sendmail License", + "Sendmail-8.23": "Sendmail License 8.23", + "Sendmail-Open-Source-1.1": "Sendmail Open Source License v1.1", + "SGI-B-1.0": "SGI Free Software License B v1.0", + "SGI-B-1.1": "SGI Free Software License B v1.1", + "SGI-B-2.0": "SGI Free Software License B v2.0", + "SGI-OpenGL": "SGI OpenGL License", + "SGMLUG-PM": "SGMLUG Parser Materials License", + "SGP4": "SGP4 Permission Notice", + "SHL-0.5": "Solderpad Hardware License v0.5", + "SHL-0.51": "Solderpad Hardware License, Version 0.51", + "SimPL-2.0": "Simple Public License 2.0", + "SISSL": "Sun Industry Standards Source License v1.1", + "SISSL-1.2": "Sun Industry Standards Source License v1.2", + "SL": "SL License", + "Sleepycat": "Sleepycat License", + "SMAIL-GPL": "SMAIL General Public License", + "SMLNJ": "Standard ML of New Jersey License", + "SMPPL": "Secure Messaging Protocol Public License", + "SNIA": "SNIA Public License 1.1", + "snprintf": "snprintf License", + "SOFA": "SOFA Software License", + "softSurfer": "softSurfer License", + "Soundex": "Soundex License", + "Spencer-86": "Spencer License 86", + "Spencer-94": "Spencer License 94", + "Spencer-99": "Spencer License 99", + "SPL-1.0": "Sun Public License v1.0", + "ssh-keyscan": "ssh-keyscan License", + "SSH-OpenSSH": "SSH OpenSSH license", + "SSH-short": "SSH short notice", + "SSLeay-standalone": "SSLeay License - standalone", + "SSPL-1.0": "Server Side Public License, v 1", + "StandardML-NJ": "Standard ML of New Jersey License", + "SugarCRM-1.1.3": "SugarCRM Public License v1.1.3", + "SUL-1.0": "Sustainable Use License v1.0", + "Sun-PPP": "Sun PPP License", + "Sun-PPP-2000": "Sun PPP License (2000)", + "SunPro": "SunPro License", + "SWL": "Scheme Widget Library (SWL) Software License Agreement", + "swrule": "swrule License", + "Symlinks": "Symlinks License", + "TAPR-OHL-1.0": "TAPR Open Hardware License v1.0", + "TCL": "TCL\/TK License", + "TCP-wrappers": "TCP Wrappers License", + "TekHVC": "TekHVC License", + "TermReadKey": "TermReadKey License", + "TGPPL-1.0": "Transitive Grace Period Public Licence 1.0", + "ThirdEye": "ThirdEye License", + "threeparttable": "threeparttable License", + "TMate": "TMate Open Source License", + "TORQUE-1.1": "TORQUE v2.5+ Software License v1.1", + "TOSL": "Trusster Open Source License", + "TPDL": "Time::ParseDate License", + "TPL-1.0": "THOR Public License 1.0", + "TrustedQSL": "TrustedQSL License", + "TTWL": "Text-Tabs+Wrap License", + "TTYP0": "TTYP0 License", + "TU-Berlin-1.0": "Technische Universitaet Berlin License 1.0", + "TU-Berlin-2.0": "Technische Universitaet Berlin License 2.0", + "Ubuntu-font-1.0": "Ubuntu Font Licence v1.0", + "UCAR": "UCAR License", + "UCL-1.0": "Upstream Compatibility License v1.0", + "ulem": "ulem License", + "UMich-Merit": "Michigan\/Merit Networks License", + "Unicode-3.0": "Unicode License v3", + "Unicode-DFS-2015": "Unicode License Agreement - Data Files and Software (2015)", + "Unicode-DFS-2016": "Unicode License Agreement - Data Files and Software (2016)", + "Unicode-TOU": "Unicode Terms of Use", + "UnixCrypt": "UnixCrypt License", + "Unlicense": "The Unlicense", + "Unlicense-libtelnet": "Unlicense - libtelnet variant", + "Unlicense-libwhirlpool": "Unlicense - libwhirlpool variant", + "UnRAR": "UnRAR License", + "UPL-1.0": "Universal Permissive License v1.0", + "URT-RLE": "Utah Raster Toolkit Run Length Encoded License", + "Vim": "Vim License", + "Vixie-Cron": "Vixie Cron License", + "VOSTROM": "VOSTROM Public License for Open Source", + "VSL-1.0": "Vovida Software License v1.0", + "W3C": "W3C Software Notice and License (2002-12-31)", + "W3C-19980720": "W3C Software Notice and License (1998-07-20)", + "W3C-20150513": "W3C Software Notice and Document License (2015-05-13)", + "w3m": "w3m License", + "Watcom-1.0": "Sybase Open Watcom Public License 1.0", + "Widget-Workshop": "Widget Workshop License", + "WordNet": "WordNet License", + "Wsuipa": "Wsuipa License", + "WTFNMFPL": "Do What The F*ck You Want To But It's Not My Fault Public License", + "WTFPL": "Do What The F*ck You Want To Public License", + "wwl": "WWL License", + "wxWindows": "wxWindows Library License", + "X11": "X11 License", + "X11-distribute-modifications-variant": "X11 License Distribution Modification Variant", + "X11-no-permit-persons": "X11 no permit persons clause", + "X11-swapped": "X11 swapped final paragraphs", + "Xdebug-1.03": "Xdebug License v 1.03", + "Xerox": "Xerox License", + "Xfig": "Xfig License", + "XFree86-1.1": "XFree86 License 1.1", + "xinetd": "xinetd License", + "xkeyboard-config-Zinoviev": "xkeyboard-config Zinoviev License", + "xlock": "xlock License", + "Xnet": "X.Net License", + "xpp": "XPP License", + "XSkat": "XSkat License", + "xzoom": "xzoom License", + "YPL-1.0": "Yahoo! Public License v1.0", + "YPL-1.1": "Yahoo! Public License v1.1", + "Zed": "Zed License", + "Zeeff": "Zeeff License", + "Zend-2.0": "Zend License v2.0", + "Zimbra-1.3": "Zimbra Public License v1.3", + "Zimbra-1.4": "Zimbra Public License v1.4", + "Zlib": "zlib License", + "zlib-acknowledgement": "zlib\/libpng License with Acknowledgement", + "ZPL-1.1": "Zope Public License 1.1", + "ZPL-2.0": "Zope Public License 2.0", + "ZPL-2.1": "Zope Public License 2.1", + "389-exception": "389 Directory Server Exception", + "Asterisk-exception": "Asterisk exception", + "Asterisk-linking-protocols-exception": "Asterisk linking protocols exception", + "Autoconf-exception-2.0": "Autoconf exception 2.0", + "Autoconf-exception-3.0": "Autoconf exception 3.0", + "Autoconf-exception-generic": "Autoconf generic exception", + "Autoconf-exception-generic-3.0": "Autoconf generic exception for GPL-3.0", + "Autoconf-exception-macro": "Autoconf macro exception", + "Bison-exception-1.24": "Bison exception 1.24", + "Bison-exception-2.2": "Bison exception 2.2", + "Bootloader-exception": "Bootloader Distribution Exception", + "CGAL-linking-exception": "CGAL Linking Exception", + "Classpath-exception-2.0": "Classpath exception 2.0", + "Classpath-exception-2.0-short": "Classpath exception 2.0 - short", + "CLISP-exception-2.0": "CLISP exception 2.0", + "cryptsetup-OpenSSL-exception": "cryptsetup OpenSSL exception", + "Digia-Qt-LGPL-exception-1.1": "Digia Qt LGPL Exception version 1.1", + "DigiRule-FOSS-exception": "DigiRule FOSS License Exception", + "eCos-exception-2.0": "eCos exception 2.0", + "erlang-otp-linking-exception": "Erlang\/OTP Linking Exception", + "Fawkes-Runtime-exception": "Fawkes Runtime Exception", + "FLTK-exception": "FLTK exception", + "fmt-exception": "fmt exception", + "Font-exception-2.0": "Font exception 2.0", + "freertos-exception-2.0": "FreeRTOS Exception 2.0", + "GCC-exception-2.0": "GCC Runtime Library exception 2.0", + "GCC-exception-2.0-note": "GCC Runtime Library exception 2.0 - note variant", + "GCC-exception-3.1": "GCC Runtime Library exception 3.1", + "Gmsh-exception": "Gmsh exception", + "GNAT-exception": "GNAT exception", + "GNOME-examples-exception": "GNOME examples exception", + "GNU-compiler-exception": "GNU Compiler Exception", + "gnu-javamail-exception": "GNU JavaMail exception", + "Google-Patent-WebM": "Google Additional IP Rights Grant (Patents) - WebM", + "GPL-3.0-389-ds-base-exception": "GPL-3.0 389 DS Base Exception", + "GPL-3.0-interface-exception": "GPL-3.0 Interface Exception", + "GPL-3.0-linking-exception": "GPL-3.0 Linking Exception", + "GPL-3.0-linking-source-exception": "GPL-3.0 Linking Exception (with Corresponding Source)", + "GPL-CC-1.0": "GPL Cooperation Commitment 1.0", + "GStreamer-exception-2005": "GStreamer Exception (2005)", + "GStreamer-exception-2008": "GStreamer Exception (2008)", + "harbour-exception": "harbour exception", + "i2p-gpl-java-exception": "i2p GPL+Java Exception", + "Independent-modules-exception": "Independent Module Linking exception", + "KiCad-libraries-exception": "KiCad Libraries Exception", + "kvirc-openssl-exception": "kvirc OpenSSL Exception", + "LGPL-3.0-linking-exception": "LGPL-3.0 Linking Exception", + "libpri-OpenH323-exception": "libpri OpenH323 exception", + "Libtool-exception": "Libtool Exception", + "Linux-syscall-note": "Linux Syscall Note", + "LLGPL": "LLGPL Preamble", + "LLVM-exception": "LLVM Exception", + "LZMA-exception": "LZMA exception", + "mif-exception": "Macros and Inline Functions Exception", + "mxml-exception": "mxml Exception", + "Nokia-Qt-exception-1.1": "Nokia Qt LGPL exception 1.1", + "OCaml-LGPL-linking-exception": "OCaml LGPL Linking Exception", + "OCCT-exception-1.0": "Open CASCADE Exception 1.0", + "OpenJDK-assembly-exception-1.0": "OpenJDK Assembly exception 1.0", + "openvpn-openssl-exception": "OpenVPN OpenSSL Exception", + "PCRE2-exception": "PCRE2 exception", + "polyparse-exception": "Polyparse Exception", + "PS-or-PDF-font-exception-20170817": "PS\/PDF font exception (2017-08-17)", + "QPL-1.0-INRIA-2004-exception": "INRIA QPL 1.0 2004 variant exception", + "Qt-GPL-exception-1.0": "Qt GPL exception 1.0", + "Qt-LGPL-exception-1.1": "Qt LGPL exception 1.1", + "Qwt-exception-1.0": "Qwt exception 1.0", + "romic-exception": "Romic Exception", + "RRDtool-FLOSS-exception-2.0": "RRDtool FLOSS exception 2.0", + "rsync-linking-exception": "rsync Linking Exception", + "SANE-exception": "SANE Exception", + "SHL-2.0": "Solderpad Hardware License v2.0", + "SHL-2.1": "Solderpad Hardware License v2.1", + "Simple-Library-Usage-exception": "Simple Library Usage Exception", + "Spelling-Provider-LGPL-exception": "Spelling Provider LGPL exception", + "sqlitestudio-OpenSSL-exception": "sqlitestudio OpenSSL exception", + "stunnel-exception": "stunnel Exception", + "SWI-exception": "SWI exception", + "Swift-exception": "Swift Exception", + "Texinfo-exception": "Texinfo exception", + "u-boot-exception-2.0": "U-Boot exception 2.0", + "UBDL-exception": "Unmodified Binary Distribution exception", + "Universal-FOSS-exception-1.0": "Universal FOSS Exception, Version 1.0", + "vsftpd-openssl-exception": "vsftpd OpenSSL exception", + "WxWindows-exception-3.1": "WxWindows Library Exception 3.1", + "x11vnc-openssl-exception": "x11vnc OpenSSL Exception" + } +} diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index 6631cc4..c1322f1 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -1,6 +1,8 @@ from __future__ import annotations +import hashlib import sys +from importlib.resources import files from typing import TypeAlias import pytest @@ -13,6 +15,39 @@ _Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" +@pytest.mark.parametrize( + ("path", "release", "digest"), + [ + pytest.param( + "schema/bom-1.6.schema.json", + "1.6.2", + "18f57f7482593bad9f21b4feed09084640cbeff419d62ad5090c5ceccca5b37d", + id="bom-1.6", + ), + pytest.param( + "schema/jsf-0.82.schema.json", + "1.6.2", + "8bae002c25e723db7ee1f26afde680ae1a2b1a8f6b4b4b0fd65dc3becb090aae", + id="jsf-0.82", + ), + pytest.param( + "schema/spdx.schema.json", + "1.7.2", + "4b345e2329f209f34e960ae2a8e7cb46a166907e6a45e94978565925dc47b359", + id="spdx", + ), + pytest.param( + "LICENSE", "1.7.2", "6c29f22a4a7385285c6f579ec9f33c5e989f00739d6b257243a0b082ec9447ae", id="license" + ), + ], +) +def test_schema_files_are_the_upstream_release(path: str, release: str, digest: str) -> None: + # the SHA-256 of https://raw.githubusercontent.com/CycloneDX/specification//, so the shipped copy + # stays the released file byte for byte; a newer release replaces the file and its digest together + shipped = files("python_sbom._validate") / "schemas" / path.rpartition("/")[2] + assert hashlib.sha256(shipped.read_bytes()).hexdigest() == digest, f"not the file of the {release} release" + + def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch) -> None: # None in sys.modules makes the import fail, as when the extra is not installed monkeypatch.setitem(sys.modules, "jsonschema_rs", None) From 60381fcda734bc40e8fe9d7ca4b6e3158bc8fdc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:14:35 -0800 Subject: [PATCH 041/115] =?UTF-8?q?=E2=9C=A8=20feat:=20raise=20the=20defau?= =?UTF-8?q?lt=20component=20cap=20to=20100,000?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With jsonschema-rs and the one-pass uniqueness checks validation is linear, about a tenth of a second for 100,000 components, while virtualenv's wheel and zipapp SBOMs list 5,207 and 5,452 and failed the old 5,000 default. A test validates an SBOM of that size with the defaults. --- docs/usage.rst | 13 ++++---- src/python_sbom/_validate/_document.py | 6 ++-- tests/_cli/test_commands.py | 8 ++--- tests/_validate/test_document.py | 41 ++++++++++++++++++++------ 4 files changed, 45 insertions(+), 23 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 8c17a75..f36c86a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -159,13 +159,12 @@ and validates the SPDX rendering; an SBOM another tool wrote, such as auditwheel Each ``--allow-license`` adds an SPDX id that bundled wheels may use. With any given, a bundled wheel whose license the ids do not allow fails the check, and so does one that declares a license only by name. -Before the schema check, ``validate`` counts the components an SBOM lists, nested ones included, and fails an SBOM that -lists more than 5,000 without checking it further. The schema's ``uniqueItems`` rule compares every pair of items in an -array, so its time grows with the square of the count: 5,000 components in one array take about ten seconds to check, -10,000 about forty, and a document crafted that way could stall a CI job. Every file of the wheel, of each bundled wheel -and of each build tool counts, so a project with a large build environment can come close. Raise the cap for an SBOM -you expect to be larger: +Before anything else, ``validate`` counts the components an SBOM lists, nested ones included, and fails an SBOM that +lists more than 100,000 without checking it further. Every check takes time in proportion to the count, about a tenth +of a second for 100,000 components, so the cap bounds what an untrusted SBOM can ask of a CI job while staying far above +real SBOMs: every file of the wheel, of each bundled wheel and of each build tool counts, and virtualenv's wheel lists +about 5,200. Raise the cap for an SBOM you expect to be larger, or lower it to refuse more: .. code-block:: console - $ python-sbom validate dist/ --max-components 10000 + $ python-sbom validate dist/ --max-components 500000 diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index f121a4a..4623f7a 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -30,9 +30,9 @@ _SERIAL: Final[re.Pattern[str]] = re.compile(r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") # the archive reader stops at the same size _MAX_CHARACTERS: Final[int] = 16 * 1024 * 1024 -# every check takes time in proportion to the count, about half a second of CPU per ten thousand components, so the cap -# bounds the work an untrusted document can ask for -MAX_COMPONENTS: Final[int] = 5_000 +# every check takes time in proportion to the count, so the cap bounds the work an untrusted document can ask for, far +# above the few thousand components a wheel with its build tools' files lists +MAX_COMPONENTS: Final[int] = 100_000 # an exception's message can quote the hostile value that caused it _MAX_TEXT: Final[int] = 200 # the schema validator recurses once per level and crashes the process past a few thousand levels on a small thread diff --git a/tests/_cli/test_commands.py b/tests/_cli/test_commands.py index cc1a32a..31f45a7 100644 --- a/tests/_cli/test_commands.py +++ b/tests/_cli/test_commands.py @@ -138,17 +138,17 @@ def crowded_wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: return write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"", _SBOM: sbom}) -def test_validate_caps_components(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: +def test_validate_max_components_sets_the_cap(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: with pytest.raises(SystemExit): - main(["validate", str(crowded_wheel)]) + main(["validate", str(crowded_wheel), "--max-components", "5000"]) assert capsys.readouterr().err == ( f"SBOM invalid: {crowded_wheel.name}: {_SBOM}: lists more than 5000 components, nested ones included;" " pass a higher --max-components to check it\n" ) -def test_validate_max_components_raises_the_cap(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: - main(["validate", str(crowded_wheel), "--max-components", "5100"]) +def test_validate_default_cap_admits_thousands(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + main(["validate", str(crowded_wheel)]) assert capsys.readouterr().out == f"SBOM in {crowded_wheel.name} is valid\n" diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py index 5ff4b5b..66ea9a2 100644 --- a/tests/_validate/test_document.py +++ b/tests/_validate/test_document.py @@ -1,11 +1,14 @@ from __future__ import annotations +import dataclasses import json from typing import TYPE_CHECKING, Final, TypeAlias import pytest from python_sbom._formats._json import dumps +from python_sbom._model._inputs import InstalledDistribution, RecordRow +from python_sbom._model._metadata import parse_metadata from python_sbom._validate._document import is_document, validate_document if TYPE_CHECKING: @@ -15,6 +18,7 @@ from pytest_mock import MockerFixture from python_sbom import Component, License + from python_sbom._model._inputs import BuildFacts from python_sbom._model._types import Document # a JSON value, as json.loads hands it over @@ -57,26 +61,45 @@ def test_oversized_document_is_not_parsed() -> None: def test_component_cap_counts_nested_components() -> None: - assert validate_document(many_components(), None) == [ - "lists more than 5000 components, nested ones included; pass a higher --max-components to check it" + assert validate_document(many_components(1_001), None) == [ + "lists more than 100000 components, nested ones included; pass a higher --max-components to check it" ] -def many_components() -> str: - # 100 groups of 50 parts, 5,100 components in all, yet no array long enough to slow the schema check - groups = [ +def many_components(groups: int) -> str: + # groups of 100 parts each + components = [ { "type": "library", "name": f"group-{group}", - "components": [{"type": "library", "name": f"part-{part}"} for part in range(50)], + "components": [{"type": "library", "name": f"part-{part}"} for part in range(99)], } - for group in range(100) + for group in range(groups) ] - return json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "components": groups}) + return json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "components": components}) def test_component_cap_can_be_raised() -> None: - assert validate_document(many_components(), None, max_components=5_100) == [] + assert validate_document(many_components(1_001), None, max_components=100_100) == [] + + +def test_component_cap_can_be_lowered() -> None: + assert validate_document(many_components(2), None, max_components=199) == [ + "lists more than 199 components, nested ones included; pass a higher --max-components to check it" + ] + + +def test_sbom_the_size_of_virtualenvs_passes_by_default(describe: Describe, facts: BuildFacts) -> None: + # virtualenv's wheel lists 5,207 components and its zipapp 5,452, nearly all of them files of its build tools + rows = [ + RecordRow(path=f"tool/module_{index}.py", algorithm="SHA-256", digest="0" * 64, size="1") + for index in range(5_500) + ] + tool = InstalledDistribution( + name="tool", metadata=parse_metadata("Name: tool\nVersion: 1.0\n"), files=rows, depends_on=[] + ) + document = describe(facts=dataclasses.replace(facts, distributions=[tool])) + assert validate_document(dumps(document).decode(), None) == [] def test_missing_fields(describe: Describe) -> None: From 2ca7b51b5c34859437e967a2534fad638835bcbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:14:35 -0800 Subject: [PATCH 042/115] =?UTF-8?q?=E2=9A=A1=20perf:=20count=20components?= =?UTF-8?q?=20before=20loading=20the=20validator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Importing the validate extra takes about half a second, so the component count, bom-ref census and depth checks run first. Whether the extra is installed is still decided up front through find_spec, so a missing extra fails every document alike; a missing license-expression now raises MissingExtraError too instead of a bare ImportError. --- src/python_sbom/_validate/_document.py | 13 ++++++++++--- src/python_sbom/_validate/_schemas.py | 14 ++++---------- tests/_validate/test_schemas.py | 18 +++++++++++++----- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 4623f7a..6981bcb 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -4,6 +4,7 @@ import math import re from collections import Counter +from importlib.util import find_spec from typing import TYPE_CHECKING, Final, TypeAlias from python_sbom._errors import MissingExtraError @@ -33,6 +34,8 @@ # every check takes time in proportion to the count, so the cap bounds the work an untrusted document can ask for, far # above the few thousand components a wheel with its build tools' files lists MAX_COMPONENTS: Final[int] = 100_000 +# the modules of the validate extra +_EXTRA: Final[tuple[str, ...]] = ("jsonschema_rs", "license_expression", "spdx_tools") # an exception's message can quote the hostile value that caused it _MAX_TEXT: Final[int] = 200 # the schema validator recurses once per level and crashes the process past a few thousand levels on a small thread @@ -62,12 +65,16 @@ def validate_document( def _problems(raw: str, allowed_licenses: frozenset[str] | None, max_components: int) -> list[str]: # every document must conform to the CycloneDX 1.6 schema; one python-sbom wrote must also carry the urn:uuid serial # actions/attest needs, resolvable and unique bom-refs and a valid SPDX rendering, and meet the license policy - from ._licenses import license_problems # ruff: ignore[import-outside-top-level] # parses with the optional extra - from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # the optional extra loads on first use - + # a missing extra fails every document alike, while importing it, half a second, waits until the cheap census passes + if any(find_spec(name) is None for name in _EXTRA): + msg = "validation needs the validate extra: pip install 'python-sbom[validate]'" + raise MissingExtraError(msg) loaded: Final = json.loads(raw) if census := _census_problems(loaded, max_components): return census + from ._licenses import license_problems # ruff: ignore[import-outside-top-level] # the optional extra, see above + from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # the optional extra, see above + if error := schema_error(loaded): return [f"does not conform to the CycloneDX 1.6 schema: {error}"] # python-sbom names its file in a metadata property, so documents other tools wrote get the schema check alone; the diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index 5f433d3..ef713e9 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -4,16 +4,10 @@ from importlib.resources import files from typing import TYPE_CHECKING, Final, TypeAlias -from python_sbom._errors import MissingExtraError - -try: - from jsonschema_rs import Draft7Validator, Registry - from spdx_tools.spdx.parser.error import SPDXParsingError - from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser - from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document -except ImportError as error: - _MESSAGE: Final[str] = "validation needs the validate extra: pip install 'python-sbom[validate]'" - raise MissingExtraError(_MESSAGE) from error +from jsonschema_rs import Draft7Validator, Registry +from spdx_tools.spdx.parser.error import SPDXParsingError +from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser +from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document if TYPE_CHECKING: from importlib.resources.abc import Traversable diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index c1322f1..d46f825 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -48,12 +48,20 @@ def test_schema_files_are_the_upstream_release(path: str, release: str, digest: assert hashlib.sha256(shipped.read_bytes()).hexdigest() == digest, f"not the file of the {release} release" -def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch) -> None: - # None in sys.modules makes the import fail, as when the extra is not installed - monkeypatch.setitem(sys.modules, "jsonschema_rs", None) - monkeypatch.delitem(sys.modules, "python_sbom._validate._schemas", raising=False) +@pytest.mark.parametrize( + "module", + [ + pytest.param("jsonschema_rs", id="jsonschema-rs"), + pytest.param("license_expression", id="license-expression"), + pytest.param("spdx_tools", id="spdx-tools"), + ], +) +def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch, module: str) -> None: + # None in sys.modules makes the import fail, as when the extra is not installed; even a document the cheap checks + # would reject reports the missing extra + monkeypatch.setitem(sys.modules, module, None) with pytest.raises(MissingExtraError, match=r"pip install 'python-sbom\[validate\]'"): - validate_document("{}", None) + validate_document("not JSON", None) def test_schema_error_names_the_path() -> None: From e7dcf4880e6ecb3106b461107c7990f770872a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:22:02 -0800 Subject: [PATCH 043/115] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20the=20perf?= =?UTF-8?q?ormance=20changelog=20fragment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog.d/+fast-validation.performance.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog.d/+fast-validation.performance.rst diff --git a/docs/changelog.d/+fast-validation.performance.rst b/docs/changelog.d/+fast-validation.performance.rst new file mode 100644 index 0000000..269312a --- /dev/null +++ b/docs/changelog.d/+fast-validation.performance.rst @@ -0,0 +1,5 @@ +``python-sbom validate`` checks the CycloneDX schema with jsonschema-rs, against the official CycloneDX schema files +python-sbom now ships, so an SBOM validates in milliseconds instead of seconds, and its time grows in step with its +size; the default ``--max-components`` cap rises from 5,000 to 100,000 components to match. Building an SBOM reads each +build-environment distribution's ``RECORD`` and metadata once, and rewriting the wheel copies its other members without +recompressing them. From 0033c11e7dece2e548a9015839c2ab06c5e85f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:16:36 -0800 Subject: [PATCH 044/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20compress=20the=20?= =?UTF-8?q?SBOM=20archive=20--embed=20adds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The archive's SBOM went in stored, adding 5.7 MB to virtualenv's 11.7 MB zipapp. It is now compressed by the method most of the archive's members use, at the level their deflate option bits name, zlib's default for archives Python's zipfile wrote, so the bytes stay fixed for one input. --- docs/extend.rst | 11 ++++++---- src/python_sbom/_cli/_commands.py | 15 ++++++++++++-- tests/_cli/test_commands.py | 34 ++++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/docs/extend.rst b/docs/extend.rst index 92e217d..2f5eb13 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -116,7 +116,10 @@ hears of every bundled wheel, and its context reads the archive's files: wrote virtualenv.pyz.cdx.json wrote virtualenv.pyz.cdx.json into virtualenv.pyz -The document takes its options, source revision and toolchain the way the wheel's does, so both name the project -alike. Its build time comes from ``SOURCE_DATE_EPOCH``, or the current time without it; set ``SOURCE_DATE_EPOCH`` for -both builds to give them one time. ``--embed`` adds the document to the archive's top level, where ``python-sbom -validate virtualenv.pyz`` finds it. +The document takes its options, source revision and toolchain the way the wheel's does, so both name the project alike. +Its build time comes from ``SOURCE_DATE_EPOCH``, or the current time without it; set ``SOURCE_DATE_EPOCH`` for both +builds to give them one time. ``--embed`` adds the document to the archive's top level, where ``python-sbom validate +virtualenv.pyz`` finds it. It goes in compressed the way most of the archive's members are: by their compression method +and, for deflate, at the level their option bits name, which Python's zipfile leaves at zlib's default. The entry +carries a fixed time and Unix attributes, so adding it to one archive gives the same bytes on every system with the same +zlib. diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py index 39e716e..926abe8 100644 --- a/src/python_sbom/_cli/_commands.py +++ b/src/python_sbom/_cli/_commands.py @@ -3,9 +3,10 @@ import json import sys import zipfile +from collections import Counter from datetime import UTC, datetime from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final from python_sbom._collect._content import load_content_provider from python_sbom._collect._facts import collect_facts @@ -22,6 +23,10 @@ from ._options import Options +# the zlib level each setting of the deflate option bits in a member's flags stands for (APPNOTE 4.4.4: normal, maximum, +# fast, super fast), as Info-ZIP sets them; None is zlib's default +_DEFLATE_LEVELS: Final[dict[int, int | None]] = {0b000: None, 0b010: 9, 0b100: 2, 0b110: 1} + def add_embed(commands: _SubParsersAction[ArgumentParser]) -> None: parser = commands.add_parser( @@ -112,7 +117,13 @@ def _archive(options: Options) -> bool: # a fixed time and Unix attributes, so the archive's bytes are the same whichever system adds the entry entry = zipfile.ZipInfo(name, datetime.fromtimestamp(max(facts.epoch, 315532800), tz=UTC).timetuple()[:6]) entry.create_system, entry.external_attr = 3, 0o644 << 16 - handle.writestr(entry, content) + # compressed by the method most members use, at the level their deflate option bits name, since the zip + # format records no level itself; Python's zipfile leaves the bits at normal, zlib's default level + methods = Counter(info.compress_type for info in handle.infolist()) + method = next(iter(methods.most_common(1)), (zipfile.ZIP_STORED, 0))[0] + flags = next((info.flag_bits for info in handle.infolist() if info.compress_type == method), 0) + level = _DEFLATE_LEVELS[flags & 0b110] if method == zipfile.ZIP_DEFLATED else None + handle.writestr(entry, content, compress_type=method, compresslevel=level) sys.stdout.write(f"wrote {name} into {options.archive.name}\n") return True diff --git a/tests/_cli/test_commands.py b/tests/_cli/test_commands.py index 31f45a7..574df00 100644 --- a/tests/_cli/test_commands.py +++ b/tests/_cli/test_commands.py @@ -2,9 +2,11 @@ import json import runpy +import struct import subprocess import sys import zipfile +import zlib from pathlib import Path from typing import TYPE_CHECKING, Final @@ -206,13 +208,43 @@ def test_archive_embeds_sbom(project: Path, zipapp: Path, tmp_path: Path) -> Non "--embed", ]) entry = zipfile.ZipFile(zipapp).getinfo("app.pyz.cdx.json") - assert (entry.create_system, entry.external_attr, zipfile.ZipFile(zipapp).read(entry)) == ( + assert (entry.create_system, entry.external_attr, entry.compress_type, zipfile.ZipFile(zipapp).read(entry)) == ( 3, 0o644 << 16, + zipfile.ZIP_STORED, (tmp_path / "out.json").read_bytes(), ) +@pytest.mark.parametrize( + ("flags", "level"), + [ + pytest.param(0b000, zlib.Z_DEFAULT_COMPRESSION, id="normal"), + pytest.param(0b010, 9, id="maximum"), + pytest.param(0b100, 2, id="fast"), + pytest.param(0b110, 1, id="super-fast"), + ], +) +def test_archive_embeds_sbom_compressed_as_its_members(project: Path, tmp_path: Path, flags: int, level: int) -> None: + with zipfile.ZipFile(app := tmp_path / "app.pyz", "w", zipfile.ZIP_DEFLATED) as handle: + handle.writestr("__main__.py", "import app\n") + handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") + # zipfile clears the option bits it writes, so they go straight into the central directory, where readers take them + data = bytearray(app.read_bytes()) + for at in (index for index in range(len(data)) if data[index : index + 4] == b"PK\x01\x02"): + struct.pack_into(" None: arguments = ["archive", str(zipapp), "--root", str(project), "--build-env", str(project), "--embed"] main(arguments) From ad06520257ad746c63bc3d85a74e6ac8f4657383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:21:33 -0800 Subject: [PATCH 045/115] =?UTF-8?q?=E2=9A=A1=20perf:=20parse=20only=20the?= =?UTF-8?q?=20metadata=20headers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The body holds the long description, which nothing reads, and the email parser spent most of its time there. Parsing stops at the empty line that ends the headers; across 7,620 METADATA files the output is unchanged. --- src/python_sbom/_model/_metadata.py | 8 ++++++-- tests/_model/test_metadata.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/python_sbom/_model/_metadata.py b/src/python_sbom/_model/_metadata.py index 33edd61..11e44a7 100644 --- a/src/python_sbom/_model/_metadata.py +++ b/src/python_sbom/_model/_metadata.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Literal, TypeAlias @@ -27,13 +28,16 @@ "version", ] +# the empty line that ends the headers, as the email parser ends them; a line of whitespace continues a header +_BODY: Final[re.Pattern[str]] = re.compile(r"^\r?\n", re.MULTILINE) # unparsed keys a field by its header name, lowercased, which is the RawMetadata key with dashes except for these _HEADERS: Final[dict[str, str]] = {"classifiers": "classifier", "license_files": "license-file"} def parse_metadata(text: str) -> CoreMetadata: - # not validated: the SBOM describes a distribution whose metadata breaks the specification as well, as written - raw, unparsed = parse_email(text) + # not validated: the SBOM describes a distribution whose metadata breaks the specification as well, as written; the + # body holds the long description, which nothing reads, and parsing it took most of the time + raw, unparsed = parse_email(text[: match.start()] if (match := _BODY.search(text)) else text) return CoreMetadata(raw=raw, unparsed=unparsed) diff --git a/tests/_model/test_metadata.py b/tests/_model/test_metadata.py index e4e9fb1..91fbd74 100644 --- a/tests/_model/test_metadata.py +++ b/tests/_model/test_metadata.py @@ -101,3 +101,15 @@ def test_metadata_the_validating_parser_rejects(text: str, described: tuple[str, component = component_from_metadata(parse_metadata(text)) names = [entry["license"]["name"] for entry in component["licenses"] if "name" in entry["license"]] assert (component["name"], component["version"], names) == described + + +@pytest.mark.parametrize("newline", [pytest.param("\n", id="lf"), pytest.param("\r\n", id="crlf")]) +def test_body_is_not_read(newline: str) -> None: + # the headers end at the first empty line; a line of whitespace still continues the one before + text = newline.join(["Name: a", "Version: 1", "Summary: one", " more", "", "Author: in the body", ""]) + metadata = parse_metadata(text) + assert (metadata.first("summary"), metadata.first("author"), "description" in metadata.raw) == ( + f"one{newline} more", + None, + False, + ) From fa8c3fd03249b6d94609e37d472fcbab277fc86f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:21:33 -0800 Subject: [PATCH 046/115] =?UTF-8?q?=E2=9A=A1=20perf:=20verify=20bundled=20?= =?UTF-8?q?wheels=20in=20parallel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit installer's RECORD check inflates and hashes every member, which releases the GIL, so the bundled wheels verify in a thread pool. Results and the first error come in the order the wheels were listed, as before. --- src/python_sbom/_collect/_source.py | 19 ++++++++++++++----- src/python_sbom/_generate/_archive.py | 10 +++++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index d4a49e4..81253fb 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -2,6 +2,7 @@ import hashlib from collections import Counter +from concurrent.futures import ThreadPoolExecutor from io import BytesIO from itertools import starmap from pathlib import PurePath @@ -42,14 +43,14 @@ def read_source_tree(request: SbomRequest) -> SourceTree: ), None, ), - bundled=[ - read_bundled_wheel((root / relative).read_bytes(), relative) - for relative in sorted({ + bundled=read_bundled_wheels( + root, + sorted({ wheel.relative_to(root).as_posix() for pattern in request.settings.bundled_wheels for wheel in root.glob(pattern) - }) - ], + }), + ), # str, so a Windows drive in an absolute path survives the join with the native root build_scripts=[ BuildScript(path=script, sha256=hashlib.sha256((root / str(script)).read_bytes()).hexdigest()) @@ -58,6 +59,13 @@ def read_source_tree(request: SbomRequest) -> SourceTree: ) +def read_bundled_wheels(root: Path, paths: list[str]) -> list[BundledWheel]: + # inflating and hashing release the GIL, so the wheels verify side by side; results and the first error come in the + # order given + with ThreadPoolExecutor() as pool: + return list(pool.map(lambda path: read_bundled_wheel((root / path).read_bytes(), path), paths)) + + def read_bundled_wheel(data: bytes, path: str) -> BundledWheel: where: Final[str] = f"bundled wheel {path}" with open_archive(BytesIO(data), where) as archive: @@ -171,6 +179,7 @@ def wheel_request(wheel: Path, root: Path, settings: Settings, build_scripts: tu __all__ = [ "read_bundled_wheel", + "read_bundled_wheels", "read_source_tree", "wheel_request", ] diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index 6b28e6c..466c939 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -10,7 +10,7 @@ from python_sbom._collect._content import load_content_provider, provider_script from python_sbom._collect._settings import settings_from_pyproject -from python_sbom._collect._source import read_bundled_wheel +from python_sbom._collect._source import read_bundled_wheels from python_sbom._errors import WheelError from python_sbom._model._components import ( bundled_wheel_component, @@ -33,7 +33,7 @@ from python_sbom._model._content import ContentProvider from python_sbom._model._identity import ProjectIdentity - from python_sbom._model._inputs import BuildFacts, Settings + from python_sbom._model._inputs import BuildFacts, BundledWheel, Settings from python_sbom._model._metadata import CoreMetadata from python_sbom._model._types import Component, Document, Metadata @@ -135,8 +135,10 @@ def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, else: loose.append(name) project = project_identity(metadata[""], settings) + wheels: Final[list[str]] = [name for prefix in sorted(metadata) for name in owned[prefix] if name.endswith(".whl")] reader = _Reader( unpacked=unpacked, + bundled=dict(zip(wheels, read_bundled_wheels(unpacked, wheels), strict=True)), project=project, settings=settings, content=content, @@ -163,6 +165,8 @@ def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, @dataclass(frozen=True, slots=True, kw_only=True) class _Reader: unpacked: Path + # read ahead, in the order the components list them + bundled: dict[str, BundledWheel] project: ProjectIdentity settings: Settings content: ContentProvider @@ -190,7 +194,7 @@ def distribution(self, prefix: str, metadata: CoreMetadata, names: list[str]) -> def wheel(self, name: str) -> Component: info = self.content.bundled_wheel(self.context, PurePosixPath(name)) return bundled_wheel_component( - read_bundled_wheel((self.unpacked / name).read_bytes(), name), + self.bundled[name], self.project.property_prefix, info.properties, info.supplier, From 0dcafe6c315446e88546c201a8db9dd0ea5da6b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:21:33 -0800 Subject: [PATCH 047/115] =?UTF-8?q?=E2=9A=A1=20perf:=20copy=20the=20wheel?= =?UTF-8?q?=20through=20the=20process?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit shutil.copyfile uses copy_file_range, which on btrfs and XFS first flushes the wheel hatchling just wrote, about 5 ms per build; a plain read and write copy skips that. --- src/python_sbom/_wheel/_archive.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index 791b4d8..fb43858 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -51,7 +51,9 @@ def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: descriptor, temporary = tempfile.mkstemp(dir=wheel.parent, prefix=".python-sbom-", suffix=".tmp") os.close(descriptor) try: # ruff: ignore[too-many-statements-in-try-clause] # whichever step fails, the temporary file goes - shutil.copyfile(wheel, temporary) + # copied through the process: copy_file_range makes some file systems first flush the wheel just written + with wheel.open("rb") as source, Path(temporary).open("wb") as target: + shutil.copyfileobj(source, target) # the other members keep their bytes as stored, still compressed, so only the SBOM and RECORD get compressed # here; both go last, RECORD after the SBOM with zipremove.ZipFile(temporary, "a") as archive: From e5213adb7334eb0130c99dba4aa4f33e1b559e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:21:33 -0800 Subject: [PATCH 048/115] =?UTF-8?q?=E2=9A=A1=20perf:=20match=20uv=20cache?= =?UTF-8?q?=20files=20without=20a=20regex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/python_sbom/_collect/_facts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index 4d9b316..b4dd1c0 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -127,7 +127,7 @@ def _describes_distribution(path: str) -> bool: return not ( path.startswith("../") or (directory.endswith(".dist-info") and name in {"INSTALLER", "REQUESTED", "direct_url.json"}) - or re.fullmatch(r"uv_.*\.json", name) + or (name.startswith("uv_") and name.endswith(".json")) ) From f014f1803379b6e0ea215a88a58e4d8a98bcf543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:21:33 -0800 Subject: [PATCH 049/115] =?UTF-8?q?=E2=9A=A1=20perf:=20load=20the=20hook's?= =?UTF-8?q?=20wheel=20code=20only=20for=20wheels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reading the source revision moves out of _collect._facts into its own module, so an sdist build loads that alone; the code a wheel needs loads in finalize. The revision is read once per process, so hatch build reads git once for the sdist and the wheel it makes from the same checkout. --- src/python_sbom/_collect/_facts.py | 75 +---------- src/python_sbom/_collect/_revision.py | 86 ++++++++++++ src/python_sbom/_generate/_embed.py | 5 +- src/python_sbom/hatch/_hook.py | 27 ++-- tests/_collect/test_facts.py | 174 +----------------------- tests/_collect/test_revision.py | 184 ++++++++++++++++++++++++++ 6 files changed, 297 insertions(+), 254 deletions(-) create mode 100644 src/python_sbom/_collect/_revision.py create mode 100644 tests/_collect/test_revision.py diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index b4dd1c0..970012d 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -3,15 +3,12 @@ import json import os import platform -import re -import shutil -import subprocess # ruff: ignore[suspicious-subprocess-import] # runs git to read the source revision import sys import time import warnings from importlib.metadata import distributions from itertools import starmap -from typing import TYPE_CHECKING, Final, Protocol +from typing import TYPE_CHECKING, Final from installer.records import InvalidRecordEntry, RecordEntry, parse_record_file from installer.utils import parse_metadata_file @@ -20,7 +17,7 @@ from packaging.utils import canonicalize_name from python_sbom._errors import SettingsError -from python_sbom._model._inputs import BuildFacts, InstalledDistribution, Interpreter, RecordRow, Revision +from python_sbom._model._inputs import BuildFacts, InstalledDistribution, Interpreter, RecordRow from python_sbom._model._metadata import parse_metadata from python_sbom._wheel._record import decode_digest @@ -31,13 +28,7 @@ from python_sbom._model._metadata import CoreMetadata -#: the file an sdist carries its source revision in, as ``node: ``, for the wheel built from it -REVISION_FILE: Final[str] = ".python-sbom-revision" -_COMMIT: Final[re.Pattern[str]] = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}") - - -class RevisionReader(Protocol): - def __call__(self, root: Path, /) -> Revision | None: ... + from ._revision import RevisionReader def collect_facts( @@ -148,66 +139,6 @@ def _depends_on(name: str, distribution: Distribution, metadata: CoreMetadata, i return sorted(names) -def default_revision(root: Path, /) -> Revision | None: - # the variable exists for builds from an sdist, so it wins over whatever checkout the sdist was unpacked into - if commit := os.environ.get("PYTHON_SBOM_COMMIT"): - return commit_revision(commit, "PYTHON_SBOM_COMMIT") - if (found := git_revision(root)) is not None: - return found - # the sdist build wrote it from the revision it resolved, so it is at least as exact as an archival file - if (recorded := root / REVISION_FILE).is_file(): - return commit_revision(recorded.read_text(encoding="utf-8").strip().removeprefix("node: "), REVISION_FILE) - archival = root / ".git_archival.txt" - if archival.is_file() and ( - match := re.search(rf"^node: ({_COMMIT.pattern})$", archival.read_text(encoding="utf-8"), re.MULTILINE) - ): - return Revision(commit=match[1], dirty=False) - return None - - -def commit_revision(commit: str, origin: str) -> Revision: - # the commit lands in the source link, so anything but a full hash could point the link elsewhere - if not _COMMIT.fullmatch(commit): - msg = f"{origin} must be a full lowercase hex commit hash of 40 or 64 characters, got {commit!r}" - raise SettingsError(msg) - return Revision(commit=commit, dirty=False) - - -def git_revision(root: Path, /) -> Revision | None: - resolved = root.resolve() - if (git := shutil.which("git")) is None or not any( - (directory / ".git").exists() for directory in (resolved, *resolved.parents) - ): - return None - try: - # a project unpacked into another checkout, such as an sdist under its build directory, is not its source - _git(git, root, "ls-files", "--error-unmatch", "--", "pyproject.toml") - commit = _git(git, root, "rev-parse", "--verify", "--quiet", "HEAD") - # untracked files count, since the build packages every file .gitignore lets through - changes = _git(git, root, "status", "--porcelain", "--", ".") - except subprocess.CalledProcessError as error: - if error.returncode != 1: # 1: the project is untracked, or the checkout has no commit yet - warnings.warn(f"git cannot read the source revision in {root}: {error.stderr.strip()}", stacklevel=2) - return None - return Revision(commit=commit, dirty=bool(changes)) - - -def _git(git: str, root: Path, *arguments: str) -> str: - # git first runs in whichever repository encloses the project, whose fsmonitor hook would run a command from its - # config; no optional locks, so status never rewrites the index under another git process - return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] # git from PATH with fixed arguments - [git, "-c", "core.fsmonitor=false", "--no-optional-locks", "-C", str(root), *arguments], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - - __all__ = [ - "REVISION_FILE", - "RevisionReader", "collect_facts", - "commit_revision", - "default_revision", - "git_revision", ] diff --git a/src/python_sbom/_collect/_revision.py b/src/python_sbom/_collect/_revision.py new file mode 100644 index 0000000..24a7933 --- /dev/null +++ b/src/python_sbom/_collect/_revision.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import os +import re +import shutil +import subprocess # ruff: ignore[suspicious-subprocess-import] # runs git to read the source revision +import warnings +from typing import TYPE_CHECKING, Final, Protocol + +from python_sbom._errors import SettingsError +from python_sbom._model._inputs import Revision + +if TYPE_CHECKING: + from pathlib import Path + +#: the file an sdist carries its source revision in, as ``node: ``, for the wheel built from it +REVISION_FILE: Final[str] = ".python-sbom-revision" +_COMMIT: Final[re.Pattern[str]] = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}") + + +class RevisionReader(Protocol): + def __call__(self, root: Path, /) -> Revision | None: ... + + +def default_revision(root: Path, /) -> Revision | None: + # the variable exists for builds from an sdist, so it wins over whatever checkout the sdist was unpacked into + if commit := os.environ.get("PYTHON_SBOM_COMMIT"): + return commit_revision(commit, "PYTHON_SBOM_COMMIT") + if (found := git_revision(root)) is not None: + return found + # the sdist build wrote it from the revision it resolved, so it is at least as exact as an archival file + if (recorded := root / REVISION_FILE).is_file(): + return commit_revision(recorded.read_text(encoding="utf-8").strip().removeprefix("node: "), REVISION_FILE) + archival = root / ".git_archival.txt" + if archival.is_file() and ( + match := re.search(rf"^node: ({_COMMIT.pattern})$", archival.read_text(encoding="utf-8"), re.MULTILINE) + ): + return Revision(commit=match[1], dirty=False) + return None + + +def commit_revision(commit: str, origin: str) -> Revision: + # the commit lands in the source link, so anything but a full hash could point the link elsewhere + if not _COMMIT.fullmatch(commit): + msg = f"{origin} must be a full lowercase hex commit hash of 40 or 64 characters, got {commit!r}" + raise SettingsError(msg) + return Revision(commit=commit, dirty=False) + + +def git_revision(root: Path, /) -> Revision | None: + resolved = root.resolve() + if (git := shutil.which("git")) is None or not any( + (directory / ".git").exists() for directory in (resolved, *resolved.parents) + ): + return None + try: + # a project unpacked into another checkout, such as an sdist under its build directory, is not its source + _git(git, root, "ls-files", "--error-unmatch", "--", "pyproject.toml") + commit = _git(git, root, "rev-parse", "--verify", "--quiet", "HEAD") + # untracked files count, since the build packages every file .gitignore lets through + changes = _git(git, root, "status", "--porcelain", "--", ".") + except subprocess.CalledProcessError as error: + if error.returncode != 1: # 1: the project is untracked, or the checkout has no commit yet + warnings.warn(f"git cannot read the source revision in {root}: {error.stderr.strip()}", stacklevel=2) + return None + return Revision(commit=commit, dirty=bool(changes)) + + +def _git(git: str, root: Path, *arguments: str) -> str: + # git first runs in whichever repository encloses the project, whose fsmonitor hook would run a command from its + # config; no optional locks, so status never rewrites the index under another git process + return subprocess.run( # ruff: ignore[subprocess-without-shell-equals-true] # git from PATH with fixed arguments + [git, "-c", "core.fsmonitor=false", "--no-optional-locks", "-C", str(root), *arguments], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +__all__ = [ + "REVISION_FILE", + "RevisionReader", + "commit_revision", + "default_revision", + "git_revision", +] diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py index 66cf3b0..40a14ca 100644 --- a/src/python_sbom/_generate/_embed.py +++ b/src/python_sbom/_generate/_embed.py @@ -4,7 +4,8 @@ from typing import TYPE_CHECKING from python_sbom._collect._content import load_content_provider, provider_script -from python_sbom._collect._facts import collect_facts, commit_revision, default_revision +from python_sbom._collect._facts import collect_facts +from python_sbom._collect._revision import commit_revision, default_revision from python_sbom._collect._settings import settings_from_pyproject from python_sbom._collect._source import read_source_tree, wheel_request from python_sbom._model._content import ProviderContext @@ -17,7 +18,7 @@ from collections.abc import Sequence from pathlib import Path - from python_sbom._collect._facts import RevisionReader + from python_sbom._collect._revision import RevisionReader from python_sbom._model._content import ContentProvider from python_sbom._model._inputs import BuildFacts, SbomRequest diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index 365b83a..98d9701 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -1,7 +1,7 @@ from __future__ import annotations import tempfile -from functools import cached_property +from functools import cache, cached_property from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Final @@ -9,16 +9,13 @@ from hatchling.builders.utils import get_reproducible_timestamp from hatchling.utils.constants import DEFAULT_BUILD_SCRIPT -from python_sbom._collect._content import load_content_provider, provider_script -from python_sbom._collect._facts import REVISION_FILE, collect_facts, default_revision +from python_sbom._collect._revision import REVISION_FILE, default_revision from python_sbom._collect._settings import settings_from_table -from python_sbom._collect._source import wheel_request from python_sbom._errors import SettingsError -from python_sbom._generate._embed import embed_into if TYPE_CHECKING: from python_sbom._model._content import ContentProvider - from python_sbom._model._inputs import Settings + from python_sbom._model._inputs import Revision, Settings # options of every hatchling hook table, which hatchling itself reads _HATCHLING_OPTIONS: Final[frozenset[str]] = frozenset({ @@ -58,7 +55,7 @@ def initialize( """ # a dirty checkout has no commit that reproduces the sdist, so a wheel built from it names none - if self.target_name != "sdist" or (revision := default_revision(Path(self.root))) is None or revision.dirty: + if self.target_name != "sdist" or (revision := _revision(Path(self.root))) is None or revision.dirty: return # outside the project, so the build leaves nothing behind in it; bytes, so the sdist holds the same LF line on # every OS @@ -82,6 +79,12 @@ def finalize( """ if self.target_name != "wheel" or version == "editable": return + # what a wheel needs loads only when one is built, so an sdist build stays light + from python_sbom._collect._content import provider_script # ruff: ignore[import-outside-top-level] # see above + from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # see above + from python_sbom._collect._source import wheel_request # ruff: ignore[import-outside-top-level] # see above + from python_sbom._generate._embed import embed_into # ruff: ignore[import-outside-top-level] # see above + root = Path(self.root) content = self.content_provider() # a custom hook script shapes the document, as a provider defined in the project does, so the SBOM lists both @@ -92,7 +95,7 @@ def finalize( # the zip entries hatchling wrote facts = collect_facts( root, - default_revision, + _revision, default_epoch=get_reproducible_timestamp() if self.build_config.reproducible else None, ) wheel = Path(artifact_path) @@ -108,6 +111,8 @@ def content_provider(self) -> ContentProvider: :returns: the ``content-provider`` option's provider, or one that adds nothing """ + from python_sbom._collect._content import load_content_provider # ruff: ignore[import-outside-top-level] # see finalize + return load_content_provider(self._settings.content_provider, Path(self.root)) @cached_property @@ -123,6 +128,12 @@ def _settings(self) -> Settings: return settings_from_table(self.metadata.config.get("tool", {}).get("python-sbom", {}), "[tool.python-sbom]") +@cache +def _revision(root: Path) -> Revision | None: + # read once per process: hatch build makes the sdist and then the wheel in one, of the same checkout + return default_revision(root) + + __all__ = [ "SbomBuildHook", ] diff --git a/tests/_collect/test_facts.py b/tests/_collect/test_facts.py index a4b29e2..80d01d5 100644 --- a/tests/_collect/test_facts.py +++ b/tests/_collect/test_facts.py @@ -9,14 +9,14 @@ import pytest -from python_sbom._collect._facts import REVISION_FILE, collect_facts, default_revision, git_revision +from python_sbom._collect._facts import collect_facts from python_sbom._errors import SettingsError from python_sbom._model._inputs import Interpreter, RecordRow, Revision if TYPE_CHECKING: from pathlib import Path - from conftest import Git, Install + from conftest import Install from python_sbom._model._inputs import BuildFacts @@ -235,173 +235,3 @@ def test_invalid_requirement_left_out(tmp_path: Path, install: Install) -> None: with pytest.warns(UserWarning, match=r"left out the requirement 'pytz \(>dev\)' of old: "): by_name = {distribution.name: distribution for distribution in facts_from(tmp_path).distributions} assert by_name["old"].depends_on == ["pytz"] - - -@pytest.mark.parametrize("commit", [pytest.param("a" * 40, id="sha1"), pytest.param("b" * 64, id="sha256")]) -def test_commit_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, commit: str) -> None: - monkeypatch.setenv("PYTHON_SBOM_COMMIT", commit) - assert default_revision(tmp_path) == Revision(commit=commit, dirty=False) - - -@pytest.mark.parametrize( - "commit", - [ - pytest.param("c0ffee", id="abbreviated"), - pytest.param("A" * 40, id="uppercase"), - pytest.param("a" * 41, id="between-lengths"), - pytest.param(f"{'a' * 40}/../../evil", id="path"), - ], -) -def test_commit_from_environment_must_be_full_hash( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, commit: str -) -> None: - monkeypatch.setenv("PYTHON_SBOM_COMMIT", commit) - with pytest.raises(SettingsError, match="PYTHON_SBOM_COMMIT must be a full lowercase hex commit hash"): - default_revision(tmp_path) - - -def test_commit_from_environment_wins_over_git(checkout: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("PYTHON_SBOM_COMMIT", "a" * 40) - assert default_revision(checkout) == Revision(commit="a" * 40, dirty=False) - - -@pytest.fixture -def checkout(tmp_path: Path, git: Git) -> Path: - (root := tmp_path / "checkout").joinpath("pyproject.toml").write_text("[project]\n", encoding="utf-8") - (root / ".gitignore").write_text("build/\n", encoding="utf-8") - git("add", ".") - git("commit", "--quiet", "--message", "first") - return root - - -def test_revision_file(tmp_path: Path) -> None: - (tmp_path / REVISION_FILE).write_text(f"node: {'d' * 40}\n", encoding="utf-8") - assert default_revision(tmp_path) == Revision(commit="d" * 40, dirty=False) - - -def test_invalid_revision_file(tmp_path: Path) -> None: - (tmp_path / REVISION_FILE).write_text("node: d\n", encoding="utf-8") - with pytest.raises(SettingsError, match=r"\.python-sbom-revision must be a full lowercase hex commit hash"): - default_revision(tmp_path) - - -def test_git_revision(checkout: Path, git: Git) -> None: - assert default_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=False) - - -def test_git_revision_dirty(checkout: Path, git: Git) -> None: - (checkout / "pyproject.toml").write_text("[project]\nname = 'changed'\n", encoding="utf-8") - assert git_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=True) - - -def test_untracked_file_marks_revision_dirty(checkout: Path) -> None: - # the build packages it, so the commit alone no longer describes the source - (checkout / "backdoor.py").write_text("import os\n", encoding="utf-8") - assert (revision := git_revision(checkout)) is not None - assert revision.dirty - - -def test_ignored_file_leaves_revision_clean(checkout: Path) -> None: - (checkout / "build").mkdir() - (checkout / "build" / "output.txt").write_text("new\n", encoding="utf-8") - assert (revision := git_revision(checkout)) is not None - assert not revision.dirty - - -@pytest.fixture -def monorepo(tmp_path: Path, git: Git) -> Path: - for project in ("one", "two"): - (root := tmp_path / "checkout" / project).mkdir() - (root / "pyproject.toml").write_text("[project]\n", encoding="utf-8") - git("add", ".") - git("commit", "--quiet", "--message", "first") - return tmp_path / "checkout" - - -def test_git_revision_of_project_in_subdirectory(monorepo: Path, git: Git) -> None: - assert git_revision(monorepo / "one") == Revision(commit=git("rev-parse", "HEAD"), dirty=False) - - -def test_sibling_changes_leave_revision_clean(monorepo: Path) -> None: - (monorepo / "two" / "pyproject.toml").write_text("[project]\nname = 'changed'\n", encoding="utf-8") - (monorepo / "two" / "new.py").write_text("", encoding="utf-8") - assert (revision := git_revision(monorepo / "one")) is not None - assert not revision.dirty - - -def test_project_untracked_by_enclosing_checkout_uses_archival(checkout: Path) -> None: - # an sdist unpacked under another project's build directory is not that project's commit - (project := checkout / "build" / "project").mkdir(parents=True) - (project / "pyproject.toml").write_text("[project]\n", encoding="utf-8") - (project / ".git_archival.txt").write_text(f"node: {'e' * 40}\n", encoding="utf-8") - assert default_revision(project) == Revision(commit="e" * 40, dirty=False) - - -@pytest.mark.parametrize( - "project", - [pytest.param(".", id="own-checkout"), pytest.param("build/project", id="enclosing-checkout")], -) -def test_git_runs_no_fsmonitor_command(tmp_path: Path, checkout: Path, git: Git, project: str) -> None: - git("config", "core.fsmonitor", f"touch {(marker := tmp_path / 'ran').as_posix()}; false") - (root := checkout / project).mkdir(parents=True, exist_ok=True) - (root / "pyproject.toml").touch() - git_revision(root) - assert not marker.exists() - - -def test_git_revision_before_first_commit(tmp_path: Path, git: Git) -> None: - (tmp_path / "checkout" / "pyproject.toml").write_text("[project]\n", encoding="utf-8") - git("add", "pyproject.toml") - assert git_revision(tmp_path / "checkout") is None - - -def test_git_revision_outside_checkout(tmp_path: Path) -> None: - assert git_revision(tmp_path) is None - - -def test_git_revision_without_git(checkout: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("PATH", "") - assert git_revision(checkout) is None - - -@pytest.mark.parametrize( - "git_file", - [ - # a worktree copied without the main checkout, as into a container - pytest.param("gitdir: {root}/missing\n", id="missing-gitdir"), - # the marker uv leaves in an unpacked sdist it builds from - pytest.param("", id="empty"), - ], -) -def test_git_failure_warns_and_falls_back(tmp_path: Path, git_file: str) -> None: - (tmp_path / ".git").write_text(git_file.format(root=tmp_path.as_posix()), encoding="utf-8") - (tmp_path / "pyproject.toml").write_text("[project]\n", encoding="utf-8") - (tmp_path / ".git_archival.txt").write_text(f"node: {'f' * 40}\n", encoding="utf-8") - with pytest.warns(UserWarning, match="git cannot read the source revision in .*: fatal: "): - assert default_revision(tmp_path) == Revision(commit="f" * 40, dirty=False) - - -def test_git_status_failure_leaves_revision_unknown(checkout: Path, git: Git) -> None: - # a tree object lost from the store, so git knows HEAD but cannot compare the checkout with it - tree = git("rev-parse", "HEAD^{tree}") - (stored := checkout / ".git" / "objects" / tree[:2] / tree[2:]).chmod(0o644) - stored.unlink() - with pytest.warns(UserWarning, match="git cannot read the source revision in .*: error: bad tree object HEAD"): - assert git_revision(checkout) is None - - -@pytest.mark.parametrize( - ("archival", "expected"), - [ - pytest.param(f"node: {'a' * 40}\n", Revision(commit="a" * 40, dirty=False), id="expanded"), - pytest.param("node: $Format:%H$\n", None, id="unexpanded"), - pytest.param(f"node: {'a' * 41}\n", None, id="not-a-commit"), - ], -) -def test_git_archival(tmp_path: Path, archival: str, expected: Revision | None) -> None: - (tmp_path / ".git_archival.txt").write_text(archival, encoding="utf-8") - assert default_revision(tmp_path) == expected - - -def test_no_revision_source(tmp_path: Path) -> None: - assert default_revision(tmp_path) is None diff --git a/tests/_collect/test_revision.py b/tests/_collect/test_revision.py new file mode 100644 index 0000000..c996509 --- /dev/null +++ b/tests/_collect/test_revision.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from python_sbom._collect._revision import REVISION_FILE, default_revision, git_revision +from python_sbom._errors import SettingsError +from python_sbom._model._inputs import Revision + +if TYPE_CHECKING: + from pathlib import Path + + from conftest import Git + + +@pytest.mark.parametrize("commit", [pytest.param("a" * 40, id="sha1"), pytest.param("b" * 64, id="sha256")]) +def test_commit_from_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, commit: str) -> None: + monkeypatch.setenv("PYTHON_SBOM_COMMIT", commit) + assert default_revision(tmp_path) == Revision(commit=commit, dirty=False) + + +@pytest.mark.parametrize( + "commit", + [ + pytest.param("c0ffee", id="abbreviated"), + pytest.param("A" * 40, id="uppercase"), + pytest.param("a" * 41, id="between-lengths"), + pytest.param(f"{'a' * 40}/../../evil", id="path"), + ], +) +def test_commit_from_environment_must_be_full_hash( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, commit: str +) -> None: + monkeypatch.setenv("PYTHON_SBOM_COMMIT", commit) + with pytest.raises(SettingsError, match="PYTHON_SBOM_COMMIT must be a full lowercase hex commit hash"): + default_revision(tmp_path) + + +def test_commit_from_environment_wins_over_git(checkout: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PYTHON_SBOM_COMMIT", "a" * 40) + assert default_revision(checkout) == Revision(commit="a" * 40, dirty=False) + + +@pytest.fixture +def checkout(tmp_path: Path, git: Git) -> Path: + (root := tmp_path / "checkout").joinpath("pyproject.toml").write_text("[project]\n", encoding="utf-8") + (root / ".gitignore").write_text("build/\n", encoding="utf-8") + git("add", ".") + git("commit", "--quiet", "--message", "first") + return root + + +def test_revision_file(tmp_path: Path) -> None: + (tmp_path / REVISION_FILE).write_text(f"node: {'d' * 40}\n", encoding="utf-8") + assert default_revision(tmp_path) == Revision(commit="d" * 40, dirty=False) + + +def test_invalid_revision_file(tmp_path: Path) -> None: + (tmp_path / REVISION_FILE).write_text("node: d\n", encoding="utf-8") + with pytest.raises(SettingsError, match=r"\.python-sbom-revision must be a full lowercase hex commit hash"): + default_revision(tmp_path) + + +def test_git_revision(checkout: Path, git: Git) -> None: + assert default_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=False) + + +def test_git_revision_dirty(checkout: Path, git: Git) -> None: + (checkout / "pyproject.toml").write_text("[project]\nname = 'changed'\n", encoding="utf-8") + assert git_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=True) + + +def test_untracked_file_marks_revision_dirty(checkout: Path) -> None: + # the build packages it, so the commit alone no longer describes the source + (checkout / "backdoor.py").write_text("import os\n", encoding="utf-8") + assert (revision := git_revision(checkout)) is not None + assert revision.dirty + + +def test_ignored_file_leaves_revision_clean(checkout: Path) -> None: + (checkout / "build").mkdir() + (checkout / "build" / "output.txt").write_text("new\n", encoding="utf-8") + assert (revision := git_revision(checkout)) is not None + assert not revision.dirty + + +@pytest.fixture +def monorepo(tmp_path: Path, git: Git) -> Path: + for project in ("one", "two"): + (root := tmp_path / "checkout" / project).mkdir() + (root / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + git("add", ".") + git("commit", "--quiet", "--message", "first") + return tmp_path / "checkout" + + +def test_git_revision_of_project_in_subdirectory(monorepo: Path, git: Git) -> None: + assert git_revision(monorepo / "one") == Revision(commit=git("rev-parse", "HEAD"), dirty=False) + + +def test_sibling_changes_leave_revision_clean(monorepo: Path) -> None: + (monorepo / "two" / "pyproject.toml").write_text("[project]\nname = 'changed'\n", encoding="utf-8") + (monorepo / "two" / "new.py").write_text("", encoding="utf-8") + assert (revision := git_revision(monorepo / "one")) is not None + assert not revision.dirty + + +def test_project_untracked_by_enclosing_checkout_uses_archival(checkout: Path) -> None: + # an sdist unpacked under another project's build directory is not that project's commit + (project := checkout / "build" / "project").mkdir(parents=True) + (project / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + (project / ".git_archival.txt").write_text(f"node: {'e' * 40}\n", encoding="utf-8") + assert default_revision(project) == Revision(commit="e" * 40, dirty=False) + + +@pytest.mark.parametrize( + "project", + [pytest.param(".", id="own-checkout"), pytest.param("build/project", id="enclosing-checkout")], +) +def test_git_runs_no_fsmonitor_command(tmp_path: Path, checkout: Path, git: Git, project: str) -> None: + git("config", "core.fsmonitor", f"touch {(marker := tmp_path / 'ran').as_posix()}; false") + (root := checkout / project).mkdir(parents=True, exist_ok=True) + (root / "pyproject.toml").touch() + git_revision(root) + assert not marker.exists() + + +def test_git_revision_before_first_commit(tmp_path: Path, git: Git) -> None: + (tmp_path / "checkout" / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + git("add", "pyproject.toml") + assert git_revision(tmp_path / "checkout") is None + + +def test_git_revision_outside_checkout(tmp_path: Path) -> None: + assert git_revision(tmp_path) is None + + +def test_git_revision_without_git(checkout: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PATH", "") + assert git_revision(checkout) is None + + +@pytest.mark.parametrize( + "git_file", + [ + # a worktree copied without the main checkout, as into a container + pytest.param("gitdir: {root}/missing\n", id="missing-gitdir"), + # the marker uv leaves in an unpacked sdist it builds from + pytest.param("", id="empty"), + ], +) +def test_git_failure_warns_and_falls_back(tmp_path: Path, git_file: str) -> None: + (tmp_path / ".git").write_text(git_file.format(root=tmp_path.as_posix()), encoding="utf-8") + (tmp_path / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + (tmp_path / ".git_archival.txt").write_text(f"node: {'f' * 40}\n", encoding="utf-8") + with pytest.warns(UserWarning, match="git cannot read the source revision in .*: fatal: "): + assert default_revision(tmp_path) == Revision(commit="f" * 40, dirty=False) + + +def test_git_status_failure_leaves_revision_unknown(checkout: Path, git: Git) -> None: + # a tree object lost from the store, so git knows HEAD but cannot compare the checkout with it + tree = git("rev-parse", "HEAD^{tree}") + (stored := checkout / ".git" / "objects" / tree[:2] / tree[2:]).chmod(0o644) + stored.unlink() + with pytest.warns(UserWarning, match="git cannot read the source revision in .*: error: bad tree object HEAD"): + assert git_revision(checkout) is None + + +@pytest.mark.parametrize( + ("archival", "expected"), + [ + pytest.param(f"node: {'a' * 40}\n", Revision(commit="a" * 40, dirty=False), id="expanded"), + pytest.param("node: $Format:%H$\n", None, id="unexpanded"), + pytest.param(f"node: {'a' * 41}\n", None, id="not-a-commit"), + ], +) +def test_git_archival(tmp_path: Path, archival: str, expected: Revision | None) -> None: + (tmp_path / ".git_archival.txt").write_text(archival, encoding="utf-8") + assert default_revision(tmp_path) == expected + + +def test_no_revision_source(tmp_path: Path) -> None: + assert default_revision(tmp_path) is None From 286033ce2391b7dd6c08c94f8541dab51a0e7419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:21:33 -0800 Subject: [PATCH 050/115] =?UTF-8?q?=E2=9A=A1=20perf:=20load=20each=20comma?= =?UTF-8?q?nd's=20modules=20when=20it=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/python_sbom/_cli/_commands.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py index 926abe8..a3aec7d 100644 --- a/src/python_sbom/_cli/_commands.py +++ b/src/python_sbom/_cli/_commands.py @@ -8,15 +8,8 @@ from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Final -from python_sbom._collect._content import load_content_provider -from python_sbom._collect._facts import collect_facts from python_sbom._errors import SbomError -from python_sbom._formats._json import dumps -from python_sbom._formats._spdx import to_spdx -from python_sbom._generate._archive import describe_archive -from python_sbom._generate._embed import embed_sbom, revision_reader -from python_sbom._validate._artifact import validate as validate_artifact -from python_sbom._validate._document import MAX_COMPONENTS, is_document +from python_sbom._validate._document import MAX_COMPONENTS if TYPE_CHECKING: from argparse import ArgumentParser, _SubParsersAction @@ -41,6 +34,9 @@ def add_embed(commands: _SubParsersAction[ArgumentParser]) -> None: def _embed(options: Options) -> bool: + from python_sbom._collect._content import load_content_provider # ruff: ignore[import-outside-top-level] # each command loads only what it runs + from python_sbom._generate._embed import embed_sbom # ruff: ignore[import-outside-top-level] # each command loads only what it runs + root = Path.cwd() if options.root is None else options.root content = None if options.content_provider is None else load_content_provider(options.content_provider, root) for wheel in options.wheels: @@ -97,6 +93,12 @@ def add_archive(commands: _SubParsersAction[ArgumentParser]) -> None: def _archive(options: Options) -> bool: + from python_sbom._collect._content import load_content_provider # ruff: ignore[import-outside-top-level] # each command loads only what it runs + from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # each command loads only what it runs + from python_sbom._formats._json import dumps # ruff: ignore[import-outside-top-level] # each command loads only what it runs + from python_sbom._generate._archive import describe_archive # ruff: ignore[import-outside-top-level] # each command loads only what it runs + from python_sbom._generate._embed import revision_reader # ruff: ignore[import-outside-top-level] # each command loads only what it runs + root = Path.cwd() if options.root is None else options.root facts = collect_facts(root, revision_reader(options.commit), path=options.build_env) document = describe_archive( @@ -136,6 +138,9 @@ def add_spdx(commands: _SubParsersAction[ArgumentParser]) -> None: def _spdx(options: Options) -> bool: + from python_sbom._formats._spdx import to_spdx # ruff: ignore[import-outside-top-level] # each command loads only what it runs + from python_sbom._validate._document import is_document # ruff: ignore[import-outside-top-level] # each command loads only what it runs + if not is_document(loaded := json.loads(options.source.read_text(encoding="utf-8"))): msg = f"{options.source} is not a CycloneDX document python-sbom can render" raise SbomError(msg) @@ -180,6 +185,8 @@ def add_validate(commands: _SubParsersAction[ArgumentParser]) -> None: def _validate(options: Options) -> bool: + from python_sbom._validate._artifact import validate as validate_artifact # ruff: ignore[import-outside-top-level] # each command loads only what it runs + allowed = None if options.allowed_licenses is None else frozenset(options.allowed_licenses) if not ( artifacts := sorted( From 7114bfef90c511c356698c397f41b6a1409c8daa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:32:20 -0800 Subject: [PATCH 051/115] =?UTF-8?q?=F0=9F=93=9D=20docs:=20wrap=20comments?= =?UTF-8?q?=20and=20prose=20at=20120?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 4 +-- bench/__main__.py | 8 ++--- bench/_fixtures.py | 4 +-- docs/explanation.rst | 50 +++++++++++++------------- docs/extend.rst | 24 ++++++------- docs/index.rst | 8 ++--- docs/py_api.rst | 4 +-- docs/usage.rst | 42 +++++++++++----------- src/python_sbom/_collect/_content.py | 6 ++-- src/python_sbom/_collect/_facts.py | 4 +-- src/python_sbom/_errors.py | 4 +-- src/python_sbom/_formats/_spdx.py | 4 +-- src/python_sbom/_model/_inputs.py | 4 +-- src/python_sbom/_model/_licenses.py | 4 +-- src/python_sbom/_model/_metadata.py | 4 +-- src/python_sbom/_model/_types.py | 6 ++-- src/python_sbom/_validate/_document.py | 4 +-- src/python_sbom/_wheel/_archive.py | 4 +-- 18 files changed, 94 insertions(+), 94 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bb0ca1b..594a58f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ -# prek runs lower priorities first and equal ones concurrently, so hooks sharing a priority never write the same -# file; pre-commit ignores the key and runs hooks in file order. +# prek runs lower priorities first and equal ones concurrently, so hooks sharing a priority never write the same file; +# pre-commit ignores the key and runs hooks in file order. ci: autoupdate_schedule: quarterly default_language_version: diff --git a/bench/__main__.py b/bench/__main__.py index cd0c0ad..78ef04f 100644 --- a/bench/__main__.py +++ b/bench/__main__.py @@ -2,14 +2,14 @@ CLI entry point: ``python -m bench ``, run from the project root or with the root on ``PYTHONPATH``. ``prepare`` builds the fixtures into the work directory, the only command that needs the network. ``run`` times each -case in its own pyperf process tree, one JSON file per case under ``//``; options after ``--`` reach -pyperf and override the case's defaults (e.g. ``--rigorous``). +case in its own pyperf process tree, one JSON file per case under ``//``; options after ``--`` reach pyperf +and override the case's defaults (e.g. ``--rigorous``). ``profile`` runs each case once under cProfile after an untimed warm-up, and under py-spy with ``--py-spy``. ``report`` renders pyperf JSON as a Markdown table, ``compare`` two result directories side by side, and ``list`` prints the cases. -The timed process imports whichever python-sbom the running interpreter has installed, so comparing two revisions -means running the same harness from two environments. +The timed process imports whichever python-sbom the running interpreter has installed, so comparing two revisions means +running the same harness from two environments. """ from __future__ import annotations diff --git a/bench/_fixtures.py b/bench/_fixtures.py index e39d761..e809c29 100644 --- a/bench/_fixtures.py +++ b/bench/_fixtures.py @@ -245,8 +245,8 @@ def _output(command: list[str]) -> str: def _snapshot(source: Path, target: Path) -> Path: - # a clone of the checkout plus its uncommitted changes, so builds write their version files there and never into - # the checkout itself; --shared borrows its objects, so even a detached HEAD resolves + # a clone of the checkout plus its uncommitted changes, so builds write their version files there and never into the + # checkout itself; --shared borrows its objects, so even a detached HEAD resolves shutil.rmtree(target, ignore_errors=True) _run(["git", "clone", "--quiet", "--shared", "--no-checkout", str(source), str(target)]) head = _output(["git", "-C", str(source), "rev-parse", "HEAD"]).strip() diff --git a/docs/explanation.rst b/docs/explanation.rst index f9796c1..bef6406 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -49,8 +49,8 @@ points at what a package imports, and a dashed arrow at what it imports only for class hook,cli,collect,api edge; class generate,validate,formats,wheel,model core; -The model imports nothing else of the package but its errors, and only the hook imports hatchling, so the provider -API and the command line work without it. Every package raises subclasses of :class:`~python_sbom.SbomError`, which the +The model imports nothing else of the package but its errors, and only the hook imports hatchling, so the provider API +and the command line work without it. Every package raises subclasses of :class:`~python_sbom.SbomError`, which the command line reports without a traceback. The one extension point is the :class:`~python_sbom.ContentProvider`, which every entry point loads from the @@ -60,19 +60,19 @@ The one extension point is the :class:`~python_sbom.ContentProvider`, which ever What the SBOM records --------------------- -The document's root component is the project, described from the wheel's ``METADATA``: name, version, summary, -license expression, maintainers, project URLs, classifiers and keywords. Links to the PyPI release, the wheel's -attestation and, with a repository, the license file follow. A build from a git checkout adds the commit and a link to -that exact revision. +The document's root component is the project, described from the wheel's ``METADATA``: name, version, summary, license +expression, maintainers, project URLs, classifiers and keywords. Links to the PyPI release, the wheel's attestation and, +with a repository, the license file follow. A build from a git checkout adds the commit and a link to that exact +revision. The components below the root are the wheels the project bundles and the runtime dependencies it declares. A bundled wheel carries its SHA-256, its license, a component per file from its ``RECORD``, and the packages it vendors. A declared dependency carries no version, because the installer picks it at install time; the document marks the root's dependencies incomplete for that reason. An extra's dependencies are left out. -The formulation records how the wheel was built: the interpreter's version and implementation, every distribution of -the build environment with its dependencies, the files and hashes of each pure-Python one, and ``SOURCE_DATE_EPOCH`` -when it set the build time. +The formulation records how the wheel was built: the interpreter's version and implementation, every distribution of the +build environment with its dependencies, the files and hashes of each pure-Python one, and ``SOURCE_DATE_EPOCH`` when it +set the build time. Property names start with the project's normalized name, such as ``virtualenv:bundled-wheel``, or with ``python:`` for values copied from core metadata. @@ -80,35 +80,35 @@ values copied from core metadata. Reproducibility --------------- -The document holds nothing about the build machine. Its timestamp comes from ``SOURCE_DATE_EPOCH``. When that is -unset, a reproducible hatchling build uses hatchling's fixed 2020-02-02, the time it stamps on the wheel's entries, and -any other build uses the current time. Its serial number derives from the content, so identical documents share one -and any difference gets a new one. +The document holds nothing about the build machine. Its timestamp comes from ``SOURCE_DATE_EPOCH``. When that is unset, +a reproducible hatchling build uses hatchling's fixed 2020-02-02, the time it stamps on the wheel's entries, and any +other build uses the current time. Its serial number derives from the content, so identical documents share one and any +difference gets a new one. The build tools are listed down to their files, so a rebuild can check it installed the same bytes. Only what is the -same on every machine goes in: platform wheels appear without their files, because those differ per operating system -and architecture, and editable installs without theirs, because they point at a checkout. Installer metadata, -console-script launchers and uv's cache files are left out too, since they name the frontend, the environment's path or -a time rather than the distribution. +same on every machine goes in: platform wheels appear without their files, because those differ per operating system and +architecture, and editable installs without theirs, because they point at a checkout. Installer metadata, console-script +launchers and uv's cache files are left out too, since they name the frontend, the environment's path or a time rather +than the distribution. Hashing each tool's ``RECORD`` instead would be smaller, but that file lists the installer metadata, so two frontends installing the same tools would give two hashes. The cost is size: python-sbom's own isolated build lists about 600 files, half of them from setuptools, which setuptools-scm pulls in, for an SBOM of about 400 KB. -So a rebuild reproduces the wheel byte for byte on any operating system when these match the original build: the -source tree, as a checkout of the same commit; ``SOURCE_DATE_EPOCH``; the Python version; and the versions of the -build backend and its dependencies. The SBOM records the last three, so it says what to install before rebuilding. +So a rebuild reproduces the wheel byte for byte on any operating system when these match the original build: the source +tree, as a checkout of the same commit; ``SOURCE_DATE_EPOCH``; the Python version; and the versions of the build backend +and its dependencies. The SBOM records the last three, so it says what to install before rebuilding. Limitations ----------- -- The document describes what the project declares and bundles, not the full tree of runtime dependencies, which - depends on the environment the wheel is installed into. +- The document describes what the project declares and bundles, not the full tree of runtime dependencies, which depends + on the environment the wheel is installed into. - The links to the PyPI release and attestation assume the project publishes to PyPI with attestations. - A wheel built from an sdist of a dirty checkout names no commit, since no commit reproduces it. -- ``python-sbom embed`` and ``archive`` record the packages of the environment ``--build-env`` names, by default the - one they run in, and always the interpreter they run on. Unless that is the environment the artifact was built in, - the build tool record does not match the real build. +- ``python-sbom embed`` and ``archive`` record the packages of the environment ``--build-env`` names, by default the one + they run in, and always the interpreter they run on. Unless that is the environment the artifact was built in, the + build tool record does not match the real build. - The SPDX rendering leaves out per-file hashes and the build record, which SPDX 2.3 cannot hold. - A tool that renames the wheel after the build, such as ``auditwheel repair``, leaves the attestation link pointing at the old file name until ``python-sbom embed`` runs on the renamed wheel. diff --git a/docs/extend.rst b/docs/extend.rst index 2f5eb13..df272e9 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -16,10 +16,10 @@ it adds. - :meth:`~python_sbom.ContentProvider.bundled_wheel` returns the properties and the supplier of one bundled wheel. The properties follow ``:bundled-wheel`` and come before those from the wheel's metadata. A wheel's supplier comes only from here, never from the wheel's own metadata. -- :meth:`~python_sbom.ContentProvider.contributions` returns extra top-level components, the dependencies between - them, root properties and build tools, such as the C libraries a native wheel links. They join the document before - the root's dependencies and the serial number derive from it. The build fails when a ``bom-ref`` repeats, a - dependency names one that no component has, or a root property uses the project's own prefix. +- :meth:`~python_sbom.ContentProvider.contributions` returns extra top-level components, the dependencies between them, + root properties and build tools, such as the C libraries a native wheel links. They join the document before the + root's dependencies and the serial number derive from it. The build fails when a ``bom-ref`` repeats, a dependency + names one that no component has, or a root property uses the project's own prefix. Name the provider in the ``content-provider`` option, as ``module:attribute`` importable from the project root, and the hook, ``python-sbom embed`` and ``python-sbom archive`` all load it. When the provider's class lives in the project, the @@ -74,9 +74,9 @@ where the requirement's marker installs it, and falls back to hatchling's plain return SbomBuildHook The ``get_build_hook`` function tells hatchling which class to load. The SBOM lists the script among the build tools, -hashed. The provider reads the table next to each wheel through the context, by name, without importing the package, -and names the project's supplier as the wheels' supplier, since virtualenv bundles only PyPA's own projects. The test -suite builds a project with virtualenv's layout with the same provider, handed over from a hook subclass: +hashed. The provider reads the table next to each wheel through the context, by name, without importing the package, and +names the project's supplier as the wheels' supplier, since virtualenv bundles only PyPA's own projects. The test suite +builds a project with virtualenv's layout with the same provider, handed over from a hook subclass: .. literalinclude:: ../tests/_seeder/hatch_build.py :language: python @@ -104,11 +104,11 @@ A bundled wheel is named by its path inside the artifact, so the wheel's SBOM an Describe another archive ------------------------ -virtualenv also publishes a zipapp, ``virtualenv.pyz``, that holds virtualenv's own ``.dist-info`` at its top level, -the wheels it seeds, and its dependencies unpacked under ``__virtualenv__/``. ``python-sbom archive`` describes such an -archive the way the hook describes a wheel: the archive's own files, the project's distribution, each wheel it -bundles as a ``.whl`` member, and each one unpacked into a directory with a ``.dist-info``. The content provider -hears of every bundled wheel, and its context reads the archive's files: +virtualenv also publishes a zipapp, ``virtualenv.pyz``, that holds virtualenv's own ``.dist-info`` at its top level, the +wheels it seeds, and its dependencies unpacked under ``__virtualenv__/``. ``python-sbom archive`` describes such an +archive the way the hook describes a wheel: the archive's own files, the project's distribution, each wheel it bundles +as a ``.whl`` member, and each one unpacked into a directory with a ``.dist-info``. The content provider hears of every +bundled wheel, and its context reads the archive's files: .. code-block:: console diff --git a/docs/index.rst b/docs/index.rst index 1041067..70083ca 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,8 +1,8 @@ python-sbom =========== -SBOMs for Python distributions. python-sbom writes a `CycloneDX `_ 1.6 SBOM into every wheel -you build, at ``-.dist-info/sboms/.cdx.json``. That is where `PEP 770 +SBOMs for Python distributions. python-sbom writes a `CycloneDX `_ 1.6 SBOM into every wheel you +build, at ``-.dist-info/sboms/.cdx.json``. That is where `PEP 770 `_ tells installers to keep SBOMs, so they land in the environment next to the installed package, where scanners look for them. @@ -17,8 +17,8 @@ and packages of the build environment. A third party can use that record to rebu Quick start ----------- -For a project built with `hatchling `_, add the hook to the build requirements and -enable it in ``pyproject.toml``: +For a project built with `hatchling `_, add the hook to the build requirements and enable +it in ``pyproject.toml``: .. code-block:: toml diff --git a/docs/py_api.rst b/docs/py_api.rst index bfdea4b..a9a4bea 100644 --- a/docs/py_api.rst +++ b/docs/py_api.rst @@ -2,8 +2,8 @@ Python API ========== The Python API is what a project needs to add its own knowledge to the SBOM: the hatchling build hook and the content -provider it calls, with the types those take and return. Everything else, from embedding to validation, runs through -the :doc:`cli`. The core is importable from ``python_sbom`` without hatchling; the hook lives in ``python_sbom.hatch``. +provider it calls, with the types those take and return. Everything else, from embedding to validation, runs through the +:doc:`cli`. The core is importable from ``python_sbom`` without hatchling; the hook lives in ``python_sbom.hatch``. .. py:data:: python_sbom.__version__ :type: str diff --git a/docs/usage.rst b/docs/usage.rst index f36c86a..d2c7b38 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -5,22 +5,22 @@ With hatchling -------------- Add the hook to ``build-system.requires`` and enable it with an empty table, as the quick start on the :doc:`index` -shows. The ``hatch`` extra brings hatchling; the command line needs none. The hook writes the SBOM into wheels only: -an editable wheel carries none, and an sdist carries the source revision instead, so a wheel built from it still names -its commit. +shows. The ``hatch`` extra brings hatchling; the command line needs none. The hook writes the SBOM into wheels only: an +editable wheel carries none, and an sdist carries the source revision instead, so a wheel built from it still names its +commit. The hook writes the SBOM once hatchling has written the wheel, so the document describes the final file name and metadata, whatever dependencies or tag other hooks changed. Hatchling runs the hooks in the order of their tables; see -its `build hook documentation `_. The order does not matter -for this hook. +its `build hook documentation `_. The order does not matter for +this hook. Options ~~~~~~~ The options live in a ``[tool.python-sbom]`` table, the one place the hook, ``python-sbom embed`` and ``python-sbom -archive`` all read, so every SBOM of a project names it alike. Each option is optional; an unknown key fails the -build, so a typo cannot pass silently. The hook's own table holds only hatchling's options, and a python-sbom option -there fails the build with a pointer to ``[tool.python-sbom]``. +archive`` all read, so every SBOM of a project names it alike. Each option is optional; an unknown key fails the build, +so a typo cannot pass silently. The hook's own table holds only hatchling's options, and a python-sbom option there +fails the build with a pointer to ``[tool.python-sbom]``. ``supplier`` The organization that supplies the project: a table with a ``name`` string and an optional ``url`` string array. It @@ -85,8 +85,8 @@ The SBOM names the commit the wheel was built from. It takes the first source th the sdist, as ``python -m build`` and ``uv build`` do by default, keeps the commit; #. the ``node`` line of an expanded ``.git_archival.txt``, for a ``git archive``. -A commit must be a full hash of 40 or 64 hex characters. A checkout with changes, including untracked files that are -not ignored, gets a ``:vcs-dirty`` property, and the link to the revision says so. When git cannot read the +A commit must be a full hash of 40 or 64 hex characters. A checkout with changes, including untracked files that are not +ignored, gets a ``:vcs-dirty`` property, and the link to the revision says so. When git cannot read the checkout, the build warns and moves on to the next source. git runs with ``core.fsmonitor`` off, so a repository's configuration runs no command. @@ -94,8 +94,8 @@ With another build backend -------------------------- A backend other than hatchling, such as `meson-python `_, cannot run the hook. -``python-sbom embed`` writes the same document into a wheel that backend built, adds the ``RECORD`` row, and keeps -every other entry of the archive as it was. It reads the options from ``[tool.python-sbom]``: +``python-sbom embed`` writes the same document into a wheel that backend built, adds the ``RECORD`` row, and keeps every +other entry of the archive as it was. It reads the options from ``[tool.python-sbom]``: .. code-block:: toml @@ -124,9 +124,9 @@ there: $ python -m build --wheel --no-isolation $ python-sbom embed dist/*.whl -``auditwheel repair`` renames a Linux wheel to its ``manylinux`` tag, so the attestation link in an SBOM written -before the repair names the old file. Run ``embed`` on the repaired wheel, after ``auditwheel``, to describe the file -you publish; auditwheel's own SBOM in the same directory stays untouched. +``auditwheel repair`` renames a Linux wheel to its ``manylinux`` tag, so the attestation link in an SBOM written before +the repair names the old file. Run ``embed`` on the repaired wheel, after ``auditwheel``, to describe the file you +publish; auditwheel's own SBOM in the same directory stays untouched. Render it as SPDX ----------------- @@ -146,10 +146,10 @@ number, so both documents name the same build. Check it in CI -------------- -``python-sbom validate`` needs the ``validate`` extra, and says so when it is missing. It checks every CycloneDX SBOM -in each wheel's ``.dist-info/sboms`` against the CycloneDX 1.6 schema. For the SBOM python-sbom wrote, it also requires -the ``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, checks that every reference resolves, -and validates the SPDX rendering; an SBOM another tool wrote, such as auditwheel's, gets the schema check alone: +``python-sbom validate`` needs the ``validate`` extra, and says so when it is missing. It checks every CycloneDX SBOM in +each wheel's ``.dist-info/sboms`` against the CycloneDX 1.6 schema. For the SBOM python-sbom wrote, it also requires the +``urn:uuid`` serial number ``actions/attest`` needs to recognize the format, checks that every reference resolves, and +validates the SPDX rendering; an SBOM another tool wrote, such as auditwheel's, gets the schema check alone: .. code-block:: console @@ -160,8 +160,8 @@ Each ``--allow-license`` adds an SPDX id that bundled wheels may use. With any g ids do not allow fails the check, and so does one that declares a license only by name. Before anything else, ``validate`` counts the components an SBOM lists, nested ones included, and fails an SBOM that -lists more than 100,000 without checking it further. Every check takes time in proportion to the count, about a tenth -of a second for 100,000 components, so the cap bounds what an untrusted SBOM can ask of a CI job while staying far above +lists more than 100,000 without checking it further. Every check takes time in proportion to the count, about a tenth of +a second for 100,000 components, so the cap bounds what an untrusted SBOM can ask of a CI job while staying far above real SBOMs: every file of the wheel, of each bundled wheel and of each build tool counts, and virtualenv's wheel lists about 5,200. Raise the cap for an SBOM you expect to be larger, or lower it to refuse more: diff --git a/src/python_sbom/_collect/_content.py b/src/python_sbom/_collect/_content.py index a34fac2..ad67b8d 100644 --- a/src/python_sbom/_collect/_content.py +++ b/src/python_sbom/_collect/_content.py @@ -35,9 +35,9 @@ def load_content_provider(spec: str | None, root: Path) -> ContentProvider: def provider_script(provider: ContentProvider, root: Path) -> PurePosixPath | None: - # a provider defined in the project shapes the document, so the SBOM lists its module among the build tools - # a method's code names the file even for a module hatchling loaded outside the import system, such as a custom - # hook script + # a provider defined in the project shapes the document, so the SBOM lists its module among the build tools a + # method's code names the file even for a module hatchling loaded outside the import system, such as a custom hook + # script cls = type(provider) methods = ( function.__code__.co_filename diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index 970012d..e2809b4 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -81,8 +81,8 @@ def _recorded_files(distribution: Distribution) -> list[RecordRow]: distribution ): return [] - # RECORD read once and as written: Distribution.files would stat every row to drop files missing on disk, and - # build a path object for each, while the SBOM lists what the install recorded + # RECORD read once and as written: Distribution.files would stat every row to drop files missing on disk, and build + # a path object for each, while the SBOM lists what the install recorded try: entries: Final[list[RecordEntry]] = list( starmap(RecordEntry.from_elements, parse_record_file((distribution.read_text("RECORD") or "").splitlines())) diff --git a/src/python_sbom/_errors.py b/src/python_sbom/_errors.py index 154f689..43400cb 100644 --- a/src/python_sbom/_errors.py +++ b/src/python_sbom/_errors.py @@ -5,8 +5,8 @@ class SbomError(Exception): """Base of the errors this package raises; the command line reports them without a traceback.""" -# callers catch these as SbomError, or as the built-in some of them also are; SettingsError covers every option, -# file or variable the project sets wrongly +# callers catch these as SbomError, or as the built-in some of them also are; SettingsError covers every option, file or +# variable the project sets wrongly class SettingsError(SbomError): ... # simplint: ignore[SMP704] # see above diff --git a/src/python_sbom/_formats/_spdx.py b/src/python_sbom/_formats/_spdx.py index 69d63f5..e0944a9 100644 --- a/src/python_sbom/_formats/_spdx.py +++ b/src/python_sbom/_formats/_spdx.py @@ -152,8 +152,8 @@ def render_spdx(bom: Document, tool_version: str) -> Spdx: def _tools(metadata: Metadata) -> tuple[list[Component], list[str]]: - # CycloneDX 1.5 lists the tools as components; earlier versions as a bare array, whose entries carry no bom-ref, - # so they become creators alone + # CycloneDX 1.5 lists the tools as components; earlier versions as a bare array, whose entries carry no bom-ref, so + # they become creators alone tools: Final[Tools | list[_LegacyTool] | None] = metadata.get("tools") if isinstance(tools, list): return [], [_creator(name, tool.get("version")) for tool in tools if (name := tool.get("name"))] diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py index 42af1ae..f92da2d 100644 --- a/src/python_sbom/_model/_inputs.py +++ b/src/python_sbom/_model/_inputs.py @@ -123,8 +123,8 @@ class Contributions: """ What a project adds to the document beyond its bundled wheels, merged before references and the serial are derived. - Each field defaults to empty, so a provider names only what it adds. The build fails when a ``bom-ref`` repeats or - a dependency names one that does not exist. + Each field defaults to empty, so a provider names only what it adds. The build fails when a ``bom-ref`` repeats or a + dependency names one that does not exist. :param components: extra top-level components, such as vendored C libraries; the root depends on each, and a component's own parts nest under its ``components`` diff --git a/src/python_sbom/_model/_licenses.py b/src/python_sbom/_model/_licenses.py index 17ff8a1..9f0248e 100644 --- a/src/python_sbom/_model/_licenses.py +++ b/src/python_sbom/_model/_licenses.py @@ -11,8 +11,8 @@ def licenses(metadata: CoreMetadata) -> list[License]: - # a License-Expression is taken as declared; otherwise each License field and License classifier name is - # declared, and the SPDX ids they map to are added as concluded + # a License-Expression is taken as declared; otherwise each License field and License classifier name is declared, + # and the SPDX ids they map to are added as concluded if expression := metadata.first("license_expression"): try: return [{"expression": canonicalize_license_expression(expression), "acknowledgement": "declared"}] diff --git a/src/python_sbom/_model/_metadata.py b/src/python_sbom/_model/_metadata.py index 11e44a7..df553c8 100644 --- a/src/python_sbom/_model/_metadata.py +++ b/src/python_sbom/_model/_metadata.py @@ -54,8 +54,8 @@ def first(self, field: Field) -> str | None: return next(iter(self.values(field)), None) def values(self, field: Field) -> list[str]: - # packaging keeps a field it cannot take as declared, such as a single-use field given twice, out of raw and - # its values as written in unparsed, from where they are still described + # packaging keeps a field it cannot take as declared, such as a single-use field given twice, out of raw and its + # values as written in unparsed, from where they are still described match self.raw.get(field): case str() as value: return [value] diff --git a/src/python_sbom/_model/_types.py b/src/python_sbom/_model/_types.py index 6c154d4..c0f524c 100644 --- a/src/python_sbom/_model/_types.py +++ b/src/python_sbom/_model/_types.py @@ -145,9 +145,9 @@ class Evidence(TypedDict): identity: list[Identity] -#: A CycloneDX 1.6 component: a distribution, an application, the interpreter or a file, as a dictionary with the -#: keys the CycloneDX component object defines, of which python-sbom uses ``type``, ``bom-ref``, ``name``, -#: ``version``, ``purl``, ``supplier``, ``authors``, ``description``, ``licenses``, ``copyright``, ``hashes``, +#: A CycloneDX 1.6 component: a distribution, an application, the interpreter or a file, as a dictionary with the keys +#: the CycloneDX component object defines, of which python-sbom uses ``type``, ``bom-ref``, ``name``, ``version``, +#: ``purl``, ``supplier``, ``authors``, ``description``, ``licenses``, ``copyright``, ``hashes``, #: ``externalReferences``, ``properties``, ``evidence`` and the nested ``components``. Component = TypedDict( "Component", diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 6981bcb..d09a551 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -64,8 +64,8 @@ def validate_document( def _problems(raw: str, allowed_licenses: frozenset[str] | None, max_components: int) -> list[str]: # every document must conform to the CycloneDX 1.6 schema; one python-sbom wrote must also carry the urn:uuid serial - # actions/attest needs, resolvable and unique bom-refs and a valid SPDX rendering, and meet the license policy - # a missing extra fails every document alike, while importing it, half a second, waits until the cheap census passes + # actions/attest needs, resolvable and unique bom-refs and a valid SPDX rendering, and meet the license policy a + # missing extra fails every document alike, while importing it, half a second, waits until the cheap census passes if any(find_spec(name) is None for name in _EXTRA): msg = "validation needs the validate extra: pip install 'python-sbom[validate]'" raise MissingExtraError(msg) diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index fb43858..abeb105 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -89,8 +89,8 @@ def open_archive(file: Path | IO[bytes], where: str) -> Generator[zipfile.ZipFil def _wheel_source(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, str]: - # installer refuses a wheel whose file name does not parse, or that holds other than one top-level .dist-info, - # since a second one is a decoy that could decide whose metadata counts + # installer refuses a wheel whose file name does not parse, or that holds other than one top-level .dist-info, since + # a second one is a decoy that could decide whose metadata counts try: wheel = WheelFile(archive) dist_info = wheel.dist_info_dir From edb4357653517d12af70b84b84a81d6ecb776c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:36:18 -0800 Subject: [PATCH 052/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20build?= =?UTF-8?q?=20every=20artifact's=20SBOM=20in=20one=20place?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The archive document duplicated the wheel's provenance outside the model and had drifted: it never wrote vcs-dirty, so a dirty checkout looked exact, and lacked the vcs link, sbom:repository, sbom:scope, the license link, copyright and compositions. An Artifact record now carries what is specific to a wheel or an archive, and _model._bom.assemble adds the supplier, source links, revision, tools, dependencies and formulation for both. The wheel's SBOM is unchanged byte for byte. --- src/python_sbom/_collect/_source.py | 22 ++-- src/python_sbom/_generate/_archive.py | 82 ++++--------- src/python_sbom/_model/_bom.py | 164 +++++++++++++++----------- src/python_sbom/_model/_inputs.py | 17 +++ tests/_generate/test_archive.py | 33 +++++- 5 files changed, 179 insertions(+), 139 deletions(-) diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index 81253fb..27a6170 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -33,16 +33,7 @@ def read_source_tree(request: SbomRequest) -> SourceTree: msg = f"`bundled-wheels` pattern {pattern!r} must be relative to the project, without `..`" raise SettingsError(msg) return SourceTree( - copyright=next( - ( - line - for license_file in request.metadata.values("license_files") - if (path := root / license_file).is_file() - for line in path.read_text(encoding="utf-8").splitlines() - if line.startswith("Copyright") - ), - None, - ), + copyright=copyright_notice(root, request.metadata.values("license_files")), bundled=read_bundled_wheels( root, sorted({ @@ -59,6 +50,16 @@ def read_source_tree(request: SbomRequest) -> SourceTree: ) +def copyright_notice(root: Path, license_files: list[str]) -> str | None: + # the first Copyright line of the license files the project names, as they sit in the project + for license_file in license_files: + if (path := root / license_file).is_file(): + for line in path.read_text(encoding="utf-8").splitlines(): + if line.startswith("Copyright"): + return line + return None + + def read_bundled_wheels(root: Path, paths: list[str]) -> list[BundledWheel]: # inflating and hashing release the GIL, so the wheels verify side by side; results and the first error come in the # order given @@ -178,6 +179,7 @@ def wheel_request(wheel: Path, root: Path, settings: Settings, build_scripts: tu __all__ = [ + "copyright_notice", "read_bundled_wheel", "read_bundled_wheels", "read_source_tree", diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index 466c939..158d6fa 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -10,22 +10,19 @@ from python_sbom._collect._content import load_content_provider, provider_script from python_sbom._collect._settings import settings_from_pyproject -from python_sbom._collect._source import read_bundled_wheels +from python_sbom._collect._source import copyright_notice, read_bundled_wheels from python_sbom._errors import WheelError +from python_sbom._model._bom import assemble from python_sbom._model._components import ( bundled_wheel_component, component_from_metadata, file_component_from_bytes, - script_tool, vendored_dependencies, ) from python_sbom._model._content import ProviderContext -from python_sbom._model._document import contributed_properties, formulation, timestamp, with_header from python_sbom._model._identity import project_identity +from python_sbom._model._inputs import Artifact, BuildScript from python_sbom._model._metadata import parse_metadata -from python_sbom._model._names import SBOM_FILE, sbom_property -from python_sbom._model._tools import tool_components -from python_sbom._model._types import Dependency from python_sbom._wheel._archive import open_archive if TYPE_CHECKING: @@ -35,7 +32,7 @@ from python_sbom._model._identity import ProjectIdentity from python_sbom._model._inputs import BuildFacts, BundledWheel, Settings from python_sbom._model._metadata import CoreMetadata - from python_sbom._model._types import Component, Document, Metadata + from python_sbom._model._types import Component, Document def describe_archive( @@ -54,56 +51,26 @@ def describe_archive( with tempfile.TemporaryDirectory() as directory: held = _read_archive(root, archive, Path(directory), settings, content) contributions = content.contributions(held.reader.context) - held.root["properties"] = list(contributed_properties(contributions.root_properties, held.project.property_prefix)) - if facts.revision is not None: - held.root["properties"].append({ - "name": f"{held.project.property_prefix}:vcs-commit", - "value": facts.revision.commit, - }) - components: Final[list[Component]] = [*held.components, *contributions.components] - tools: Final[list[Component]] = [ - *( - script_tool( - path, - hashlib.sha256((root / str(path)).read_bytes()).hexdigest(), - held.version, - None - if held.project.repository is None - else f"{held.project.repository}/blob/{settings.default_branch}/{path}", - ) + license_files: Final[list[str]] = held.metadata.values("license_files") + artifact = Artifact( + root=held.root, + components=held.components, + dependencies=vendored_dependencies(held.components), + build_scripts=[ + BuildScript(path=path, sha256=hashlib.sha256((root / str(path)).read_bytes()).hexdigest()) for path in [*build_scripts, *filter(None, [provider_script(content, root)])] - ), - *(environment := tool_components(facts))[0], - *contributions.tools, - ] - metadata: Final[Metadata] = { - "timestamp": timestamp(facts), - "lifecycles": [{"phase": "build"}], - "tools": {"components": tools}, - "component": held.root, - # names the document's file, which marks it as one python-sbom built, so validation checks it in full - "properties": [ - {"name": sbom_property(held.project.property_prefix, SBOM_FILE), "value": f"{archive.name}.cdx.json"} ], - } - if settings.supplier is not None: - held.root["supplier"] = metadata["manufacturer"] = metadata["supplier"] = settings.supplier - return with_header( - { - "metadata": metadata, - "components": components, - "dependencies": [ - Dependency(ref=held.root["bom-ref"], dependsOn=[component["bom-ref"] for component in components]), - *vendored_dependencies(components), - *environment[1], - *contributions.dependencies, - ], - "formulation": [ - formulation("archive-build", f"build {archive.name} and this SBOM", tools, held.root["bom-ref"], facts) - ], - }, - held.project.serial_namespace, + version=held.metadata.version, + sbom_file=f"{archive.name}.cdx.json", + scope=( + "Components are the distributions the archive holds, as wheels or unpacked, each with its files; the" + " archive's own top-level files sit under the root component." + ), + workflow=("archive-build", f"build {archive.name} and this SBOM"), + license_file=next(iter(license_files), None), + copyright=copyright_notice(root, license_files), ) + return assemble(artifact, held.reader.project, settings, facts, contributions) def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, content: ContentProvider) -> _Held: @@ -148,8 +115,7 @@ def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, ref = PackageURL(type="generic", name=archive.name, version=version).to_string() return _Held( reader=reader, - project=reader.project, - version=version, + metadata=metadata[""], root={ "type": "application", "bom-ref": ref, @@ -207,8 +173,8 @@ def file(self, parent: str, name: str) -> Component: @dataclass(frozen=True, slots=True, kw_only=True) class _Held: reader: _Reader - project: ProjectIdentity - version: str + # the project's own, from the archive's top-level .dist-info + metadata: CoreMetadata root: Component components: list[Component] diff --git a/src/python_sbom/_model/_bom.py b/src/python_sbom/_model/_bom.py index 530168b..ec3d74c 100644 --- a/src/python_sbom/_model/_bom.py +++ b/src/python_sbom/_model/_bom.py @@ -9,6 +9,7 @@ from ._components import bundled_wheel_component, component_from_metadata, script_tool, vendored_dependencies from ._document import contributed_properties, formulation, timestamp, with_header from ._identity import project_identity +from ._inputs import Artifact from ._markers import applies_without_extras from ._names import SBOM_FILE, SBOM_REPOSITORY, purl, pypi_url, sbom_file_name, sbom_property from ._tools import tool_components @@ -28,8 +29,8 @@ def describe( bundled_info: Mapping[str, BundledWheelInfo], contributions: Contributions, ) -> Document: - project = project_identity(request.metadata, request.settings) - root = _root_component(request, source, facts, project, contributions) + metadata, settings = request.metadata, request.settings + project = project_identity(metadata, settings) bundled = [ bundled_wheel_component( wheel, @@ -39,75 +40,118 @@ def describe( ) for wheel in source.bundled ] - components = [ - *bundled, - # hatchling repeats a Requires-Dist when a hook re-adds a dependency, and each bom-ref must be unique - *( - _declared_dependency(text, requirement, project.property_prefix) - for text in dict.fromkeys(request.metadata.values("requires_dist")) - if applies_without_extras(requirement := Requirement(text)) - ), - *contributions.components, + root = component_from_metadata(metadata) + root["type"] = "application" + name, version = metadata.name, metadata.version + root["externalReferences"] = [ + *root["externalReferences"], + {"type": "distribution", "url": pypi_url(name, version)}, + {"type": "attestation", "url": f"https://pypi.org/integrity/{name}/{version}/{request.wheel_name}/provenance"}, ] + artifact = Artifact( + root=root, + components=[ + *bundled, + # hatchling repeats a Requires-Dist when a hook re-adds a dependency, and each bom-ref must be unique + *( + _declared_dependency(text, requirement, project.property_prefix) + for text in dict.fromkeys(metadata.values("requires_dist")) + if applies_without_extras(requirement := Requirement(text)) + ), + ], + dependencies=vendored_dependencies(bundled), + build_scripts=source.build_scripts, + version=version, + sbom_file=sbom_file_name(name), + scope=_scope(settings), + workflow=("wheel-build", "build the wheel and this SBOM"), + license_file=next(iter(metadata.values("license_files")), None), + copyright=source.copyright, + ) + return assemble(artifact, project, settings, facts, contributions) + + +def _declared_dependency(text: str, requirement: Requirement, namespace: str) -> Component: + component: Final[Component] = { + "type": "library", + # the same distribution can be declared more than once with different markers, so the purl is not unique + "bom-ref": f"requires-dist:{text}", + "name": requirement.name, + "purl": purl(requirement.name), + "externalReferences": [{"type": "distribution", "url": pypi_url(requirement.name)}], + "properties": [{"name": f"{namespace}:requires-dist", "value": text}], + } + if requirement.marker is not None: + component["properties"].append({"name": f"{namespace}:environment-marker", "value": str(requirement.marker)}) + return component + + +def _scope(settings: Settings) -> str: + declared = ( + "the direct runtime dependencies declared in the wheel's core metadata, which carry no version because the" + " installer resolves them" + ) + if directories := " and ".join(sorted({str(PurePosixPath(pattern).parent) for pattern in settings.bundled_wheels})): + return f"Components are the wheels bundled as data files under {directories} and {declared}." + return f"Components are {declared}." + + +def assemble( + artifact: Artifact, project: ProjectIdentity, settings: Settings, facts: BuildFacts, contributions: Contributions +) -> Document: + # what every artifact of the project carries alike: who supplies it, where its source lives and at which revision, + # the tools that built it and what the project contributes + root = _provenance(artifact, project, settings, facts, contributions) + components: Final[list[Component]] = [*artifact.components, *contributions.components] environment, tool_dependencies = tool_components(facts) - tools = [ + tools: Final[list[Component]] = [ *( - script_tool( - script.path, - script.sha256, - request.metadata.version, - _file_link(project, request.settings, str(script.path)), - ) - for script in source.build_scripts + script_tool(script.path, script.sha256, artifact.version, _file_link(project, settings, script.path)) + for script in artifact.build_scripts ), *environment, *contributions.tools, ] + uid, name = artifact.workflow return with_header( { - "metadata": _metadata(root, tools, request, project, facts), + "metadata": _metadata(tools, artifact, project, settings, facts), "components": components, "dependencies": [ {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in components]}, - *vendored_dependencies(bundled), + *artifact.dependencies, *tool_dependencies, *contributions.dependencies, ], - # transitive runtime dependencies are unknown until install time + # what the artifact needs at run time beyond what it ships, an interpreter and the dependencies an installer + # resolves, is not listed "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], - "formulation": [formulation("wheel-build", "build the wheel and this SBOM", tools, root["bom-ref"], facts)], + "formulation": [formulation(uid, name, tools, root["bom-ref"], facts)], }, project.serial_namespace, ) -def _root_component( - request: SbomRequest, source: SourceTree, facts: BuildFacts, project: ProjectIdentity, contributions: Contributions +def _provenance( + artifact: Artifact, project: ProjectIdentity, settings: Settings, facts: BuildFacts, contributions: Contributions ) -> Component: - metadata, settings = request.metadata, request.settings - name, version = metadata.name, metadata.version - root = component_from_metadata(metadata) - root["type"] = "application" + root = artifact.root if settings.supplier is not None: root["supplier"] = settings.supplier - if source.copyright is not None: - root["copyright"] = source.copyright - references: Final[list[ExternalReference]] = [ - *root["externalReferences"], - {"type": "distribution", "url": pypi_url(name, version)}, - {"type": "attestation", "url": f"https://pypi.org/integrity/{name}/{version}/{request.wheel_name}/provenance"}, - ] + if artifact.copyright is not None: + root["copyright"] = artifact.copyright + references: Final[list[ExternalReference]] = [*root.get("externalReferences", [])] if project.repository and urlsplit(project.repository).netloc == "github.com": references.extend([ {"type": "security-contact", "url": f"{project.repository}/security/policy"}, {"type": "advisories", "url": f"{project.repository}/security/advisories"}, ]) - if (license_files := metadata.values("license_files")) and ( - link := _file_link(project, settings, license_files[0]) + if artifact.license_file is not None and ( + link := _file_link(project, settings, PurePosixPath(artifact.license_file)) ): references.append({"type": "license", "url": link}) properties: Final[list[Property]] = [ - *root["properties"], + *root.get("properties", []), *contributed_properties(contributions.root_properties, project.property_prefix), ] if (revision := facts.revision) is not None: @@ -126,34 +170,22 @@ def _root_component( return root -def _declared_dependency(text: str, requirement: Requirement, namespace: str) -> Component: - component: Final[Component] = { - "type": "library", - # the same distribution can be declared more than once with different markers, so the purl is not unique - "bom-ref": f"requires-dist:{text}", - "name": requirement.name, - "purl": purl(requirement.name), - "externalReferences": [{"type": "distribution", "url": pypi_url(requirement.name)}], - "properties": [{"name": f"{namespace}:requires-dist", "value": text}], - } - if requirement.marker is not None: - component["properties"].append({"name": f"{namespace}:environment-marker", "value": str(requirement.marker)}) - return component - - -def _file_link(project: ProjectIdentity, settings: Settings, path: str) -> str | None: +def _file_link(project: ProjectIdentity, settings: Settings, path: PurePosixPath) -> str | None: return None if project.repository is None else f"{project.repository}/blob/{settings.default_branch}/{path}" def _metadata( - root: Component, tools: list[Component], request: SbomRequest, project: ProjectIdentity, facts: BuildFacts + tools: list[Component], + artifact: Artifact, + project: ProjectIdentity, + settings: Settings, + facts: BuildFacts, ) -> Metadata: - settings = request.settings metadata: Final[Metadata] = { "timestamp": timestamp(facts), "lifecycles": [{"phase": "build"}], "tools": {"components": tools}, - "component": root, + "component": (root := artifact.root), } if settings.supplier is not None: metadata["manufacturer"] = metadata["supplier"] = settings.supplier @@ -162,8 +194,9 @@ def _metadata( if "licenses" in root: metadata["licenses"] = root["licenses"] properties: Final[list[Property]] = [ - {"name": sbom_property(project.property_prefix, "sbom:scope"), "value": _scope(settings)}, - {"name": sbom_property(project.property_prefix, SBOM_FILE), "value": sbom_file_name(request.metadata.name)}, + {"name": sbom_property(project.property_prefix, "sbom:scope"), "value": artifact.scope}, + # names the document's file, which marks it as one python-sbom built, so validation checks it in full + {"name": sbom_property(project.property_prefix, SBOM_FILE), "value": artifact.sbom_file}, ] if project.repository is not None: properties.append({ @@ -174,16 +207,7 @@ def _metadata( return metadata -def _scope(settings: Settings) -> str: - declared = ( - "the direct runtime dependencies declared in the wheel's core metadata, which carry no version because the" - " installer resolves them" - ) - if directories := " and ".join(sorted({str(PurePosixPath(pattern).parent) for pattern in settings.bundled_wheels})): - return f"Components are the wheels bundled as data files under {directories} and {declared}." - return f"Components are {declared}." - - __all__ = [ + "assemble", "describe", ] diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py index f92da2d..f7685cf 100644 --- a/src/python_sbom/_model/_inputs.py +++ b/src/python_sbom/_model/_inputs.py @@ -94,6 +94,22 @@ class BuildScript: sha256: str +@dataclass(frozen=True, slots=True, kw_only=True) +class Artifact: + # what one artifact, a wheel or an archive, puts in its SBOM; the provenance every artifact of the project shares is + # added around it + root: Component + components: list[Component] + dependencies: list[Dependency] + build_scripts: list[BuildScript] + version: str + sbom_file: str + scope: str + workflow: tuple[str, str] + license_file: str | None + copyright: str | None + + @dataclass(frozen=True, slots=True, kw_only=True) class SourceTree: copyright: str | None @@ -142,6 +158,7 @@ class Contributions: __all__ = [ + "Artifact", "BuildFacts", "BuildScript", "BundledWheel", diff --git a/tests/_generate/test_archive.py b/tests/_generate/test_archive.py index aa7f755..823ba5d 100644 --- a/tests/_generate/test_archive.py +++ b/tests/_generate/test_archive.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import io import json import zipfile @@ -13,6 +14,7 @@ from python_sbom._errors import WheelError from python_sbom._generate._archive import describe_archive from python_sbom._generate._embed import revision_reader +from python_sbom._model._inputs import Revision from python_sbom._validate._artifact import validate if TYPE_CHECKING: @@ -104,6 +106,15 @@ def test_archive_root(document: Document) -> None: "version": "1.0", "purl": "pkg:generic/app.pyz@1.0", "supplier": {"name": "Example Org", "url": []}, + "externalReferences": [ + {"type": "security-contact", "url": "https://github.com/example/app/security/policy"}, + {"type": "advisories", "url": "https://github.com/example/app/security/advisories"}, + { + "type": "vcs", + "url": f"https://github.com/example/app/tree/{_COMMIT}", + "comment": "exact source revision", + }, + ], "properties": [ {"name": "extra:project", "value": "project"}, {"name": "extra:table", "value": "loaded.json"}, @@ -112,6 +123,18 @@ def test_archive_root(document: Document) -> None: } +def test_archive_of_dirty_checkout(project: Path, archive: Path) -> None: + # the same provenance a wheel built from a dirty checkout carries, so the commit is not taken for the exact source + facts = dataclasses.replace( + collect_facts(project, revision_reader(_COMMIT)), revision=Revision(commit=_COMMIT, dirty=True) + ) + root = describe_archive(project, archive, facts=facts)["metadata"]["component"] + assert (root.get("properties", [])[-1], root.get("externalReferences", [])[-1]["comment"]) == ( + {"name": "app:vcs-dirty", "value": "true"}, + "source revision, with uncommitted changes", + ) + + def test_archive_loose_files_belong_to_the_archive(document: Document) -> None: files = document["metadata"]["component"].get("components", []) assert [file["name"] for file in files] == ["__main__.py", "loaded.json"] @@ -156,7 +179,15 @@ def test_archive_tools_and_metadata(document: Document) -> None: assert (script["name"], script.get("externalReferences"), metadata.get("properties"), metadata.get("supplier")) == ( "build_app.py", [{"type": "vcs", "url": "https://github.com/example/app/blob/main/build_app.py"}], - [{"name": "app:sbom:file", "value": "app.pyz.cdx.json"}], + [ + { + "name": "app:sbom:scope", + "value": "Components are the distributions the archive holds, as wheels or unpacked, each with its" + " files; the archive's own top-level files sit under the root component.", + }, + {"name": "app:sbom:file", "value": "app.pyz.cdx.json"}, + {"name": "app:sbom:repository", "value": "https://github.com/example/app"}, + ], {"name": "Example Org", "url": []}, ) From 9faebb2487a18bf014461037a957f9ebf0fb98e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:40:03 -0800 Subject: [PATCH 053/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20fail=20the=20buil?= =?UTF-8?q?d=20on=20contributions=20validate=20rejects?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A provider could build an SBOM that validate then rejected: a second dependency entry for a ref, the root's included, a component taking a formula: or workflow: ref, or, with no bom-ref, a KeyError traceback. The build now refuses each with an SbomError naming the problem, and the provider for a component lacking type, bom-ref or name. --- docs/extend.rst | 6 ++- src/python_sbom/_generate/_archive.py | 3 +- src/python_sbom/_generate/_pipeline.py | 3 +- src/python_sbom/_model/_document.py | 58 +++++++++++++++++---- src/python_sbom/_model/_inputs.py | 6 ++- tests/_model/test_document.py | 72 ++++++++++++++++++++++++-- 6 files changed, 127 insertions(+), 21 deletions(-) diff --git a/docs/extend.rst b/docs/extend.rst index df272e9..933a85e 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -18,8 +18,10 @@ it adds. only from here, never from the wheel's own metadata. - :meth:`~python_sbom.ContentProvider.contributions` returns extra top-level components, the dependencies between them, root properties and build tools, such as the C libraries a native wheel links. They join the document before the - root's dependencies and the serial number derive from it. The build fails when a ``bom-ref`` repeats, a dependency - names one that no component has, or a root property uses the project's own prefix. + root's dependencies and the serial number derive from it. The build fails when a component or tool lacks ``type``, + ``bom-ref`` or ``name``, a ``bom-ref`` repeats, the ``formula:`` and ``workflow:`` refs of the build's formulation + included, a dependency names one that no component has, a ``bom-ref`` gets a second dependency entry (the root's + included, which python-sbom writes), or a root property uses the project's own prefix. Name the provider in the ``content-provider`` option, as ``module:attribute`` importable from the project root, and the hook, ``python-sbom embed`` and ``python-sbom archive`` all load it. When the provider's class lives in the project, the diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index 158d6fa..2b552f7 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -20,6 +20,7 @@ vendored_dependencies, ) from python_sbom._model._content import ProviderContext +from python_sbom._model._document import checked_contributions from python_sbom._model._identity import project_identity from python_sbom._model._inputs import Artifact, BuildScript from python_sbom._model._metadata import parse_metadata @@ -50,7 +51,7 @@ def describe_archive( content = content or load_content_provider(settings.content_provider, root) with tempfile.TemporaryDirectory() as directory: held = _read_archive(root, archive, Path(directory), settings, content) - contributions = content.contributions(held.reader.context) + contributions = checked_contributions(content.contributions(held.reader.context), content) license_files: Final[list[str]] = held.metadata.values("license_files") artifact = Artifact( root=held.root, diff --git a/src/python_sbom/_generate/_pipeline.py b/src/python_sbom/_generate/_pipeline.py index 9a751ef..5428bde 100644 --- a/src/python_sbom/_generate/_pipeline.py +++ b/src/python_sbom/_generate/_pipeline.py @@ -6,6 +6,7 @@ from python_sbom._formats._json import dumps from python_sbom._model._bom import describe +from python_sbom._model._document import checked_contributions from python_sbom._model._names import sbom_file_name if TYPE_CHECKING: @@ -24,7 +25,7 @@ def build_sbom( source, facts, {wheel.path: content.bundled_wheel(context, PurePosixPath(wheel.path)) for wheel in source.bundled}, - content.contributions(context), + checked_contributions(content.contributions(context), content), ), ) diff --git a/src/python_sbom/_model/_document.py b/src/python_sbom/_model/_document.py index ced9142..390be2c 100644 --- a/src/python_sbom/_model/_document.py +++ b/src/python_sbom/_model/_document.py @@ -9,27 +9,41 @@ from python_sbom._errors import SbomError if TYPE_CHECKING: - from ._inputs import BuildFacts + from collections.abc import Iterable, Iterator + + from ._content import ContentProvider + from ._inputs import BuildFacts, Contributions from ._types import Body, Component, Document, Formula, Property def with_header(body: Body, namespace: uuid.UUID) -> Document: # a provider contributes refs of its own, so the document is checked before it is written rather than by validate - refs: Final[Counter[str]] = Counter() - pending: Final[list[Component]] = [ - body["metadata"]["component"], - *body["components"], - *body["metadata"]["tools"]["components"], + components: Final[list[Component]] = [ + *_walk_components([ + body["metadata"]["component"], + *body["components"], + *body["metadata"]["tools"]["components"], + ]) ] - while pending: - component = pending.pop() - refs[component["bom-ref"]] += 1 - pending.extend(component.get("components", [])) + refs: Final[Counter[str]] = Counter(component["bom-ref"] for component in components) + # formulas and workflows share the bom-ref namespace, so a component cannot take theirs + refs.update( + ref + for formula in body.get("formulation", []) + for ref in (formula["bom-ref"], *(workflow["bom-ref"] for workflow in formula["workflows"])) + ) if repeated := sorted(ref for ref, count in refs.items() if count > 1): msg = f"bom-ref values repeat: {', '.join(repeated)}" raise SbomError(msg) + # one entry per component, as validation requires, so a provider adds to a component's dependencies, not a second + # entry for it + entries: Final[Counter[str]] = Counter(entry["ref"] for entry in body["dependencies"]) + if repeated := sorted(ref for ref, count in entries.items() if count > 1): + msg = f"dependencies list more than one entry for: {', '.join(repeated)}" + raise SbomError(msg) + known: Final[set[str]] = {component["bom-ref"] for component in components} named = {ref for entry in body["dependencies"] for ref in (entry["ref"], *entry.get("dependsOn", []))} - if unknown := sorted(named - refs.keys()): + if unknown := sorted(named - known): msg = f"dependencies name bom-refs no component has: {', '.join(unknown)}" raise SbomError(msg) return { @@ -42,6 +56,27 @@ def with_header(body: Body, namespace: uuid.UUID) -> Document: } +def _walk_components(components: Iterable[Component]) -> Iterator[Component]: + # depth first, each component before the ones it nests, in document order; iterative, so deep nesting cannot exhaust + # the stack + pending: Final[list[Component]] = list(reversed(list(components))) + while pending: + component = pending.pop() + yield component + pending.extend(reversed(component.get("components", []))) + + +def checked_contributions(contributions: Contributions, provider: ContentProvider) -> Contributions: + # what a provider adds is checked for the fields everything else here reads, so a mistake names the provider rather + # than surfacing as a KeyError deep in the build + for component in _walk_components([*contributions.components, *contributions.tools]): + if missing := [key for key in ("type", "bom-ref", "name") if key not in component]: + name = type(provider).__qualname__ + msg = f"content provider {name} added a component without {', '.join(missing)}: {component!r:.200}" + raise SbomError(msg) + return contributions + + def contributed_properties(properties: list[Property], prefix: str) -> list[Property]: if claimed := sorted(prop["name"] for prop in properties if prop["name"].startswith(f"{prefix}:")): msg = f"contributed root properties {', '.join(claimed)} use the {prefix}: prefix python-sbom writes" @@ -75,6 +110,7 @@ def timestamp(facts: BuildFacts) -> str: __all__ = [ + "checked_contributions", "contributed_properties", "formulation", "timestamp", diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py index f7685cf..d62decf 100644 --- a/src/python_sbom/_model/_inputs.py +++ b/src/python_sbom/_model/_inputs.py @@ -139,8 +139,10 @@ class Contributions: """ What a project adds to the document beyond its bundled wheels, merged before references and the serial are derived. - Each field defaults to empty, so a provider names only what it adds. The build fails when a ``bom-ref`` repeats or a - dependency names one that does not exist. + Each field defaults to empty, so a provider names only what it adds. The build fails when a component or tool lacks + ``type``, ``bom-ref`` or ``name``, a ``bom-ref`` repeats, the ``formula:`` and ``workflow:`` refs of the build's + formulation included, a dependency names one that does not exist, or a ``bom-ref`` gets a second dependency entry, + as the root does if a provider adds one, since python-sbom writes the root's. :param components: extra top-level components, such as vendored C libraries; the root depends on each, and a component's own parts nest under its ``components`` diff --git a/tests/_model/test_document.py b/tests/_model/test_document.py index 6cf9626..641707a 100644 --- a/tests/_model/test_document.py +++ b/tests/_model/test_document.py @@ -2,16 +2,18 @@ import json import uuid -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast import pytest +from python_sbom import ContentProvider, Contributions from python_sbom._errors import SbomError -from python_sbom._model._document import contributed_properties, with_header +from python_sbom._model._document import checked_contributions, contributed_properties, formulation, with_header if TYPE_CHECKING: - from python_sbom import Property - from python_sbom._model._types import Body + from python_sbom import Component, Dependency, Property + from python_sbom._model._inputs import BuildFacts + from python_sbom._model._types import Body, Formula def test_document_serial_derives_from_body() -> None: @@ -74,3 +76,65 @@ def test_contributed_properties_keep_their_own_prefix() -> None: def test_contributed_properties_refuse_the_project_prefix() -> None: with pytest.raises(SbomError, match="example:vcs-commit use the example: prefix python-sbom writes"): contributed_properties([{"name": "example:vcs-commit", "value": "0"}], "example") + + +def _partial(**fields: str | list[dict[str, str]]) -> Component: + # a component a provider got wrong, which the types alone would not let through + return cast("Component", fields) + + +def test_header_refuses_a_second_dependency_entry_for_a_ref() -> None: + # as when a provider adds an entry for the root, whose entry python-sbom writes + body = _body([], [{"ref": "app", "dependsOn": []}, {"ref": "app", "dependsOn": []}], []) + with pytest.raises(SbomError, match="dependencies list more than one entry for: app"): + with_header(body, uuid.uuid4()) + + +def _body(components: list[Component], dependencies: list[Dependency], formulation: list[Formula]) -> Body: + return { + "metadata": { + "timestamp": "2023-11-14T22:13:20+00:00", + "lifecycles": [{"phase": "build"}], + "tools": {"components": []}, + "component": {"type": "application", "bom-ref": "app", "name": "app"}, + }, + "components": components, + "dependencies": dependencies, + "formulation": formulation, + } + + +@pytest.mark.parametrize( + "ref", [pytest.param("formula:build", id="formula"), pytest.param("workflow:build", id="workflow")] +) +def test_header_refuses_a_component_taking_a_formulation_ref(facts: BuildFacts, ref: str) -> None: + formula = formulation("build", "build the app", [], "app", facts) + body = _body([{"type": "library", "bom-ref": ref, "name": "clash"}], [], [formula]) + with pytest.raises(SbomError, match=f"bom-ref values repeat: {ref}"): + with_header(body, uuid.uuid4()) + + +@pytest.mark.parametrize( + ("contributions", "missing"), + [ + pytest.param(Contributions(components=[_partial(type="library", name="zlib")]), "bom-ref", id="component"), + pytest.param( + Contributions( + components=[ + _partial(type="library", name="zlib", **{"bom-ref": "zlib", "components": [{"type": "file"}]}) + ] + ), + "bom-ref, name", + id="nested", + ), + pytest.param(Contributions(tools=[_partial(name="cmake", **{"bom-ref": "tool:cmake"})]), "type", id="tool"), + ], +) +def test_contributions_need_the_fields_the_build_reads(contributions: Contributions, missing: str) -> None: + with pytest.raises(SbomError, match=f"content provider ContentProvider added a component without {missing}: "): + checked_contributions(contributions, ContentProvider()) + + +def test_complete_contributions_pass() -> None: + contributions = Contributions(components=[{"type": "library", "bom-ref": "zlib", "name": "zlib"}]) + assert checked_contributions(contributions, ContentProvider()) is contributions From 83e18c8f8823eacffaa3d012362d8eef0e5c1f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:46:27 -0800 Subject: [PATCH 054/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20resolv?= =?UTF-8?q?e=20the=20content=20provider=20in=20one=20place?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five call sites loaded the provider with two meanings of None, and `content or load(...)` replaced a provider that is falsy. resolve_provider takes an instance, a module:attribute string or nothing, and returns the provider with the build scripts it brings: every project module in the provider class's MRO, so a base class that shapes the document is hashed too, and none for python-sbom's own no-op provider, which a self-build listed as a tool. --- src/python_sbom/_cli/_commands.py | 9 ++-- src/python_sbom/_collect/_content.py | 53 ++++++++++++++------ src/python_sbom/_generate/_archive.py | 10 ++-- src/python_sbom/_generate/_embed.py | 9 ++-- src/python_sbom/hatch/_hook.py | 10 ++-- tests/_collect/test_content.py | 69 ++++++++++++++++++++------- tests/_generate/test_archive.py | 18 ++++--- tests/_generate/test_embed.py | 4 +- 8 files changed, 123 insertions(+), 59 deletions(-) diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py index a3aec7d..7f3ca1c 100644 --- a/src/python_sbom/_cli/_commands.py +++ b/src/python_sbom/_cli/_commands.py @@ -34,13 +34,13 @@ def add_embed(commands: _SubParsersAction[ArgumentParser]) -> None: def _embed(options: Options) -> bool: - from python_sbom._collect._content import load_content_provider # ruff: ignore[import-outside-top-level] # each command loads only what it runs from python_sbom._generate._embed import embed_sbom # ruff: ignore[import-outside-top-level] # each command loads only what it runs root = Path.cwd() if options.root is None else options.root - content = None if options.content_provider is None else load_content_provider(options.content_provider, root) for wheel in options.wheels: - written = embed_sbom(wheel, root, content=content, commit=options.commit, build_env=options.build_env) + written = embed_sbom( + wheel, root, provider=options.content_provider, commit=options.commit, build_env=options.build_env + ) sys.stdout.write(f"wrote {written} into {wheel.name}\n") return True @@ -93,7 +93,6 @@ def add_archive(commands: _SubParsersAction[ArgumentParser]) -> None: def _archive(options: Options) -> bool: - from python_sbom._collect._content import load_content_provider # ruff: ignore[import-outside-top-level] # each command loads only what it runs from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # each command loads only what it runs from python_sbom._formats._json import dumps # ruff: ignore[import-outside-top-level] # each command loads only what it runs from python_sbom._generate._archive import describe_archive # ruff: ignore[import-outside-top-level] # each command loads only what it runs @@ -105,7 +104,7 @@ def _archive(options: Options) -> bool: root, options.archive, facts=facts, - content=None if options.content_provider is None else load_content_provider(options.content_provider, root), + provider=options.content_provider, build_scripts=tuple(PurePosixPath(script) for script in options.build_scripts or []), ) name = f"{options.archive.name}.cdx.json" diff --git a/src/python_sbom/_collect/_content.py b/src/python_sbom/_collect/_content.py index ad67b8d..3558e5a 100644 --- a/src/python_sbom/_collect/_content.py +++ b/src/python_sbom/_collect/_content.py @@ -4,14 +4,30 @@ import pkgutil import sys from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final from python_sbom._errors import SettingsError from python_sbom._model._content import ContentProvider +if TYPE_CHECKING: + from python_sbom._model._inputs import Settings -def load_content_provider(spec: str | None, root: Path) -> ContentProvider: - if spec is None: - return ContentProvider() + +def resolve_provider( + given: ContentProvider | str | None, settings: Settings, root: Path +) -> tuple[ContentProvider, tuple[PurePosixPath, ...]]: + # an instance wins, as a hook subclass builds one; a module:attribute string overrides the option, as the CLI's flag + # does; otherwise the content-provider option, or the provider that adds nothing + if isinstance(given, ContentProvider): + provider = given + elif (spec := settings.content_provider if given is None else given) is None: + provider = ContentProvider() + else: + provider = _load(spec, root) + return provider, _provider_scripts(provider, root) + + +def _load(spec: str, root: Path) -> ContentProvider: # resolve_name also takes a bare module, which is no provider module_name, _, attribute = spec.partition(":") if not (module_name and attribute): @@ -34,23 +50,32 @@ def load_content_provider(spec: str | None, root: Path) -> ContentProvider: return provider -def provider_script(provider: ContentProvider, root: Path) -> PurePosixPath | None: - # a provider defined in the project shapes the document, so the SBOM lists its module among the build tools a - # method's code names the file even for a module hatchling loaded outside the import system, such as a custom hook - # script - cls = type(provider) +def _provider_scripts(provider: ContentProvider, root: Path) -> tuple[PurePosixPath, ...]: + # each class of the provider defined in the project shapes the document, a base class or mixin as much as the class + # itself, so the SBOM lists their modules among the build tools; python-sbom's own ContentProvider adds nothing + found: Final[dict[PurePosixPath, None]] = {} + for cls in type(provider).__mro__: + if cls in {ContentProvider, object}: + continue + if (source := _source(cls)) is None: + continue + resolved, base = Path(source).resolve(), root.resolve() + if resolved.is_relative_to(base): + found[PurePosixPath(resolved.relative_to(base).as_posix())] = None + return tuple(found) + + +def _source(cls: type) -> str | None: + # a method's code names the file even for a module hatchling loaded outside the import system, such as a custom + # hook script methods = ( function.__code__.co_filename for _, function in inspect.getmembers(cls, inspect.isfunction) if function.__qualname__.startswith(f"{cls.__qualname__}.") ) - if (source := next(methods, getattr(sys.modules.get(cls.__module__), "__file__", None))) is None: - return None - resolved, base = Path(source).resolve(), root.resolve() - return PurePosixPath(resolved.relative_to(base).as_posix()) if resolved.is_relative_to(base) else None + return next(methods, getattr(sys.modules.get(cls.__module__), "__file__", None)) __all__ = [ - "load_content_provider", - "provider_script", + "resolve_provider", ] diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index 2b552f7..a02f4ae 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -8,7 +8,7 @@ from packageurl import PackageURL -from python_sbom._collect._content import load_content_provider, provider_script +from python_sbom._collect._content import resolve_provider from python_sbom._collect._settings import settings_from_pyproject from python_sbom._collect._source import copyright_notice, read_bundled_wheels from python_sbom._errors import WheelError @@ -41,14 +41,14 @@ def describe_archive( archive: Path, *, facts: BuildFacts, - content: ContentProvider | None = None, - build_scripts: Sequence[PurePosixPath] = (), + provider: ContentProvider | str | None, + build_scripts: Sequence[PurePosixPath], ) -> Document: # the archive's top-level .dist-info names the project; every other .dist-info marks a wheel bundled unpacked into # its parent directory and every .whl member a bundled wheel; other files belong to the deepest distribution that # holds them, or, at the top level, to the archive settings = settings_from_pyproject(root, {}) - content = content or load_content_provider(settings.content_provider, root) + content, provider_scripts = resolve_provider(provider, settings, root) with tempfile.TemporaryDirectory() as directory: held = _read_archive(root, archive, Path(directory), settings, content) contributions = checked_contributions(content.contributions(held.reader.context), content) @@ -59,7 +59,7 @@ def describe_archive( dependencies=vendored_dependencies(held.components), build_scripts=[ BuildScript(path=path, sha256=hashlib.sha256((root / str(path)).read_bytes()).hexdigest()) - for path in [*build_scripts, *filter(None, [provider_script(content, root)])] + for path in dict.fromkeys([*build_scripts, *provider_scripts]) ], version=held.metadata.version, sbom_file=f"{archive.name}.cdx.json", diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py index 40a14ca..40515ad 100644 --- a/src/python_sbom/_generate/_embed.py +++ b/src/python_sbom/_generate/_embed.py @@ -3,7 +3,7 @@ import dataclasses from typing import TYPE_CHECKING -from python_sbom._collect._content import load_content_provider, provider_script +from python_sbom._collect._content import resolve_provider from python_sbom._collect._facts import collect_facts from python_sbom._collect._revision import commit_revision, default_revision from python_sbom._collect._settings import settings_from_pyproject @@ -27,13 +27,12 @@ def embed_sbom( wheel: Path, root: Path, *, - content: ContentProvider | None, - commit: str | None = None, + provider: ContentProvider | str | None, + commit: str | None, build_env: Sequence[str] | None, ) -> str: settings = settings_from_pyproject(root, {}) - content = content or load_content_provider(settings.content_provider, root) - scripts = tuple(filter(None, [provider_script(content, root)])) + content, scripts = resolve_provider(provider, settings, root) return embed_into( wheel, wheel_request(wheel, root, settings, scripts), diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index 98d9701..5996c1c 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -80,17 +80,17 @@ def finalize( if self.target_name != "wheel" or version == "editable": return # what a wheel needs loads only when one is built, so an sdist build stays light - from python_sbom._collect._content import provider_script # ruff: ignore[import-outside-top-level] # see above + from python_sbom._collect._content import resolve_provider # ruff: ignore[import-outside-top-level] # see above from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # see above from python_sbom._collect._source import wheel_request # ruff: ignore[import-outside-top-level] # see above from python_sbom._generate._embed import embed_into # ruff: ignore[import-outside-top-level] # see above root = Path(self.root) - content = self.content_provider() + content, provider_scripts = resolve_provider(self.content_provider(), self._settings, root) # a custom hook script shapes the document, as a provider defined in the project does, so the SBOM lists both # among the build tools own = [PurePosixPath(self.config.get("path", DEFAULT_BUILD_SCRIPT))] if self.PLUGIN_NAME == "custom" else [] - scripts = tuple(dict.fromkeys([*own, *filter(None, [provider_script(content, root)])])) + scripts = tuple(dict.fromkeys([*own, *provider_scripts])) # hatchling's own fixed time for a reproducible build when SOURCE_DATE_EPOCH is unset, so the SBOM agrees with # the zip entries hatchling wrote facts = collect_facts( @@ -111,9 +111,9 @@ def content_provider(self) -> ContentProvider: :returns: the ``content-provider`` option's provider, or one that adds nothing """ - from python_sbom._collect._content import load_content_provider # ruff: ignore[import-outside-top-level] # see finalize + from python_sbom._collect._content import resolve_provider # ruff: ignore[import-outside-top-level] # see finalize - return load_content_provider(self._settings.content_provider, Path(self.root)) + return resolve_provider(None, self._settings, Path(self.root))[0] @cached_property def _settings(self) -> Settings: diff --git a/tests/_collect/test_content.py b/tests/_collect/test_content.py index 73e7da6..62e779e 100644 --- a/tests/_collect/test_content.py +++ b/tests/_collect/test_content.py @@ -1,13 +1,19 @@ from __future__ import annotations +import dataclasses from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING import pytest +import python_sbom from python_sbom import ContentProvider -from python_sbom._collect._content import load_content_provider, provider_script +from python_sbom._collect._content import resolve_provider from python_sbom._errors import SettingsError +if TYPE_CHECKING: + from python_sbom import Settings + @pytest.fixture def module(tmp_path: Path) -> str: @@ -40,24 +46,55 @@ class Holder: pytest.param("Holder.nested", id="dotted-attribute"), ], ) -def test_load_from_project(tmp_path: Path, module: str, attribute: str) -> None: - provider = load_content_provider(f"{module}:{attribute}", tmp_path) - assert (type(provider).__name__, provider_script(provider, tmp_path)) == ( - "Content", - PurePosixPath(f"tools/{tmp_path.name}.py"), - ) +def test_load_from_project(tmp_path: Path, module: str, attribute: str, settings: Settings) -> None: + provider, scripts = resolve_provider(f"{module}:{attribute}", settings, tmp_path) + assert (type(provider).__name__, scripts) == ("Content", (PurePosixPath(f"tools/{tmp_path.name}.py"),)) + + +def test_option_names_the_provider(tmp_path: Path, module: str, settings: Settings) -> None: + configured = dataclasses.replace(settings, content_provider=f"{module}:Content") + assert type(resolve_provider(None, configured, tmp_path)[0]).__name__ == "Content" + + +def test_given_name_wins_over_option(tmp_path: Path, module: str, settings: Settings) -> None: + configured = dataclasses.replace(settings, content_provider="not_a_module_anywhere:Content") + assert type(resolve_provider(f"{module}:Content", configured, tmp_path)[0]).__name__ == "Content" -def test_provider_outside_project_is_no_tool(tmp_path: Path) -> None: - provider = load_content_provider("python_sbom:ContentProvider", tmp_path) - assert (type(provider), provider_script(provider, tmp_path)) == (ContentProvider, None) +def test_given_instance_wins_over_option(tmp_path: Path, settings: Settings) -> None: + # even one that is falsy, as a provider defining __len__ may be + class Empty(ContentProvider): + def __len__(self) -> int: + return 0 + + given = Empty() + configured = dataclasses.replace(settings, content_provider="not_a_module_anywhere:Content") + assert resolve_provider(given, configured, tmp_path) == (given, ()) + + +def test_no_provider_adds_nothing_and_is_no_tool(settings: Settings) -> None: + # python-sbom's own no-op provider is no build tool, not even when python-sbom builds itself + provider, scripts = resolve_provider(None, settings, Path(python_sbom.__file__).parents[2]) + assert (type(provider), scripts) == (ContentProvider, ()) + + +def test_project_base_classes_are_tools(tmp_path: Path, settings: Settings) -> None: + # a base class in the project shapes the document as much as the provider class itself + (package := tmp_path / f"bases_{tmp_path.name}").mkdir() + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "base.py").write_text( + "from python_sbom import ContentProvider\n\n\nclass Base(ContentProvider):\n pass\n", encoding="utf-8" + ) + (package / "leaf.py").write_text("from .base import Base\n\n\nclass Leaf(Base):\n pass\n", encoding="utf-8") + _, scripts = resolve_provider(f"{package.name}.leaf:Leaf", settings, tmp_path) + assert scripts == (PurePosixPath(f"{package.name}/leaf.py"), PurePosixPath(f"{package.name}/base.py")) -def test_provider_class_outside_project_is_no_tool(tmp_path: Path) -> None: +def test_provider_class_outside_project_is_no_tool(tmp_path: Path, settings: Settings) -> None: class Local(ContentProvider): pass - assert provider_script(Local(), tmp_path) is None + assert resolve_provider(Local(), settings, tmp_path)[1] == () @pytest.mark.parametrize( @@ -71,11 +108,11 @@ class Local(ContentProvider): pytest.param("MODULE:missing", "does not load from .*: .* has no attribute 'missing'", id="missing-attribute"), ], ) -def test_load_refuses(tmp_path: Path, module: str, spec: str, message: str) -> None: +def test_load_refuses(tmp_path: Path, module: str, spec: str, message: str, settings: Settings) -> None: with pytest.raises(SettingsError, match=message): - load_content_provider(spec.replace("MODULE", module), tmp_path) + resolve_provider(spec.replace("MODULE", module), settings, tmp_path) -def test_provider_without_source_is_no_tool(tmp_path: Path) -> None: +def test_provider_without_source_is_no_tool(tmp_path: Path, settings: Settings) -> None: provider = type("Generated", (ContentProvider,), {"__module__": "not_a_loaded_module"})() - assert provider_script(provider, tmp_path) is None + assert resolve_provider(provider, settings, tmp_path)[1] == () diff --git a/tests/_generate/test_archive.py b/tests/_generate/test_archive.py index 823ba5d..ca3e3c1 100644 --- a/tests/_generate/test_archive.py +++ b/tests/_generate/test_archive.py @@ -64,7 +64,7 @@ def archive(tmp_path: Path, write_wheel: WriteWheel) -> Path: def document(project: Path, archive: Path) -> Document: facts = collect_facts(project, revision_reader(_COMMIT)) return describe_archive( - project, archive, facts=facts, content=Bundles("loaded.json"), build_scripts=(PurePosixPath("build_app.py"),) + project, archive, facts=facts, provider=Bundles("loaded.json"), build_scripts=(PurePosixPath("build_app.py"),) ) @@ -128,7 +128,7 @@ def test_archive_of_dirty_checkout(project: Path, archive: Path) -> None: facts = dataclasses.replace( collect_facts(project, revision_reader(_COMMIT)), revision=Revision(commit=_COMMIT, dirty=True) ) - root = describe_archive(project, archive, facts=facts)["metadata"]["component"] + root = describe_archive(project, archive, facts=facts, provider=None, build_scripts=())["metadata"]["component"] assert (root.get("properties", [])[-1], root.get("externalReferences", [])[-1]["comment"]) == ( {"name": "app:vcs-dirty", "value": "true"}, "source revision, with uncommitted changes", @@ -201,7 +201,9 @@ def test_archive_contributions(document: Document) -> None: def test_archive_defaults(tmp_path: Path, archive: Path) -> None: (tmp_path / "pyproject.toml").write_text("[project]\nname = 'app'\n", encoding="utf-8") - document = describe_archive(tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None))) + document = describe_archive( + tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None)), provider=None, build_scripts=() + ) assert ("supplier" in document["metadata"], document["metadata"]["component"].get("properties")) == (False, []) @@ -212,9 +214,9 @@ def test_archive_content_provider_option(tmp_path: Path, archive: Path) -> None: (tmp_path / f"{tmp_path.name}.py").write_text( "from python_sbom import ContentProvider\n\n\nclass Provider(ContentProvider):\n pass\n", encoding="utf-8" ) - tools = describe_archive(tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None)))["metadata"][ - "tools" - ]["components"] + tools = describe_archive( + tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None)), provider=None, build_scripts=() + )["metadata"]["tools"]["components"] assert tools[0]["name"] == f"{tmp_path.name}.py" @@ -227,4 +229,6 @@ def test_archive_without_project_distribution(tmp_path: Path) -> None: with pytest.raises( WheelError, match=r"bare\.pyz holds 0 top-level \.dist-info/METADATA, not the one naming the project" ): - describe_archive(tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None))) + describe_archive( + tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None)), provider=None, build_scripts=() + ) diff --git a/tests/_generate/test_embed.py b/tests/_generate/test_embed.py index 1db1afa..3386cf5 100644 --- a/tests/_generate/test_embed.py +++ b/tests/_generate/test_embed.py @@ -34,7 +34,7 @@ def test_bundled_wheel_named_by_its_path_inside_the_wheel(tmp_path: Path, write_ "pkg-1.0.dist-info/METADATA": b"Name: pkg\nVersion: 1.0\n", }, ) - written = embed_sbom(wheel, root, content=Table("table.txt"), build_env=[str(root)]) + written = embed_sbom(wheel, root, provider=Table("table.txt"), commit=None, build_env=[str(root)]) (pip,) = json.loads(zipfile.ZipFile(wheel).read(written))["components"] assert (pip["bom-ref"], pip["properties"][:2]) == ( "bundled:pkg/wheels/pip-1.0-py3-none-any.whl", @@ -112,7 +112,7 @@ def test_embed_output_is_pinned( }, compression=0, ) - written = embed_sbom(wheel, root, content=None, commit="a" * 40, build_env=[str(site)]) + written = embed_sbom(wheel, root, provider=None, commit="a" * 40, build_env=[str(site)]) with zipfile.ZipFile(wheel) as archive: sbom = archive.read(written) members = [ From 94ffd78626ec323d9057dfbe25a027b12f3e1731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:46:01 -0800 Subject: [PATCH 055/115] =?UTF-8?q?=E2=9C=85=20test:=20check=20member=20ty?= =?UTF-8?q?pes=20in=20the=20API=20closure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The closure test read only plain functions, so a public class could expose a private type through a property, cached_property, classmethod or staticmethod unnoticed; it now reads each python_sbom class dictionary along the MRO. It also stops at the first private python_sbom type, so a leak whose own hints do not resolve is reported instead of a NameError. --- tests/test_public_api.py | 83 ++++++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 54ab95f..1b9334a 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -1,6 +1,7 @@ from __future__ import annotations import collections.abc +import functools import importlib import inspect import pathlib @@ -12,6 +13,14 @@ import python_sbom import python_sbom.hatch +from python_sbom import Settings +from python_sbom._model._inputs import RecordRow +from python_sbom._model._metadata import CoreMetadata + +if typing.TYPE_CHECKING: + from collections.abc import Callable, Iterator + +_Hint = object # simplint: ignore[SMP602] # hints are arbitrary typing objects _PUBLIC: Final[set[str]] = { "BundledWheelInfo", @@ -45,23 +54,47 @@ def test_hatch_api() -> None: assert set(python_sbom.hatch.__all__) == {"SbomBuildHook"} -def reached(start: object) -> set[object]: # simplint: ignore[SMP602] # hints are arbitrary typing objects - # every python_sbom type a hint of a public name leads to, through fields, parameters and nested hints - found: set[object] = set() # simplint: ignore[SMP602] # see above - pending: list[object] = [start] # simplint: ignore[SMP602] # see above +def reached(start: _Hint) -> set[_Hint]: + # every python_sbom type a hint of a public name leads to, through fields, members and nested hints; a private type + # ends the walk there, so the leak is what gets reported even when its own hints do not resolve + found: set[_Hint] = set() + pending: list[_Hint] = [start] + public = exported() while pending: if (item := pending.pop()) in found: continue found.add(item) - if typing.is_typeddict(item) or inspect.isclass(item): + if _ours(item) and item not in public: + continue + if inspect.isclass(item): pending.extend(typing.get_type_hints(item, globalns=_names(), include_extras=True).values()) - for _, member in inspect.getmembers(item, inspect.isfunction): - if member.__module__.startswith("python_sbom"): - pending.extend(typing.get_type_hints(member, globalns=_names(), include_extras=True).values()) + for member in _members(item): + pending.extend(typing.get_type_hints(member, globalns=_names(), include_extras=True).values()) pending.extend(typing.get_args(item)) if (origin := typing.get_origin(item)) is not None: pending.append(origin) - return {item for item in found if getattr(item, "__module__", "").startswith("python_sbom")} + return {item for item in found if _ours(item)} + + +def _members(cls: type) -> Iterator[_Hint]: + # read the class dictionaries, since attribute access hides a property's getter and unwraps class and static methods + for owner in inspect.getmro(cls): + if not _ours(owner): + continue + for value in owner.__dict__.values(): + match value: + case property(): + yield value.fget + case classmethod() | staticmethod(): + yield value.__func__ + case functools.cached_property(): + yield value.func + case _ if inspect.isfunction(value): + yield value + + +def _ours(item: _Hint) -> bool: + return getattr(item, "__module__", "").startswith("python_sbom") @pytest.mark.parametrize("name", sorted(_PUBLIC - {"__version__"}), ids=str) @@ -73,11 +106,39 @@ def test_hook_only_reaches_public_types() -> None: assert reached(python_sbom.hatch.SbomBuildHook) - exported() == set() -def exported() -> set[object]: # simplint: ignore[SMP602] # see above +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(property, id="property"), + pytest.param(functools.cached_property, id="cached_property"), + pytest.param(classmethod, id="classmethod"), + pytest.param(staticmethod, id="staticmethod"), + ], +) +def test_closure_reports_private_member_types( + monkeypatch: pytest.MonkeyPatch, wrap: Callable[[Callable[[Settings], RecordRow]], _Hint] +) -> None: + def leak(_: Settings) -> RecordRow: + raise NotImplementedError + + monkeypatch.setattr(Settings, "leak", wrap(leak), raising=False) + assert reached(Settings) - exported() == {RecordRow} + + +def test_closure_reports_private_type_with_unresolved_hints(monkeypatch: pytest.MonkeyPatch) -> None: + # CoreMetadata names packaging's RawMetadata, which no python_sbom module imports at runtime + def leak(_: Settings) -> CoreMetadata: + raise NotImplementedError + + monkeypatch.setattr(Settings, "leak", property(leak), raising=False) + assert reached(Settings) - exported() == {CoreMetadata} + + +def exported() -> set[_Hint]: return {getattr(python_sbom, name) for name in _PUBLIC} | {python_sbom.hatch.SbomBuildHook} -def _names() -> dict[str, object]: # simplint: ignore[SMP602] # see above +def _names() -> dict[str, _Hint]: # the annotations name types the modules import only for type checking, so every module's names resolve them modules = [ pathlib, From 06c6f74bfa6f882a082829700bcaa646d9aaa725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:50:18 -0800 Subject: [PATCH 056/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20read=20archives?= =?UTF-8?q?=20from=20the=20zip,=20not=20the=20disk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit describe_archive unpacked the archive and then joined raw member names onto the temporary directory, so a member named /etc/hostname or ../../x had the host's file hashed, and a .whl named so was read from the host. Archives are now read from the zip by member name alone, in _collect, with no unpacking: digests come from the open member, and ProviderContext reads through a callable _collect builds on open_archive, with its duplicate and corrupt member checks, instead of a plain ZipFile in the model. Bundled wheels stream from disk and verify four at a time, so their memory stays bounded. RECORD rows are read in one place before installer's check, which raised a TypeError on a shake_128 row; that row is now a WheelError. --- bench/_cases.py | 9 +- src/python_sbom/_collect/_archive.py | 94 ++++++++++++++++++ src/python_sbom/_collect/_facts.py | 17 +--- src/python_sbom/_collect/_source.py | 50 ++++++---- src/python_sbom/_generate/_archive.py | 133 +++++++------------------- src/python_sbom/_generate/_embed.py | 3 +- src/python_sbom/_model/_components.py | 12 +-- src/python_sbom/_model/_content.py | 20 ++-- src/python_sbom/_model/_inputs.py | 13 +++ src/python_sbom/_wheel/_record.py | 22 ++++- tests/_collect/test_archive.py | 49 ++++++++++ tests/_collect/test_source.py | 52 +++++++--- tests/_model/test_components.py | 12 --- tests/_model/test_content.py | 19 +--- tests/conftest.py | 5 +- tests/test_public_api.py | 4 +- 16 files changed, 308 insertions(+), 206 deletions(-) create mode 100644 src/python_sbom/_collect/_archive.py create mode 100644 tests/_collect/test_archive.py diff --git a/bench/_cases.py b/bench/_cases.py index 06314cc..4fd4eb0 100644 --- a/bench/_cases.py +++ b/bench/_cases.py @@ -143,13 +143,18 @@ def run() -> None: def _bundled(key: str) -> Callable[[dict[str, str], Path], Step]: def make(fixtures: dict[str, str], scratch: Path) -> Step: # ruff: ignore[unused-function-argument] # one signature - from python_sbom._collect._source import read_bundled_wheel # ruff: ignore[import-outside-top-level] # see _validate + from python_sbom._collect import _source # ruff: ignore[import-outside-top-level] # see _validate wheel = Path(fixtures[key]) data = wheel.read_bytes() + # revisions before the batch reader take one wheel's bytes; later ones a list and a way to open each + single = getattr(_source, "read_bundled_wheel", None) def run() -> None: - read_bundled_wheel(data, wheel.name) + if single is not None: + single(data, wheel.name) + else: + _source.read_bundled_wheels([wheel.name], lambda _name: io.BytesIO(data)) return Step(run=run) diff --git a/src/python_sbom/_collect/_archive.py b/src/python_sbom/_collect/_archive.py new file mode 100644 index 0000000..427521d --- /dev/null +++ b/src/python_sbom/_collect/_archive.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from io import BytesIO +from typing import TYPE_CHECKING, Final + +from python_sbom._errors import WheelError +from python_sbom._model._inputs import ArchiveTree, RecordRow +from python_sbom._model._metadata import parse_metadata +from python_sbom._wheel._archive import member_digest, open_archive, read_member + +from ._source import read_bundled_wheels + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path, PurePosixPath + + from python_sbom._model._metadata import CoreMetadata + +# a bundled wheel is read whole, to hash it and open it as a zip, so its size is capped as metadata's is +_WHEEL_LIMIT: Final[int] = 512 * 1024 * 1024 + + +def read_archive(archive: Path) -> ArchiveTree: + # every member is read from the zip by its name, never from a file system path, so a name such as /etc/hostname or + # ../outside names a member, not a file of the host + where: Final[str] = archive.name + with open_archive(archive, where) as handle: + members: Final[list[str]] = sorted( + info.filename for info in handle.infolist() if not info.is_dir() and info.filename != f"{where}.cdx.json" + ) + metadata: Final[dict[str, CoreMetadata]] = { + _directory(name): parse_metadata(read_member(handle, name, where).decode("utf-8")) + for name in members + if name.endswith(".dist-info/METADATA") + } + if (top := sum(name.count("/") == 1 and name.endswith(".dist-info/METADATA") for name in members)) != 1: + msg = f"{where} holds {top} top-level .dist-info/METADATA, not the one naming the project" + raise WheelError(msg) + files: Final[dict[str, RecordRow]] = { + name: RecordRow( + path=name, + algorithm="SHA-256", + digest=member_digest(handle, name, "sha256"), + size=str(handle.getinfo(name).file_size), + ) + for name in members + if not name.endswith(".whl") + } + owned: Final[dict[str, list[str]]] = {prefix: [] for prefix in metadata} + loose: Final[list[str]] = [] + for name in members: + # the deepest distribution directory that holds the file; the project's own holds every other subdirectory + if owner := max((prefix for prefix in metadata if prefix and name.startswith(prefix)), key=len, default=""): + owned[owner].append(name) + elif "/" in name: + owned[""].append(name) + else: + loose.append(name) + # in the order the components list them, so the first broken wheel is the one reported + wheels: Final[list[str]] = [name for prefix in sorted(metadata) for name in owned[prefix] if name.endswith(".whl")] + return ArchiveTree( + metadata=metadata, + owned=owned, + loose=loose, + files=files, + bundled=dict(zip(wheels, read_bundled_wheels(wheels, lambda name: BytesIO(_read(archive, name))), strict=True)), + ) + + +def _directory(name: str) -> str: + # each distribution by the directory that holds its .dist-info, the project's own at the top level + return f"{name.rsplit('/', 2)[0]}/" if name.count("/") > 1 else "" + + +def _read(archive: Path, name: str) -> bytes: + with open_archive(archive, archive.name) as handle: + return read_member(handle, name, archive.name, _WHEEL_LIMIT) + + +def member_reader(artifact: Path) -> Callable[[PurePosixPath], bytes]: + # what a content provider reads: a member of the built wheel or archive, through the same checks as every read + def read(path: PurePosixPath) -> bytes: + with open_archive(artifact, str(artifact)) as handle: + if str(path) not in handle.namelist(): + raise KeyError(str(path)) + return read_member(handle, str(path), str(artifact)) + + return read + + +__all__ = [ + "member_reader", + "read_archive", +] diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index e2809b4..b8856fa 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -7,10 +7,9 @@ import time import warnings from importlib.metadata import distributions -from itertools import starmap from typing import TYPE_CHECKING, Final -from installer.records import InvalidRecordEntry, RecordEntry, parse_record_file +from installer.records import InvalidRecordEntry, parse_record_file from installer.utils import parse_metadata_file from packaging.direct_url import DirectUrl, DirectUrlValidationError from packaging.requirements import InvalidRequirement, Requirement @@ -19,7 +18,7 @@ from python_sbom._errors import SettingsError from python_sbom._model._inputs import BuildFacts, InstalledDistribution, Interpreter, RecordRow from python_sbom._model._metadata import parse_metadata -from python_sbom._wheel._record import decode_digest +from python_sbom._wheel._record import record_rows if TYPE_CHECKING: from collections.abc import Container, Sequence @@ -83,20 +82,12 @@ def _recorded_files(distribution: Distribution) -> list[RecordRow]: return [] # RECORD read once and as written: Distribution.files would stat every row to drop files missing on disk, and build # a path object for each, while the SBOM lists what the install recorded + rows: Final = parse_record_file((distribution.read_text("RECORD") or "").splitlines()) try: - entries: Final[list[RecordEntry]] = list( - starmap(RecordEntry.from_elements, parse_record_file((distribution.read_text("RECORD") or "").splitlines())) - ) + return record_rows((row for row in rows if _describes_distribution(row[0])), distribution.name) except InvalidRecordEntry as error: warnings.warn(f"listed no files of {distribution.name}, its RECORD is malformed: {error}", stacklevel=2) return [] - rows: Final[list[RecordRow]] = [] - for entry in entries: - if entry.hash_ is None or not _describes_distribution(entry.path): - continue - algorithm, digest = decode_digest(str(entry.hash_), entry.path) - rows.append(RecordRow(path=entry.path, algorithm=algorithm, digest=digest, size=str(entry.size))) - return rows def _editable(distribution: Distribution) -> bool: diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index 27a6170..46e6c82 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -3,12 +3,10 @@ import hashlib from collections import Counter from concurrent.futures import ThreadPoolExecutor -from io import BytesIO -from itertools import starmap from pathlib import PurePath from typing import TYPE_CHECKING, Final -from installer.records import InvalidRecordEntry, RecordEntry, parse_record_file +from installer.records import InvalidRecordEntry, parse_record_file from installer.sources import WheelFile from packaging.requirements import InvalidRequirement, Requirement @@ -16,14 +14,21 @@ from python_sbom._model._inputs import BuildScript, BundledWheel, RecordRow, SbomRequest, SourceTree from python_sbom._model._metadata import CoreMetadata, parse_metadata from python_sbom._wheel._archive import open_archive, read_member, read_wheel_metadata -from python_sbom._wheel._record import decode_digest +from python_sbom._wheel._record import record_rows if TYPE_CHECKING: import zipfile + from collections.abc import Callable from pathlib import Path, PurePosixPath + from typing import IO from python_sbom._model._inputs import Settings +# a bundled wheel is hashed in chunks of this size +_CHUNK: Final[int] = 1024 * 1024 +# wheels verified at once, each perhaps in memory whole +_WORKERS: Final[int] = 4 + def read_source_tree(request: SbomRequest) -> SourceTree: root = request.root @@ -35,12 +40,12 @@ def read_source_tree(request: SbomRequest) -> SourceTree: return SourceTree( copyright=copyright_notice(root, request.metadata.values("license_files")), bundled=read_bundled_wheels( - root, sorted({ wheel.relative_to(root).as_posix() for pattern in request.settings.bundled_wheels for wheel in root.glob(pattern) }), + lambda path: (root / path).open("rb"), ), # str, so a Windows drive in an absolute path survives the join with the native root build_scripts=[ @@ -60,16 +65,24 @@ def copyright_notice(root: Path, license_files: list[str]) -> str | None: return None -def read_bundled_wheels(root: Path, paths: list[str]) -> list[BundledWheel]: - # inflating and hashing release the GIL, so the wheels verify side by side; results and the first error come in the - # order given - with ThreadPoolExecutor() as pool: - return list(pool.map(lambda path: read_bundled_wheel((root / path).read_bytes(), path), paths)) +def read_bundled_wheels(paths: list[str], open_wheel: Callable[[str], IO[bytes]]) -> list[BundledWheel]: + # inflating and hashing release the GIL, so the wheels verify side by side, a few at a time, as each may be held in + # memory whole; results and the first error come in the order given + def read(path: str) -> BundledWheel: + with open_wheel(path) as stream: + return _read_bundled_wheel(stream, path) + + with ThreadPoolExecutor(max_workers=_WORKERS) as pool: + return list(pool.map(read, paths)) -def read_bundled_wheel(data: bytes, path: str) -> BundledWheel: +def _read_bundled_wheel(stream: IO[bytes], path: str) -> BundledWheel: where: Final[str] = f"bundled wheel {path}" - with open_archive(BytesIO(data), where) as archive: + digest: Final = hashlib.sha256() + while chunk := stream.read(_CHUNK): + digest.update(chunk) + stream.seek(0) + with open_archive(stream, where) as archive: # installer takes the project name from the archive's file name, which an archive read from memory lacks archive.filename = path members: Final[list[str]] = sorted(name for name in archive.namelist() if not name.endswith("/")) @@ -87,7 +100,7 @@ def read_bundled_wheel(data: bytes, path: str) -> BundledWheel: ] return BundledWheel( path=path, - sha256=hashlib.sha256(data).hexdigest(), + sha256=digest.hexdigest(), metadata_name=f"{dist_info}/METADATA", metadata=metadata, files=files, @@ -108,6 +121,9 @@ def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> record: Final[str] = read_member(archive, f"{dist_info}/RECORD", where).decode("utf-8") try: rows: Final[list[tuple[str, str, str]]] = list(parse_record_file(record.splitlines())) + # read before installer's check, which takes any hashlib name and fails on one needing a length, such as + # shake_128, with a TypeError + found: Final[list[RecordRow]] = record_rows(rows, where) except InvalidRecordEntry as error: msg = f"{where}: RECORD does not parse: {error}" raise WheelError(msg) from error @@ -135,13 +151,6 @@ def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> prefix: Final[str] = f"In {archive.filename}, " msg = f"{where}: RECORD disagrees with the wheel: {'; '.join(issue.removeprefix(prefix) for issue in issues)}" raise WheelError(msg) from error - found: Final[list[RecordRow]] = [] - for entry in starmap(RecordEntry.from_elements, rows): - # the row of RECORD itself - if entry.hash_ is None: - continue - algorithm, digest = decode_digest(str(entry.hash_), f"{where}: RECORD row for {entry.path}") - found.append(RecordRow(path=entry.path, algorithm=algorithm, digest=digest, size=str(entry.size))) return found @@ -180,7 +189,6 @@ def wheel_request(wheel: Path, root: Path, settings: Settings, build_scripts: tu __all__ = [ "copyright_notice", - "read_bundled_wheel", "read_bundled_wheels", "read_source_tree", "wheel_request", diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index a02f4ae..e4da50e 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -1,38 +1,35 @@ from __future__ import annotations import hashlib -import tempfile from dataclasses import dataclass -from pathlib import Path, PurePosixPath +from pathlib import PurePosixPath from typing import TYPE_CHECKING, Final from packageurl import PackageURL +from python_sbom._collect._archive import member_reader, read_archive from python_sbom._collect._content import resolve_provider from python_sbom._collect._settings import settings_from_pyproject -from python_sbom._collect._source import copyright_notice, read_bundled_wheels -from python_sbom._errors import WheelError +from python_sbom._collect._source import copyright_notice from python_sbom._model._bom import assemble from python_sbom._model._components import ( bundled_wheel_component, component_from_metadata, - file_component_from_bytes, + file_component, vendored_dependencies, ) from python_sbom._model._content import ProviderContext from python_sbom._model._document import checked_contributions from python_sbom._model._identity import project_identity from python_sbom._model._inputs import Artifact, BuildScript -from python_sbom._model._metadata import parse_metadata -from python_sbom._wheel._archive import open_archive if TYPE_CHECKING: from collections.abc import Sequence + from pathlib import Path from python_sbom._model._content import ContentProvider from python_sbom._model._identity import ProjectIdentity - from python_sbom._model._inputs import BuildFacts, BundledWheel, Settings - from python_sbom._model._metadata import CoreMetadata + from python_sbom._model._inputs import ArchiveTree, BuildFacts from python_sbom._model._types import Component, Document @@ -49,19 +46,34 @@ def describe_archive( # holds them, or, at the top level, to the archive settings = settings_from_pyproject(root, {}) content, provider_scripts = resolve_provider(provider, settings, root) - with tempfile.TemporaryDirectory() as directory: - held = _read_archive(root, archive, Path(directory), settings, content) - contributions = checked_contributions(content.contributions(held.reader.context), content) - license_files: Final[list[str]] = held.metadata.values("license_files") + tree = read_archive(archive) + project = project_identity(tree.metadata[""], settings) + reader = _Reader( + tree=tree, + project=project, + content=content, + context=ProviderContext(root=root, settings=settings, identity=project, read=member_reader(archive)), + ) + version: Final[str] = tree.metadata[""].version + ref: Final[str] = PackageURL(type="generic", name=archive.name, version=version).to_string() + components: Final[list[Component]] = [reader.distribution(prefix) for prefix in sorted(tree.metadata)] + license_files: Final[list[str]] = tree.metadata[""].values("license_files") artifact = Artifact( - root=held.root, - components=held.components, - dependencies=vendored_dependencies(held.components), + root={ + "type": "application", + "bom-ref": ref, + "name": archive.name, + "version": version, + "purl": ref, + "components": [file_component(ref, tree.files[name]) for name in tree.loose], + }, + components=components, + dependencies=vendored_dependencies(components), build_scripts=[ BuildScript(path=path, sha256=hashlib.sha256((root / str(path)).read_bytes()).hexdigest()) for path in dict.fromkeys([*build_scripts, *provider_scripts]) ], - version=held.metadata.version, + version=version, sbom_file=f"{archive.name}.cdx.json", scope=( "Components are the distributions the archive holds, as wheels or unpacked, each with its files; the" @@ -71,76 +83,19 @@ def describe_archive( license_file=next(iter(license_files), None), copyright=copyright_notice(root, license_files), ) - return assemble(artifact, held.reader.project, settings, facts, contributions) - - -def _read_archive(root: Path, archive: Path, unpacked: Path, settings: Settings, content: ContentProvider) -> _Held: - # unpacked, so a content provider reads the archive's files the way it reads the project's - with open_archive(archive, archive.name) as handle: - members = sorted( - name for name in handle.namelist() if not name.endswith("/") and name != f"{archive.name}.cdx.json" - ) - handle.extractall(unpacked) - # each distribution by the directory that holds its .dist-info, the project's own at the top level - metadata = { - f"{name.rsplit('/', 2)[0]}/" if name.count("/") > 1 else "": parse_metadata( - (unpacked / name).read_text(encoding="utf-8") - ) - for name in members - if name.endswith(".dist-info/METADATA") - } - if (top := sum(name.count("/") == 1 and name.endswith(".dist-info/METADATA") for name in members)) != 1: - msg = f"{archive.name} holds {top} top-level .dist-info/METADATA, not the one naming the project" - raise WheelError(msg) - owned: Final[dict[str, list[str]]] = {prefix: [] for prefix in metadata} - loose: Final[list[str]] = [] - for name in members: - # the deepest distribution directory that holds the file; the project's own holds every other subdirectory - if owner := max((prefix for prefix in metadata if prefix and name.startswith(prefix)), key=len, default=""): - owned[owner].append(name) - elif "/" in name: - owned[""].append(name) - else: - loose.append(name) - project = project_identity(metadata[""], settings) - wheels: Final[list[str]] = [name for prefix in sorted(metadata) for name in owned[prefix] if name.endswith(".whl")] - reader = _Reader( - unpacked=unpacked, - bundled=dict(zip(wheels, read_bundled_wheels(unpacked, wheels), strict=True)), - project=project, - settings=settings, - content=content, - context=ProviderContext(root=root, settings=settings, identity=project, artifact=unpacked), - ) - version = metadata[""].version - ref = PackageURL(type="generic", name=archive.name, version=version).to_string() - return _Held( - reader=reader, - metadata=metadata[""], - root={ - "type": "application", - "bom-ref": ref, - "name": archive.name, - "version": version, - "purl": ref, - "components": [reader.file(ref, name) for name in loose], - }, - components=[reader.distribution(prefix, metadata[prefix], owned[prefix]) for prefix in sorted(metadata)], - ) + contributions = checked_contributions(content.contributions(reader.context), content) + return assemble(artifact, project, settings, facts, contributions) @dataclass(frozen=True, slots=True, kw_only=True) class _Reader: - unpacked: Path - # read ahead, in the order the components list them - bundled: dict[str, BundledWheel] + tree: ArchiveTree project: ProjectIdentity - settings: Settings content: ContentProvider context: ProviderContext - def distribution(self, prefix: str, metadata: CoreMetadata, names: list[str]) -> Component: - component = component_from_metadata(metadata) + def distribution(self, prefix: str) -> Component: + component = component_from_metadata(self.tree.metadata[prefix]) if prefix: # a wheel bundled unpacked: the directory tells copies of one release apart, as the path does for a .whl directory = PurePosixPath(prefix.rstrip("/")) @@ -154,31 +109,17 @@ def distribution(self, prefix: str, metadata: CoreMetadata, names: list[str]) -> *component["properties"], ] component["components"] = [ - self.wheel(name) if name.endswith(".whl") else self.file(component["bom-ref"], name) for name in names + self.wheel(name) if name.endswith(".whl") else file_component(component["bom-ref"], self.tree.files[name]) + for name in self.tree.owned[prefix] ] return component def wheel(self, name: str) -> Component: info = self.content.bundled_wheel(self.context, PurePosixPath(name)) return bundled_wheel_component( - self.bundled[name], - self.project.property_prefix, - info.properties, - info.supplier, + self.tree.bundled[name], self.project.property_prefix, info.properties, info.supplier ) - def file(self, parent: str, name: str) -> Component: - return file_component_from_bytes(parent, name, (self.unpacked / name).read_bytes()) - - -@dataclass(frozen=True, slots=True, kw_only=True) -class _Held: - reader: _Reader - # the project's own, from the archive's top-level .dist-info - metadata: CoreMetadata - root: Component - components: list[Component] - __all__ = [ "describe_archive", diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py index 40515ad..3e7f5ae 100644 --- a/src/python_sbom/_generate/_embed.py +++ b/src/python_sbom/_generate/_embed.py @@ -3,6 +3,7 @@ import dataclasses from typing import TYPE_CHECKING +from python_sbom._collect._archive import member_reader from python_sbom._collect._content import resolve_provider from python_sbom._collect._facts import collect_facts from python_sbom._collect._revision import commit_revision, default_revision @@ -54,7 +55,7 @@ def embed_into(wheel: Path, request: SbomRequest, content: ContentProvider, fact ], ) project = project_identity(request.metadata, request.settings) - context = ProviderContext(root=request.root, settings=request.settings, identity=project, artifact=wheel) + context = ProviderContext(root=request.root, settings=request.settings, identity=project, read=member_reader(wheel)) sbom = build_sbom(request, source, facts, content, context) return write_sbom(wheel, sbom.file_name, sbom.content, facts.epoch) diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index ae8e465..371520a 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -1,13 +1,11 @@ from __future__ import annotations -import hashlib import re from email.utils import getaddresses from itertools import starmap from typing import TYPE_CHECKING, Final from urllib.parse import urlsplit -from ._inputs import RecordRow from ._licenses import licenses from ._names import property_prefix, purl, pypi_url, reference_type, tool_ref from ._types import Dependency @@ -15,7 +13,7 @@ if TYPE_CHECKING: from pathlib import PurePosixPath - from ._inputs import BundledWheel + from ._inputs import BundledWheel, RecordRow from ._metadata import CoreMetadata from ._types import Component, Contact, ExternalReference, Identity, OrganizationalEntity, Property @@ -149,13 +147,6 @@ def vendored_dependencies(components: list[Component]) -> list[Dependency]: return found -def file_component_from_bytes(parent_ref: str, name: str, content: bytes) -> Component: - return file_component( - parent_ref, - RecordRow(path=name, algorithm="SHA-256", digest=hashlib.sha256(content).hexdigest(), size=str(len(content))), - ) - - def script_tool(path: PurePosixPath, sha256: str, version: str, link: str | None) -> Component: tool: Final[Component] = { "type": "application", @@ -173,7 +164,6 @@ def script_tool(path: PurePosixPath, sha256: str, version: str, link: str | None "bundled_wheel_component", "component_from_metadata", "file_component", - "file_component_from_bytes", "script_tool", "vendored_dependencies", ] diff --git a/src/python_sbom/_model/_content.py b/src/python_sbom/_model/_content.py index d712d97..4be1e4e 100644 --- a/src/python_sbom/_model/_content.py +++ b/src/python_sbom/_model/_content.py @@ -1,11 +1,11 @@ from __future__ import annotations -import zipfile from typing import TYPE_CHECKING from ._inputs import BundledWheelInfo, Contributions if TYPE_CHECKING: + from collections.abc import Callable from pathlib import Path, PurePosixPath from ._identity import ProjectIdentity @@ -19,18 +19,17 @@ class ProviderContext: :param root: the project directory :param settings: the project's options :param identity: how the project's SBOMs name it, including the prefix of its properties - :param artifact: the built wheel, or the directory an archive is unpacked into, which :meth:`read` reads - :ivar root: the project directory - :ivar settings: the project's options - :ivar identity: how the project's SBOMs name it, including the prefix of its properties + :param read: reads a member of the artifact being described, for :meth:`read` """ - def __init__(self, *, root: Path, settings: Settings, identity: ProjectIdentity, artifact: Path) -> None: + def __init__( + self, *, root: Path, settings: Settings, identity: ProjectIdentity, read: Callable[[PurePosixPath], bytes] + ) -> None: self.root = root self.settings = settings self.identity = identity - self._artifact = artifact + self._read = read def read(self, path: PurePosixPath) -> bytes: """ @@ -41,12 +40,7 @@ def read(self, path: PurePosixPath) -> bytes: :raises KeyError: when the artifact holds no such file """ - if self._artifact.is_dir(): - if not (target := self._artifact / str(path)).is_file(): - raise KeyError(str(path)) - return target.read_bytes() - with zipfile.ZipFile(self._artifact) as archive: - return archive.read(str(path)) + return self._read(path) class ContentProvider: diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py index d62decf..0fb4d76 100644 --- a/src/python_sbom/_model/_inputs.py +++ b/src/python_sbom/_model/_inputs.py @@ -94,6 +94,18 @@ class BuildScript: sha256: str +@dataclass(frozen=True, slots=True, kw_only=True) +class ArchiveTree: + # what an archive holds, read from the zip without unpacking it: each distribution's metadata and members by the + # directory that holds its .dist-info, "" for the project's own, the archive's own top-level files, the digest of + # every member but the bundled wheels, and those wheels + metadata: dict[str, CoreMetadata] + owned: dict[str, list[str]] + loose: list[str] + files: dict[str, RecordRow] + bundled: dict[str, BundledWheel] + + @dataclass(frozen=True, slots=True, kw_only=True) class Artifact: # what one artifact, a wheel or an archive, puts in its SBOM; the provenance every artifact of the project shares is @@ -160,6 +172,7 @@ class Contributions: __all__ = [ + "ArchiveTree", "Artifact", "BuildFacts", "BuildScript", diff --git a/src/python_sbom/_wheel/_record.py b/src/python_sbom/_wheel/_record.py index a7af8da..cfbdf18 100644 --- a/src/python_sbom/_wheel/_record.py +++ b/src/python_sbom/_wheel/_record.py @@ -2,9 +2,16 @@ import base64 import binascii -from typing import Final +from itertools import starmap +from typing import TYPE_CHECKING, Final + +from installer.records import RecordEntry from python_sbom._errors import WheelError +from python_sbom._model._inputs import RecordRow + +if TYPE_CHECKING: + from collections.abc import Iterable # the algorithms a wheel RECORD may name, as CycloneDX spells them _ALGORITHMS: Final[dict[str, str]] = { @@ -16,6 +23,18 @@ } +def record_rows(rows: Iterable[tuple[str, str, str]], where: str) -> list[RecordRow]: + # the rows with a digest, as the SBOM lists them: RECORD's own row has none, nor has a file an installer left + # unhashed; installer's InvalidRecordEntry passes to the caller, which knows what a malformed row means there + found: Final[list[RecordRow]] = [] + for entry in starmap(RecordEntry.from_elements, rows): + if entry.hash_ is None: + continue + algorithm, digest = decode_digest(str(entry.hash_), f"{where}: RECORD row for {entry.path}") + found.append(RecordRow(path=entry.path, algorithm=algorithm, digest=digest, size=str(entry.size))) + return found + + def decode_digest(digest: str, where: str) -> tuple[str, str]: # installer checks RECORD digests as urlsafe base64 under any hashlib name, while CycloneDX takes hex under its own # names for a few algorithms @@ -33,4 +52,5 @@ def decode_digest(digest: str, where: str) -> tuple[str, str]: __all__ = [ "decode_digest", + "record_rows", ] diff --git a/tests/_collect/test_archive.py b/tests/_collect/test_archive.py new file mode 100644 index 0000000..3d29cd9 --- /dev/null +++ b/tests/_collect/test_archive.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import hashlib +import zipfile +from pathlib import Path, PurePosixPath + +import pytest + +from python_sbom._collect._archive import member_reader, read_archive +from python_sbom._errors import WheelError + + +def test_member_reader_reads_a_member(tmp_path: Path) -> None: + with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: + archive.writestr("embed/table.py", "x = 1\n") + assert member_reader(wheel)(PurePosixPath("embed/table.py")) == b"x = 1\n" + + +def test_member_reader_misses(tmp_path: Path) -> None: + with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: + archive.writestr("other.py", "") + with pytest.raises(KeyError, match=r"missing\.py"): + member_reader(wheel)(PurePosixPath("missing.py")) + + +def test_member_reader_checks_the_archive(tmp_path: Path) -> None: + # the same checks as every other read, so names equal ignoring case are refused here too + with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: + archive.writestr("table.py", "") + archive.writestr("TABLE.py", "") + with pytest.raises(WheelError, match=r"holds more than one member named table\.py, ignoring case"): + member_reader(wheel)(PurePosixPath("table.py")) + + +@pytest.mark.parametrize( + "name", + [ + pytest.param("/etc/hostname", id="absolute"), + pytest.param("lib/../../../../etc/os-release", id="parent"), + pytest.param("sub/../../outside.txt", id="outside"), + ], +) +def test_archive_member_names_never_reach_the_host(tmp_path: Path, name: str) -> None: + # each member is hashed from the zip, so a name that would point outside an unpacked copy names the member alone + with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w") as handle: + handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") + handle.writestr(zipfile.ZipInfo(name), b"inside the archive") + row = read_archive(archive).files[name] + assert (row.digest, row.size) == (hashlib.sha256(b"inside the archive").hexdigest(), "18") diff --git a/tests/_collect/test_source.py b/tests/_collect/test_source.py index 84cc09a..a5e2128 100644 --- a/tests/_collect/test_source.py +++ b/tests/_collect/test_source.py @@ -5,15 +5,16 @@ import hashlib import re import zipfile +from io import BytesIO from itertools import starmap from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Final import pytest -from python_sbom._collect._source import read_bundled_wheel, read_source_tree, wheel_request +from python_sbom._collect._source import read_bundled_wheels, read_source_tree, wheel_request from python_sbom._errors import SettingsError, WheelError -from python_sbom._model._inputs import BuildScript, RecordRow, SbomRequest +from python_sbom._model._inputs import BuildScript, BundledWheel, RecordRow, SbomRequest from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: @@ -118,7 +119,7 @@ def test_bundled_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: "pip/_vendor/rich-13.0.dist-info/METADATA": b"Name: rich\nVersion: 13.0\n", }, ) - bundled = read_bundled_wheel(wheel.read_bytes(), "src/pip-1.0-py3-none-any.whl") + bundled = _bundled(wheel.read_bytes(), "src/pip-1.0-py3-none-any.whl") assert (bundled.path, bundled.sha256, bundled.metadata_name, bundled.metadata.name, bundled.files[0]) == ( "src/pip-1.0-py3-none-any.whl", hashlib.sha256(wheel.read_bytes()).hexdigest(), @@ -169,7 +170,7 @@ def test_malformed_bundled_wheel( ) -> None: wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) with pytest.raises(WheelError, match=message): - read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") @pytest.mark.parametrize( @@ -186,12 +187,12 @@ def test_bundled_wheel_file_name_names_the_project( ) -> None: wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", _PIP) with pytest.raises(WheelError, match=f"bundled wheel {name} is not a wheel: {message}"): - read_bundled_wheel(wheel.read_bytes(), name) + _bundled(wheel.read_bytes(), name) def test_corrupt_bundled_wheel() -> None: with pytest.raises(WheelError, match=r"bundled wheel pip\.whl is not a valid zip archive"): - read_bundled_wheel(b"not a zip", "pip.whl") + _bundled(b"not a zip", "pip.whl") def test_bundled_wheel_with_duplicate_member(tmp_path: Path) -> None: @@ -203,7 +204,7 @@ def test_bundled_wheel_with_duplicate_member(tmp_path: Path) -> None: archive.writestr(_METADATA, _PIP[_METADATA]) archive.writestr("pip-1.0.dist-info/RECORD", f"{row(_INIT, b'')}{_METADATA_ROW}") with pytest.raises(WheelError, match=f"holds more than one member named {_INIT}, ignoring case"): - read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") @pytest.mark.parametrize( @@ -220,7 +221,7 @@ def test_bundled_wheel_record_algorithm(tmp_path: Path, write_wheel: WriteWheel, wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) assert [ (file.path, file.algorithm, file.digest) - for file in read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files + for file in _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files ] == [(path, expected, hashlib.new(algorithm, content).hexdigest()) for path, content in _PIP.items()] @@ -250,6 +251,12 @@ def test_bundled_wheel_record_algorithm(tmp_path: Path, write_wheel: WriteWheel, f"RECORD row for {_INIT}: unknown hash algorithm 'sha224'", id="algorithm-cyclonedx-lacks", ), + # installer would call digest() without the length shake_128 needs and raise a TypeError + pytest.param( + f"{_INIT},shake_128=AAAA,0\n{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD row for {_INIT}: unknown hash algorithm 'shake_128'", + id="algorithm-needing-a-length", + ), pytest.param(_METADATA_ROW, f"RECORD lacks a digest for {_INIT}", id="member-not-recorded"), pytest.param( f"{row(_INIT, b'', encoded='A' * 43)}{row(_INIT, b'')}{_METADATA_ROW}", @@ -282,16 +289,14 @@ def test_record_disagrees_with_wheel(tmp_path: Path, write_wheel: WriteWheel, re files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) with pytest.raises(WheelError, match=message): - read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") def test_record_signatures_need_no_row(tmp_path: Path, write_wheel: WriteWheel) -> None: record = "".join(starmap(row, _PIP.items())) + _RECORD_ROW files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), "pip-1.0.dist-info/RECORD.jws": b"{}"} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - assert [file.path for file in read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files] == list( - _PIP - ) + assert [file.path for file in _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files] == list(_PIP) @pytest.mark.parametrize( @@ -307,14 +312,14 @@ def test_signature_away_from_record_needs_a_row(tmp_path: Path, write_wheel: Wri files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), signature: b"import os\n"} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) with pytest.raises(WheelError, match=f"RECORD lacks a digest for {re.escape(signature)}"): - read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") def test_vendored_metadata_needs_name_and_version(tmp_path: Path, write_wheel: WriteWheel) -> None: files = {**_PIP, "pip/_vendor/rich-13.0.dist-info/METADATA": b"Metadata-Version: 2.1\n"} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files) with pytest.raises(WheelError, match=r"pip/_vendor/rich-13\.0\.dist-info/METADATA lacks Name or Version"): - read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") @pytest.mark.parametrize( @@ -331,7 +336,7 @@ def test_vendor_manifest_needs_exact_pins( ) -> None: wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", {**_PIP, "pip/_vendor/vendor.txt": manifest}) with pytest.raises(WheelError, match=message): - read_bundled_wheel(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") def test_wheel_request(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: @@ -363,3 +368,20 @@ def test_wheel_request_needs_a_wheel( archive = write_wheel(tmp_path / name, {"native/__init__.py": b""}, record=False) with pytest.raises(WheelError, match=message): wheel_request(archive, Path(), settings, ()) + + +def _bundled(data: bytes, path: str) -> BundledWheel: + return read_bundled_wheels([path], lambda _: BytesIO(data))[0] + + +def test_bundled_wheels_come_in_the_order_given(tmp_path: Path, write_wheel: WriteWheel) -> None: + # read side by side, yet reported in order, the first broken wheel's error included + good = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", _PIP).read_bytes() + wheels = {"pip-1.0-py3-none-any.whl": good, "b.whl": b"not a zip", "c.whl": b"neither"} + paths = ["pip-1.0-py3-none-any.whl", "b.whl", "c.whl"] + assert [bundled.path for bundled in read_bundled_wheels(paths[:1] * 2, lambda name: BytesIO(wheels[name]))] == [ + paths[0], + paths[0], + ] + with pytest.raises(WheelError, match=r"^bundled wheel b\.whl is not a valid zip archive"): + read_bundled_wheels(paths, lambda name: BytesIO(wheels[name])) diff --git a/tests/_model/test_components.py b/tests/_model/test_components.py index ba274b8..0edf0d4 100644 --- a/tests/_model/test_components.py +++ b/tests/_model/test_components.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib from pathlib import PurePosixPath from typing import TYPE_CHECKING, Final @@ -9,7 +8,6 @@ from python_sbom._model._components import ( bundled_wheel_component, component_from_metadata, - file_component_from_bytes, script_tool, ) from python_sbom._model._inputs import BundledWheel, RecordRow @@ -309,16 +307,6 @@ def test_bundled_wheel_supplier_never_comes_from_the_wheel() -> None: assert "supplier" not in component -def test_file_component_from_bytes() -> None: - assert file_component_from_bytes("app", "a.py", b"abc") == { - "type": "file", - "bom-ref": "app#a.py", - "name": "a.py", - "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(b"abc").hexdigest()}], - "properties": [{"name": "size", "value": "3"}], - } - - def test_script_tool_with_link() -> None: tool = script_tool(PurePosixPath("build/hook.py"), "ef" * 32, "1.0", "https://example.com/hook.py") assert tool == { diff --git a/tests/_model/test_content.py b/tests/_model/test_content.py index 0a969a0..e1f2eba 100644 --- a/tests/_model/test_content.py +++ b/tests/_model/test_content.py @@ -4,8 +4,6 @@ from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING -import pytest - from python_sbom import BundledWheelInfo, ContentProvider, Contributions if TYPE_CHECKING: @@ -30,22 +28,7 @@ def test_context_carries_project(context_for: ContextFor, tmp_path: Path) -> Non ) -def test_context_reads_a_directory(tmp_path: Path, context_for: ContextFor) -> None: - (tmp_path / "embed").mkdir() - (tmp_path / "embed" / "table.py").write_bytes(b"x = 1\n") - assert context_for(tmp_path, tmp_path).read(PurePosixPath("embed/table.py")) == b"x = 1\n" - - -def test_context_reads_a_zip(tmp_path: Path, context_for: ContextFor) -> None: +def test_context_reads_through_its_reader(context_for: ContextFor, tmp_path: Path) -> None: with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: archive.writestr("embed/table.py", "x = 1\n") assert context_for(tmp_path, wheel).read(PurePosixPath("embed/table.py")) == b"x = 1\n" - - -@pytest.mark.parametrize("artifact", [pytest.param("dir", id="directory"), pytest.param("a.whl", id="zip")]) -def test_context_misses(tmp_path: Path, context_for: ContextFor, artifact: str) -> None: - (tmp_path / "dir").mkdir() - with zipfile.ZipFile(tmp_path / "a.whl", "w") as archive: - archive.writestr("other.py", "") - with pytest.raises(KeyError): - context_for(tmp_path, tmp_path / artifact).read(PurePosixPath("missing.py")) diff --git a/tests/conftest.py b/tests/conftest.py index 21eee23..7ae0db7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ import pytest from python_sbom import ContentProvider, ProviderContext +from python_sbom._collect._archive import member_reader from python_sbom._generate._pipeline import build_sbom from python_sbom._model._identity import project_identity from python_sbom._model._inputs import BuildFacts, Interpreter, SbomRequest, Settings, SourceTree @@ -158,7 +159,7 @@ def describe( root=Path(), settings=settings, identity=project_identity(request.metadata, settings), - artifact=Path(), + read=member_reader(Path("example-1.0-py3-none-any.whl")), ) return build_sbom( request, @@ -175,6 +176,6 @@ def describe( def context_for(settings: Settings) -> ContextFor: def context(root: Path, artifact: Path) -> ProviderContext: identity = project_identity(parse_metadata("Name: Example\nVersion: 1.0\n"), settings) - return ProviderContext(root=root, settings=settings, identity=identity, artifact=artifact) + return ProviderContext(root=root, settings=settings, identity=identity, read=member_reader(artifact)) return context diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 1b9334a..02281c7 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -70,7 +70,9 @@ def reached(start: _Hint) -> set[_Hint]: pending.extend(typing.get_type_hints(item, globalns=_names(), include_extras=True).values()) for member in _members(item): pending.extend(typing.get_type_hints(member, globalns=_names(), include_extras=True).values()) - pending.extend(typing.get_args(item)) + # a Callable's parameters come as one list among its arguments + for argument in typing.get_args(item): + pending.extend(argument if isinstance(argument, list) else [argument]) if (origin := typing.get_origin(item)) is not None: pending.append(origin) return {item for item in found if _ours(item)} From 09c0aa4b3a3f02bc52e7f6c5f235d5d61735deab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:58:17 -0800 Subject: [PATCH 057/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20write?= =?UTF-8?q?=20every=20zip=20entry=20one=20way?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit archive --embed built its entry in the CLI, with another clock call and no regular-file bit, and counted directories and empty files when picking the compression, so an archive Info-ZIP wrote got its SBOM stored. Both writers now share one entry builder in _wheel, and append_member picks the method the archive's files use. A RECORD whose last row lacks its line end gets one before the SBOM's row, which ran onto it. --- src/python_sbom/_cli/_commands.py | 25 ++------------- src/python_sbom/_wheel/_archive.py | 41 ++++++++++++++++++++++--- tests/_cli/test_commands.py | 3 +- tests/_wheel/test_archive.py | 49 +++++++++++++++++++++++++++++- 4 files changed, 90 insertions(+), 28 deletions(-) diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py index 7f3ca1c..f0b35c4 100644 --- a/src/python_sbom/_cli/_commands.py +++ b/src/python_sbom/_cli/_commands.py @@ -2,11 +2,8 @@ import json import sys -import zipfile -from collections import Counter -from datetime import UTC, datetime from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING from python_sbom._errors import SbomError from python_sbom._validate._document import MAX_COMPONENTS @@ -16,10 +13,6 @@ from ._options import Options -# the zlib level each setting of the deflate option bits in a member's flags stands for (APPNOTE 4.4.4: normal, maximum, -# fast, super fast), as Info-ZIP sets them; None is zlib's default -_DEFLATE_LEVELS: Final[dict[int, int | None]] = {0b000: None, 0b010: 9, 0b100: 2, 0b110: 1} - def add_embed(commands: _SubParsersAction[ArgumentParser]) -> None: parser = commands.add_parser( @@ -97,6 +90,7 @@ def _archive(options: Options) -> bool: from python_sbom._formats._json import dumps # ruff: ignore[import-outside-top-level] # each command loads only what it runs from python_sbom._generate._archive import describe_archive # ruff: ignore[import-outside-top-level] # each command loads only what it runs from python_sbom._generate._embed import revision_reader # ruff: ignore[import-outside-top-level] # each command loads only what it runs + from python_sbom._wheel._archive import append_member # ruff: ignore[import-outside-top-level] # each command loads only what it runs root = Path.cwd() if options.root is None else options.root facts = collect_facts(root, revision_reader(options.commit), path=options.build_env) @@ -111,20 +105,7 @@ def _archive(options: Options) -> bool: (target := options.output or options.archive.with_name(name)).write_bytes(content := dumps(document)) sys.stdout.write(f"wrote {target}\n") if options.embed: - with zipfile.ZipFile(options.archive, "a") as handle: - if name in handle.namelist(): - msg = f"{options.archive.name} already holds {name}" - raise SbomError(msg) - # a fixed time and Unix attributes, so the archive's bytes are the same whichever system adds the entry - entry = zipfile.ZipInfo(name, datetime.fromtimestamp(max(facts.epoch, 315532800), tz=UTC).timetuple()[:6]) - entry.create_system, entry.external_attr = 3, 0o644 << 16 - # compressed by the method most members use, at the level their deflate option bits name, since the zip - # format records no level itself; Python's zipfile leaves the bits at normal, zlib's default level - methods = Counter(info.compress_type for info in handle.infolist()) - method = next(iter(methods.most_common(1)), (zipfile.ZIP_STORED, 0))[0] - flags = next((info.flag_bits for info in handle.infolist() if info.compress_type == method), 0) - level = _DEFLATE_LEVELS[flags & 0b110] if method == zipfile.ZIP_DEFLATED else None - handle.writestr(entry, content, compress_type=method, compresslevel=level) + append_member(options.archive, name, content, facts.epoch) sys.stdout.write(f"wrote {name} into {options.archive.name}\n") return True diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index abeb105..9113314 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -29,6 +29,11 @@ # metadata files beyond this size are refused, so a hostile wheel cannot exhaust memory _METADATA_LIMIT: Final[int] = 16 * 1024 * 1024 _CHUNK: Final[int] = 1024 * 1024 +# 1980-01-01, the earliest time a zip entry holds +_ZIP_EPOCH: Final[int] = 315532800 +# the zlib level each setting of the deflate option bits in a member's flags stands for (APPNOTE 4.4.4: normal, maximum, +# fast, super fast), as Info-ZIP sets them; None is zlib's default +_DEFLATE_LEVELS: Final[dict[int, int | None]] = {0b000: None, 0b010: 9, 0b100: 2, 0b110: 1} def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: @@ -41,11 +46,8 @@ def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: record_info: Final[zipfile.ZipInfo] = archive.getinfo(record) record_data: Final[bytes] = archive.read(record_info) sbom: Final[str] = f"{dist_info}/sboms/{name}" - # zip timestamps start at 1980 - entry = zipfile.ZipInfo(sbom, time.gmtime(max(epoch, 315532800))[:6]) + entry: Final[zipfile.ZipInfo] = _entry(sbom, epoch) entry.compress_type = zipfile.ZIP_DEFLATED - entry.create_system = 3 # Unix, whichever system writes the entry, so the bytes match across operating systems - entry.external_attr = (stat.S_IFREG | 0o644) << 16 rewritten: Final[bytes] = _rewritten_record(record_data, record, sbom, content) # a name no "dist/*.whl" glob matches, in the wheel's directory so the final rename stays on one file system descriptor, temporary = tempfile.mkstemp(dir=wheel.parent, prefix=".python-sbom-", suffix=".tmp") @@ -100,11 +102,22 @@ def _wheel_source(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, str] return wheel, dist_info +def _entry(name: str, epoch: int) -> zipfile.ZipInfo: + # zip times start at 1980; Unix attributes whichever system writes the entry, so the bytes match across systems + entry: Final[zipfile.ZipInfo] = zipfile.ZipInfo(name, time.gmtime(max(epoch, _ZIP_EPOCH))[:6]) + entry.create_system = 3 + entry.external_attr = (stat.S_IFREG | 0o644) << 16 + return entry + + def _rewritten_record(record: bytes, name: str, sbom: str, content: bytes) -> bytes: lines: Final[list[str]] = [ line for line in record.decode("utf-8").splitlines(keepends=True) if not line.startswith(f"{sbom},") ] newline: Final[str] = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" + # a last row without its line end would take the SBOM's row onto its own line + if lines and not lines[-1].endswith(("\n", "\r")): + lines[-1] += newline # the row of RECORD itself stays last at: Final[int] = next((index for index, line in enumerate(lines) if line.startswith(f"{name},")), len(lines)) # installer's helper hashes as it copies; the copy is thrown away @@ -146,7 +159,27 @@ def member_digest(archive: zipfile.ZipFile, name: str | zipfile.ZipInfo, algorit return digest.hexdigest() +def append_member(archive: Path, name: str, content: bytes, epoch: int) -> None: + with zipfile.ZipFile(archive, "a") as handle: + if name in handle.namelist(): + msg = f"{archive.name} already holds {name}" + raise WheelError(msg) + # compressed by the method most of the archive's files use, at the level their deflate option bits name, since + # the zip format records no level itself and Python's zipfile leaves the bits at normal, zlib's default; + # directories and empty files count for none, as zip tools store them whatever they compress + files: Final[list[zipfile.ZipInfo]] = [ + info for info in handle.infolist() if info.file_size and not info.is_dir() + ] + method: Final[int] = next( + iter(Counter(info.compress_type for info in files).most_common(1)), (zipfile.ZIP_STORED, 0) + )[0] + flags: Final[int] = next((info.flag_bits for info in files if info.compress_type == method), 0) + level: Final[int | None] = _DEFLATE_LEVELS[flags & 0b110] if method == zipfile.ZIP_DEFLATED else None + handle.writestr(_entry(name, epoch), content, compress_type=method, compresslevel=level) + + __all__ = [ + "append_member", "member_digest", "open_archive", "read_member", diff --git a/tests/_cli/test_commands.py b/tests/_cli/test_commands.py index 574df00..a62f8d3 100644 --- a/tests/_cli/test_commands.py +++ b/tests/_cli/test_commands.py @@ -2,6 +2,7 @@ import json import runpy +import stat import struct import subprocess import sys @@ -210,7 +211,7 @@ def test_archive_embeds_sbom(project: Path, zipapp: Path, tmp_path: Path) -> Non entry = zipfile.ZipFile(zipapp).getinfo("app.pyz.cdx.json") assert (entry.create_system, entry.external_attr, entry.compress_type, zipfile.ZipFile(zipapp).read(entry)) == ( 3, - 0o644 << 16, + (stat.S_IFREG | 0o644) << 16, zipfile.ZIP_STORED, (tmp_path / "out.json").read_bytes(), ) diff --git a/tests/_wheel/test_archive.py b/tests/_wheel/test_archive.py index 24854c3..fef80c8 100644 --- a/tests/_wheel/test_archive.py +++ b/tests/_wheel/test_archive.py @@ -13,7 +13,7 @@ import pytest from python_sbom._errors import WheelError -from python_sbom._wheel._archive import open_archive, read_member, write_sbom +from python_sbom._wheel._archive import append_member, open_archive, read_member, write_sbom if TYPE_CHECKING: from collections.abc import Iterator @@ -264,3 +264,50 @@ def test_wheel_with_two_dist_info_directories(tmp_path: Path, write_wheel: Write message = r"Wheel doesn't contain exactly one \.dist-info directory .*'aaa-0\.dist-info', 'native-1\.0\.dist-info'" with pytest.raises(WheelError, match=message): write_sbom(wheel, "native.cdx.json", b"{}\n", 0) + + +def test_embed_ends_an_unterminated_last_row(tmp_path: Path, write_wheel: WriteWheel) -> None: + # no row for RECORD itself, and no line end after the last row, so the SBOM's row goes after it on a line of its own + record = b"native-1.0.dist-info/METADATA,," + wheel = write_wheel( + tmp_path / "native-1.0-py3-none-any.whl", + {"native-1.0.dist-info/METADATA": b"Name: native\n", "native-1.0.dist-info/RECORD": record}, + record=False, + ) + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert zipfile.ZipFile(wheel).read("native-1.0.dist-info/RECORD").decode().splitlines() == [ + "native-1.0.dist-info/METADATA,,", + _SBOM_ROW, + ] + + +@pytest.mark.parametrize( + ("members", "method"), + [ + pytest.param([("a.py", b"x" * 100, zipfile.ZIP_DEFLATED)], zipfile.ZIP_DEFLATED, id="deflated"), + pytest.param([("a.py", b"x" * 100, zipfile.ZIP_STORED)], zipfile.ZIP_STORED, id="stored"), + # Info-ZIP stores directories and empty files whatever it compresses, so they outvote no one + pytest.param( + [("d/", b"", zipfile.ZIP_STORED), ("e", b"", zipfile.ZIP_STORED), ("a.py", b"x", zipfile.ZIP_DEFLATED)], + zipfile.ZIP_DEFLATED, + id="directories-and-empty-files-skipped", + ), + pytest.param([], zipfile.ZIP_STORED, id="empty-archive"), + ], +) +def test_append_member_compresses_as_the_files_are( + tmp_path: Path, members: list[tuple[str, bytes, int]], method: int +) -> None: + with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w") as handle: + for name, content, compression in members: + handle.writestr(name, content, compression) + append_member(archive, "app.pyz.cdx.json", b"{}\n", 1_700_000_000) + entry = zipfile.ZipFile(archive).getinfo("app.pyz.cdx.json") + assert (entry.compress_type, entry.create_system, entry.date_time) == (method, 3, (2023, 11, 14, 22, 13, 20)) + + +def test_append_member_refuses_a_second_copy(tmp_path: Path) -> None: + with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w") as handle: + handle.writestr("app.pyz.cdx.json", b"{}") + with pytest.raises(WheelError, match=r"app\.pyz already holds app\.pyz\.cdx\.json"): + append_member(archive, "app.pyz.cdx.json", b"{}\n", 1_700_000_000) From 53a236ee178df2f69dbe20527666fdbc30ab8a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:46:27 -0800 Subject: [PATCH 058/115] =?UTF-8?q?=F0=9F=93=A6=20build:=20declare=20the?= =?UTF-8?q?=20schemas'=20Apache-2.0=20license?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wheel ships the CycloneDX schema files under their Apache-2.0 LICENSE, so the license expression names both licenses and the schemas' LICENSE joins the license files. That file now sits in the wheel twice, beside the schemas and under .dist-info/licenses, so pkg_meta lets check-wheel-contents accept the duplicate. --- pyproject.toml | 4 ++-- tox.toml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 94149dc..4a7ace9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,8 +20,8 @@ keywords = [ "sbom", "spdx", ] -license = "MIT" -license-files = [ "LICENSE" ] +license = "MIT AND Apache-2.0" +license-files = [ "LICENSE", "src/python_sbom/_validate/schemas/LICENSE" ] maintainers = [ { name = "Bernรกt Gรกbor", email = "gaborjbernat@gmail.com" }, ] diff --git a/tox.toml b/tox.toml index bcb78cd..5ce9dc0 100644 --- a/tox.toml +++ b/tox.toml @@ -109,6 +109,9 @@ commands = [ [ "check-wheel-contents", "--no-config", + # the CycloneDX schemas' Apache-2.0 LICENSE ships beside them and, as a license file, again under .dist-info/licenses + "--ignore", + "W002", "{env_tmp_dir}", ], [ From 744e8f779968c77f26f5e6f32420d4af8bf7e2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:47:13 -0800 Subject: [PATCH 059/115] =?UTF-8?q?=E2=9C=85=20test:=20validate=20the=20bu?= =?UTF-8?q?ilt=20wheel's=20own=20SBOM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pkg_meta built the wheel and checked its tool list but never ran the validator on it. It now installs that wheel with the validate extra and runs python-sbom validate on it, so a schema file or validator dependency the wheel leaves out fails the check. --- tox.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tox.toml b/tox.toml index 5ce9dc0..d558966 100644 --- a/tox.toml +++ b/tox.toml @@ -125,6 +125,20 @@ commands = [ tool in tools), f'hatch_build.py missing from {sbom}'\ """, ], + [ + "python", + "-c", + # the built wheel validates its own SBOM, so a schema file or validator dependency left out of it fails here + """\ + import glob, pathlib, subprocess, sys; (wheel,) = glob.glob(r'{env_tmp_dir}{/}*.whl'); subprocess.check_call(['uv', \ + 'pip', 'install', '--python', sys.executable, f'python-sbom[validate] @ {pathlib.Path(wheel).as_uri()}'])\ + """, + ], + [ + "python-sbom", + "validate", + "{env_tmp_dir}", + ], ] [env.type] From a8a689795f9497574fa77164bb078b417144e60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:47:28 -0800 Subject: [PATCH 060/115] =?UTF-8?q?=F0=9F=93=9D=20docs(changelog):=20fold?= =?UTF-8?q?=20validation=20into=20feature?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The performance fragment compared validation against behavior no release had, so it goes. The feature fragment now names each subcommand, archive included, and says validate checks the shipped CycloneDX schemas with jsonschema-rs and caps an SBOM at 100,000 components by default. --- docs/changelog.d/+fast-validation.performance.rst | 5 ----- docs/changelog.d/+python-sbom.feature.rst | 8 +++++--- 2 files changed, 5 insertions(+), 8 deletions(-) delete mode 100644 docs/changelog.d/+fast-validation.performance.rst diff --git a/docs/changelog.d/+fast-validation.performance.rst b/docs/changelog.d/+fast-validation.performance.rst deleted file mode 100644 index 269312a..0000000 --- a/docs/changelog.d/+fast-validation.performance.rst +++ /dev/null @@ -1,5 +0,0 @@ -``python-sbom validate`` checks the CycloneDX schema with jsonschema-rs, against the official CycloneDX schema files -python-sbom now ships, so an SBOM validates in milliseconds instead of seconds, and its time grows in step with its -size; the default ``--max-components`` cap rises from 5,000 to 100,000 components to match. Building an SBOM reads each -build-environment distribution's ``RECORD`` and metadata once, and rewriting the wheel copies its other members without -recompressing them. diff --git a/docs/changelog.d/+python-sbom.feature.rst b/docs/changelog.d/+python-sbom.feature.rst index d356c8f..d5e6604 100644 --- a/docs/changelog.d/+python-sbom.feature.rst +++ b/docs/changelog.d/+python-sbom.feature.rst @@ -1,3 +1,5 @@ -Add the ``python-sbom`` hatchling build hook, which writes a CycloneDX 1.6 SBOM into each wheel, the ``python-sbom`` -command that embeds the same SBOM into wheels other build backends build and renders or validates it, and the Python API -behind both. +Add the ``python-sbom`` hatchling build hook, which writes a CycloneDX 1.6 SBOM into each wheel, the Python API behind +it, and the ``python-sbom`` command: ``embed`` writes the same SBOM into wheels other build backends build, ``archive`` +describes another zip archive a project publishes, such as a zipapp, ``spdx`` renders an SBOM as SPDX, and ``validate`` +checks the SBOMs in wheels and archives against the official CycloneDX schema files python-sbom ships, with +jsonschema-rs, and rejects one listing more than 100,000 components unless ``--max-components`` allows more. From fee1ec22a73bb8997a4639b0f8b9757a9e4a6bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:47:57 -0800 Subject: [PATCH 061/115] =?UTF-8?q?=F0=9F=93=9D=20docs(security):=20name?= =?UTF-8?q?=20python-sbom=20archive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The policy named the hook, embed, spdx and validate but not archive, and its scope left out wheels bundled in an archive it describes. --- .github/SECURITY.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 90a06ec..68357af 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -1,8 +1,9 @@ # Security policy -python-sbom writes a CycloneDX SBOM into wheels, from a hatchling build hook or from a command that rewrites a wheel -another build backend built, and renders or validates that SBOM. This policy says how to report a security problem and -how we decide whether a report is one. +python-sbom writes a CycloneDX SBOM into wheels, from a hatchling build hook or from `python-sbom embed`, which rewrites +a wheel another build backend built. `python-sbom archive` describes another zip archive a project publishes, such as a +zipapp, and `python-sbom spdx` and `validate` render or check an SBOM. This policy says how to report a security problem +and how we decide whether a report is one. ## Reporting a vulnerability @@ -34,8 +35,8 @@ publish no advisory. Classify by who supplies the dangerous input, not by how ba **In scope** -- A bundled wheel, or an SBOM given to `python-sbom spdx` or `validate`, that makes python-sbom write outside its output - file or the wheel it rewrites, or run code. +- A wheel bundled in a wheel or in an archive given to `python-sbom archive`, or an SBOM given to `python-sbom spdx` or + `validate`, that makes python-sbom write outside its output file or the wheel or archive it rewrites, or run code. - An SBOM that misstates what a wheel holds in a way its inputs cannot explain, such as a wrong hash. **Out of scope** From 6daa53e62b9bee9f6b01878bcb417b908c976b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:49:23 -0800 Subject: [PATCH 062/115] =?UTF-8?q?=F0=9F=93=9D=20docs(extend):=20show=20t?= =?UTF-8?q?he=20tested=20seeder=20hook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The virtualenv example named a provider module it never showed, a hook script that returned the plain hook, and a provider whose constructor takes an argument the content-provider option cannot pass, with a table of virtualenv values next to the seeder fixture's code. It now shows the seeder fixture the test suite builds, its options and its hook subclass, with the property values that build yields, and says why a provider every SBOM shares belongs in the option instead. --- docs/extend.rst | 86 +++++++++++++++++++------------------------------ 1 file changed, 33 insertions(+), 53 deletions(-) diff --git a/docs/extend.rst b/docs/extend.rst index 933a85e..2055ceb 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -37,80 +37,60 @@ SBOM lists its module among the build tools, hashed, because it shapes the docum zlib = {'type': 'library', 'bom-ref': 'pkg:generic/zlib@1.3.1', 'name': 'zlib', 'version': '1.3.1'} return Contributions(components=[zlib]) -Worked example: virtualenv --------------------------- - -`virtualenv `_ ships the pip and setuptools wheels it installs into new -environments, under ``src/virtualenv/seed/wheels/embed``. A ``BUNDLE_SUPPORT`` table in that package's ``__init__.py``, -after a ``BUNDLE_FOLDER`` assignment, decides which Python versions receive each wheel. Its options name the supplier -and the wheels: - -.. code-block:: toml - - [build-system] - build-backend = 'hatchling.build' - requires = ['hatch-vcs>=0.5', 'hatchling>=1.28', "python-sbom[hatch]>=0.1; python_version >= '3.11'"] - - [tool.hatch.build.hooks.custom] - path = 'hatch_build.py' - - [tool.python-sbom] - supplier = { name = 'Python Packaging Authority', url = ['https://www.pypa.io'] } - bundled-wheels = ['src/virtualenv/seed/wheels/embed/*.whl'] - content-provider = 'tasks.sbom_content:VirtualenvContent' - -virtualenv still builds on Python versions python-sbom does not support, so its ``hatch_build.py`` loads the hook only -where the requirement's marker installs it, and falls back to hatchling's plain hook elsewhere: - -.. code-block:: python - - import sys - - if sys.version_info >= (3, 11): - from python_sbom.hatch import SbomBuildHook - else: # python-sbom needs Python 3.11, so builds on older interpreters carry no SBOM - from hatchling.builders.hooks.plugin.interface import BuildHookInterface as SbomBuildHook +Worked example: a seeder +------------------------ +A seeder, a project like `virtualenv `_, ships the wheels it installs into new +environments under ``src/seeder/embed``. A ``BUNDLE_SUPPORT`` table in that package's ``__init__.py``, after a +``BUNDLE_FOLDER`` assignment, decides which Python versions receive each wheel. The project's options name the supplier +and the wheels, and enable hatchling's custom hook, which loads ``hatch_build.py`` from the project root: - def get_build_hook(): - return SbomBuildHook +.. literalinclude:: ../tests/_seeder/pyproject.toml + :language: toml + :start-at: [tool.hatch] -The ``get_build_hook`` function tells hatchling which class to load. The SBOM lists the script among the build tools, -hashed. The provider reads the table next to each wheel through the context, by name, without importing the package, and -names the project's supplier as the wheels' supplier, since virtualenv bundles only PyPA's own projects. The test suite -builds a project with virtualenv's layout with the same provider, handed over from a hook subclass: +The hook script subclasses :class:`~python_sbom.hatch.SbomBuildHook` and overrides +:meth:`~python_sbom.hatch.SbomBuildHook.content_provider`, so it builds the provider with the table's name, an argument +the ``content-provider`` option has no way to pass. ``get_build_hook`` tells hatchling which class to load, since the +script also imports the base class. The provider reads the table next to each wheel through the context, by name, +without importing the package, and names the project's supplier as the wheels' supplier, since the seeder bundles only +that supplier's wheels. The test suite builds this project: .. literalinclude:: ../tests/_seeder/hatch_build.py :language: python - :pyobject: BundleSupport -The pip component then carries these properties, in this order: +The SBOM lists ``hatch_build.py`` among the build tools, hashed. The bundled pip wheel's component then carries these +properties first, in this order, followed by those from its own metadata: .. list-table:: :header-rows: 1 * - Property - Value - * - ``virtualenv:bundled-wheel`` - - ``virtualenv/seed/wheels/embed/pip-26.1-py3-none-any.whl`` - * - ``virtualenv:seeded-for-python`` + * - ``seeder:bundled-wheel`` + - ``seeder/embed/pip-1.0-py3-none-any.whl`` + * - ``seeder:seeded-for-python`` - ``3.14`` - * - ``virtualenv:seeded-for-python`` + * - ``seeder:seeded-for-python`` - ``3.15`` - * - ``python:requires-python`` - - ``>=3.10`` -A bundled wheel is named by its path inside the artifact, so the wheel's SBOM and the zipapp's give it the same +A bundled wheel is named by its path inside the artifact, so the wheel's SBOM and an archive's give it the same ``bom-ref``. +A hook subclass serves only the hatchling build: ``python-sbom embed`` and ``python-sbom archive`` load the provider the +``content-provider`` option or their ``--content-provider`` names, and build a class without arguments. A project that +wants the same provider in every SBOM it publishes gives the provider a constructor without arguments, or a module-level +instance, and names it in the option instead of overriding the hook. + Describe another archive ------------------------ -virtualenv also publishes a zipapp, ``virtualenv.pyz``, that holds virtualenv's own ``.dist-info`` at its top level, the -wheels it seeds, and its dependencies unpacked under ``__virtualenv__/``. ``python-sbom archive`` describes such an -archive the way the hook describes a wheel: the archive's own files, the project's distribution, each wheel it bundles -as a ``.whl`` member, and each one unpacked into a directory with a ``.dist-info``. The content provider hears of every -bundled wheel, and its context reads the archive's files: +A project may also publish a zipapp, such as virtualenv's ``virtualenv.pyz``, which holds virtualenv's own +``.dist-info`` at its top level, the wheels it seeds, and its dependencies unpacked under ``__virtualenv__/``. +``python-sbom archive`` describes such an archive the way the hook describes a wheel: the archive's own files, the +project's distribution, each wheel it bundles as a ``.whl`` member, and each one unpacked into a directory with a +``.dist-info``. The content provider the option names hears of every bundled wheel, and its context reads the archive's +files: .. code-block:: console From 4a3d379f9e825807787c593ea1474c10bdc24719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:49:36 -0800 Subject: [PATCH 063/115] =?UTF-8?q?=F0=9F=93=9D=20docs(extend):=20read=20R?= =?UTF-8?q?ECORD=20to=20list=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ProviderContext has no member listing, so a provider that needs one reads the project's .dist-info/RECORD through read. Document that as the supported pattern rather than widening the API with a names() method. --- docs/extend.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/extend.rst b/docs/extend.rst index 2055ceb..a6c8d66 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -37,6 +37,12 @@ SBOM lists its module among the build tools, hashed, because it shapes the docum zlib = {'type': 'library', 'bom-ref': 'pkg:generic/zlib@1.3.1', 'name': 'zlib', 'version': '1.3.1'} return Contributions(components=[zlib]) +The context has no method that lists files. A provider that needs the wheel's file list reads its ``RECORD`` through +:meth:`~python_sbom.ProviderContext.read`, by the path of the project's ``.dist-info`` directory, such as +``seeder-1.0.dist-info/RECORD``; an archive holds that directory at its top level, and the path reads the same there. The +provider knows the project's name, and a hook subclass, shown below, can hand it hatchling's ``self.metadata.version``. +Any other member reads the same way, by its path inside the artifact. + Worked example: a seeder ------------------------ From f48e640bfa682252f5c4b43e1380f1addf1072e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:50:31 -0800 Subject: [PATCH 064/115] =?UTF-8?q?=F0=9F=93=9D=20docs(contributing):=20ex?= =?UTF-8?q?plain=20updating=20the=20schemas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The schemas' upstream tags lived only in a code comment. The contributing guide now names the CycloneDX/specification tag of each shipped file, the raw URLs that fetch them, the test whose SHA-256 pins to refresh, and why .gitattributes and the pre-commit exclude leave the directory alone. --- docs/contributing.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 7530dac..d86f5bc 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -14,6 +14,30 @@ To record a new field, read it at the edge into ``_model/_inputs.py`` and write add a registration function and a handler next to the others in ``_cli/_commands.py`` and list the registration in ``_cli/_parser.py``. Tests mirror the source: ``tests//test_.py``. +Updating the schemas +-------------------- + +``python-sbom validate`` checks documents against the official CycloneDX schema files, which ship byte for byte under +``src/python_sbom/_validate/schemas``. Each comes from a tag of `CycloneDX/specification +`_: ``bom-1.6.schema.json`` and the ``jsf-0.82.schema.json`` it references +from ``1.6.2``, and ``spdx.schema.json`` and the Apache-2.0 ``LICENSE`` from ``1.7.2``, the newest, since the +cyclonedx.org URL every schema version references serves the newest license list. These commands fetch the shipped files +from their raw URLs; change the tag to take a newer release: + +.. code-block:: console + + $ cd src/python_sbom/_validate/schemas + $ curl -fsSLO https://raw.githubusercontent.com/CycloneDX/specification/1.6.2/schema/bom-1.6.schema.json + $ curl -fsSLO https://raw.githubusercontent.com/CycloneDX/specification/1.6.2/schema/jsf-0.82.schema.json + $ curl -fsSLO https://raw.githubusercontent.com/CycloneDX/specification/1.7.2/schema/spdx.schema.json + $ curl -fsSLO https://raw.githubusercontent.com/CycloneDX/specification/1.7.2/LICENSE + $ sha256sum * + +Put each file's tag and SHA-256 into the cases of ``test_schema_files_are_the_upstream_release`` in +``tests/_validate/test_schemas.py``, which fails when a shipped file is not the release it names. ``.gitattributes`` +marks the directory ``-text`` and ``.pre-commit-config.yaml`` excludes it, so neither git nor the hooks rewrite the +files' line endings, whitespace or formatting. + Changelog fragments ------------------- From 9fce8b9e088665c5f056a2a8db711e9933588085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:54:27 -0800 Subject: [PATCH 065/115] =?UTF-8?q?=F0=9F=93=9D=20docs(contributing):=20de?= =?UTF-8?q?scribe=20the=20benchmarks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contributing guide never mentioned bench/. It now shows how the bench tox environment runs python -m bench, what each subcommand does, where results land, and how to compare two revisions. --- docs/contributing.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index d86f5bc..24a0fdb 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -38,6 +38,25 @@ Put each file's tag and SHA-256 into the cases of ``test_schema_files_are_the_up marks the directory ``-text`` and ``.pre-commit-config.yaml`` excludes it, so neither git nor the hooks rewrite the files' line endings, whitespace or formatting. +Benchmarks +---------- + +``bench/`` times python-sbom's hot paths with `pyperf `_; CI does not run it. The +``bench`` tox environment passes its arguments to ``python -m bench``: + +.. code-block:: console + + $ tox r -e bench -- prepare + $ tox r -e bench -- run + $ tox r -e bench -- compare ../baseline/.tox/bench-results/cpython3.15 .tox/bench-results/cpython3.15 + +``prepare`` builds the fixtures, the only step that needs the network. ``run`` times each case in its own pyperf process +tree and writes one JSON file per case under ``.tox/bench-results//``; options after a second ``--`` reach +pyperf. ``profile`` runs each case once under cProfile, and under py-spy with ``--py-spy``. Both take case names to run +only those, and ``list`` prints the names. ``report`` renders result files as a Markdown table, and ``compare`` puts two +result directories side by side. The timed process imports the python-sbom its environment installed, so to compare two +revisions, run the cases from a checkout of each and compare their result directories. + Changelog fragments ------------------- From 7ca9c92a5cdf94e522695f4f40a259afd79a7361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:54:47 -0800 Subject: [PATCH 066/115] =?UTF-8?q?=F0=9F=93=9D=20docs(explanation):=20dro?= =?UTF-8?q?p=20the=20drifting=20SBOM=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file count and SBOM size of python-sbom's own build went stale with each toolchain update. Say what makes the list grow instead. --- docs/explanation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/explanation.rst b/docs/explanation.rst index bef6406..e07941c 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -92,8 +92,8 @@ launchers and uv's cache files are left out too, since they name the frontend, t than the distribution. Hashing each tool's ``RECORD`` instead would be smaller, but that file lists the installer metadata, so two frontends -installing the same tools would give two hashes. The cost is size: python-sbom's own isolated build lists about 600 -files, half of them from setuptools, which setuptools-scm pulls in, for an SBOM of about 400 KB. +installing the same tools would give two hashes. The cost is size: the file list grows with the build environment, and +a large build tool such as setuptools adds hundreds of entries. So a rebuild reproduces the wheel byte for byte on any operating system when these match the original build: the source tree, as a checkout of the same commit; ``SOURCE_DATE_EPOCH``; the Python version; and the versions of the build backend From 3929e4640db4f2f86bcc80312361a17d7df9f13d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:54:51 -0800 Subject: [PATCH 067/115] =?UTF-8?q?=F0=9F=93=9D=20docs(explanation):=20spl?= =?UTF-8?q?it=20the=20architecture=20intro?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The opening paragraph ran to 102 words; the diagram's legend now starts its own paragraph. --- docs/explanation.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/explanation.rst b/docs/explanation.rst index e07941c..cbb4e6d 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -8,8 +8,10 @@ Each concern has its own subpackage, and only the adapters know a build system o and the ``embed`` and ``archive`` commands gather what the edge knows: the request (core metadata, file name, options, build scripts), the source tree, and the build facts (time, revision, toolchain). They hand those, with a :class:`~python_sbom.ContentProvider` and its :class:`~python_sbom.ProviderContext`, to one pipeline, and the model -turns them into the document with pure functions. The diagram follows the imports between the subpackages: an arrow -points at what a package imports, and a dashed arrow at what it imports only for type annotations. +turns them into the document with pure functions. + +The diagram follows the imports between the subpackages: an arrow points at what a package imports, and a dashed arrow +at what it imports only for type annotations. .. mermaid:: From 75ca33b03fa64aa6ea914ce74431cdb7e9ebfccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:55:15 -0800 Subject: [PATCH 068/115] =?UTF-8?q?=F0=9F=93=9D=20docs(bench):=20explain?= =?UTF-8?q?=20why=20=5Fonce=20stays=20apart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SMP505 suppression on _once said only "ruff's C901 on main". Say that inlining it into main would push main over ruff's C901 complexity limit. --- bench/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/__main__.py b/bench/__main__.py index 78ef04f..91b3704 100644 --- a/bench/__main__.py +++ b/bench/__main__.py @@ -208,7 +208,8 @@ def _profile(work: Path, out: Path, names: list[str], *, py_spy: bool) -> None: sys.stdout.write(f"wrote {target}\n") -def _once(name: str, work: Path, stats_file: Path | None) -> None: # simplint: ignore[SMP505] # ruff's C901 on main +# a helper of main's because, inlined into main, it pushes main over ruff's C901 complexity limit +def _once(name: str, work: Path, stats_file: Path | None) -> None: # simplint: ignore[SMP505] # see above # without a stats file for an outside profiler such as py-spy with tempfile.TemporaryDirectory(dir=work, prefix="scratch-") as scratch: step = make_step(name, load(work), Path(scratch)) From 5b9bb66d2f51ad4555ff09652cc5786eb743cec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:55:43 -0800 Subject: [PATCH 069/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20keep=20requiremen?= =?UTF-8?q?ts=20if=20packaging's=20tree=20moves?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requirement-marker reduction reads packaging's private packaging._parser and Marker._markers, which a packaging release can move or rename. The import now happens on use, and an ImportError or AttributeError from either keeps the requirement listed instead of failing the build. --- src/python_sbom/_model/_markers.py | 13 ++++++++++--- tests/_model/test_markers.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/python_sbom/_model/_markers.py b/src/python_sbom/_model/_markers.py index e2513ea..0c0535d 100644 --- a/src/python_sbom/_model/_markers.py +++ b/src/python_sbom/_model/_markers.py @@ -2,11 +2,10 @@ from typing import TYPE_CHECKING, Final -from packaging._parser import Op, Value, Variable # ruff: ignore[import-private-name] # packaging exposes the parsed marker tree only privately from packaging.markers import Marker, UndefinedComparison if TYPE_CHECKING: - from packaging._parser import MarkerAtom, MarkerList # see above + from packaging._parser import MarkerAtom, MarkerList # see _extra_verdict from packaging.requirements import Requirement @@ -17,10 +16,18 @@ def applies_without_extras(requirement: Requirement) -> bool: # packaging's marker grammar for the whole build process if requirement.marker is None: return True - return _extra_verdict(requirement.marker._markers) is not False # ruff: ignore[private-member-access] # the tree, see the import above + try: + verdict = _extra_verdict(requirement.marker._markers) # ruff: ignore[private-member-access] # see _extra_verdict + # packaging keeps the parsed tree private, so a release that moves or renames it leaves the requirement listed + # rather than dropping one that applies + except (AttributeError, ImportError): + return True + return verdict is not False def _extra_verdict(markers: MarkerList | MarkerAtom) -> bool | None: + from packaging._parser import Op, Value, Variable # ruff: ignore[import-private-name, import-outside-top-level] # packaging keeps the marker tree private, the caller handles its absence + # three-valued, with "and" binding tighter than "or", as packaging's own evaluation groups them groups: list[list[bool | None]] = [[]] for item in markers: diff --git a/tests/_model/test_markers.py b/tests/_model/test_markers.py index 5456cea..8664285 100644 --- a/tests/_model/test_markers.py +++ b/tests/_model/test_markers.py @@ -57,3 +57,16 @@ def test_verdicts_hold_under_python_optimize() -> None: ) result = subprocess.run([sys.executable, "-O", "-c", script, *markers], capture_output=True, text=True, check=True) assert result.stdout.split() == [str(verdict) for verdict in markers.values()] + + +def test_requirement_kept_without_packaging_parser(monkeypatch: pytest.MonkeyPatch) -> None: + # None in sys.modules makes the import fail, as when a packaging release moves its private marker tree + monkeypatch.setitem(sys.modules, "packaging._parser", None) + assert applies_without_extras(Requirement('click; extra == "cli"')) is True + + +def test_requirement_kept_without_marker_tree(monkeypatch: pytest.MonkeyPatch) -> None: + # a packaging release that renames the private attribute holding the parsed tree + requirement = Requirement('click; extra == "cli"') + monkeypatch.delattr(requirement.marker, "_markers") + assert applies_without_extras(requirement) is True From 286ee5c78754298d973d880e812cf0f2ac14796f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:56:10 -0800 Subject: [PATCH 070/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20copy=20the=20Keyw?= =?UTF-8?q?ords=20header=20as=20declared?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packaging splits Keywords at commas and strips each keyword, so joining its list back gave "a,b" for a distribution that wrote "a, b", although the SBOM promises the header verbatim. CoreMetadata now keeps the header block, and keywords() reads the Keywords header from it with the email policy packaging uses, only when packaging found one. --- src/python_sbom/_model/_metadata.py | 14 ++++++++++---- tests/_model/test_metadata.py | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/python_sbom/_model/_metadata.py b/src/python_sbom/_model/_metadata.py index df553c8..2fa08d8 100644 --- a/src/python_sbom/_model/_metadata.py +++ b/src/python_sbom/_model/_metadata.py @@ -2,6 +2,8 @@ import re from dataclasses import dataclass +from email.parser import HeaderParser +from email.policy import compat32 from typing import TYPE_CHECKING, Final, Literal, TypeAlias from packaging.metadata import parse_email @@ -37,14 +39,17 @@ def parse_metadata(text: str) -> CoreMetadata: # not validated: the SBOM describes a distribution whose metadata breaks the specification as well, as written; the # body holds the long description, which nothing reads, and parsing it took most of the time - raw, unparsed = parse_email(text[: match.start()] if (match := _BODY.search(text)) else text) - return CoreMetadata(raw=raw, unparsed=unparsed) + headers: Final = text[: match.start()] if (match := _BODY.search(text)) else text + raw, unparsed = parse_email(headers) + return CoreMetadata(raw=raw, unparsed=unparsed, headers=headers) @dataclass(frozen=True, slots=True, kw_only=True) class CoreMetadata: raw: RawMetadata unparsed: dict[str, list[str]] + # the header block as written, for the values packaging reshapes + headers: str = "" @property def name(self) -> str: @@ -68,9 +73,10 @@ def version(self) -> str: return self.first("version") or "" def keywords(self) -> list[str]: - # packaging splits Keywords at commas, so they are joined back into the one value the distribution declared + # packaging splits Keywords at commas and strips each keyword, so the header is read again, as the distribution + # wrote it, only when one is there; packaging parses the block with the same policy if "keywords" in self.raw: - return [",".join(self.raw["keywords"])] + return HeaderParser(policy=compat32).parsestr(self.headers).get_all("keywords", []) return self.unparsed.get("keywords", []) def project_urls(self) -> list[tuple[str, str]]: diff --git a/tests/_model/test_metadata.py b/tests/_model/test_metadata.py index 91fbd74..588f459 100644 --- a/tests/_model/test_metadata.py +++ b/tests/_model/test_metadata.py @@ -28,8 +28,8 @@ def test_without_name_and_version() -> None: [ pytest.param("Keywords: a,b\n", ["a,b"], id="comma-separated"), pytest.param("Keywords: a b\n", ["a b"], id="space-separated"), - # packaging splits at commas and strips each keyword, so the spacing around them is not kept - pytest.param("Keywords: a, b\n", ["a,b"], id="comma-and-space"), + pytest.param("Keywords: a, b\n", ["a, b"], id="comma-and-space"), + pytest.param("Keywords: a , b,c\n", ["a , b,c"], id="mixed-spacing"), pytest.param("Keywords: a\nKeywords: b\n", ["a", "b"], id="repeated"), pytest.param("", [], id="absent"), ], From 66ec2410579825c670410fb583ac4454ceaacb89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:54:43 -0800 Subject: [PATCH 071/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20share?= =?UTF-8?q?=20one=20JSON=20alias=20and=20text=20cap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _document and _schemas each defined the recursive JSON alias and the 200-character cap on quoted text. _document now exports Json and MAX_TEXT; _schemas and the validate tests import them. --- src/python_sbom/_validate/_document.py | 16 +++++++++------- src/python_sbom/_validate/_schemas.py | 15 +++++++-------- tests/_validate/test_document.py | 9 ++++----- tests/_validate/test_schemas.py | 22 +++++++++++----------- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index d09a551..7e49a47 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -18,7 +18,7 @@ from python_sbom._model._types import Component, Document # a JSON value, as json.loads hands it over -_Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" +Json: TypeAlias = "str | int | float | bool | list[Json] | dict[str, Json] | None" # the schema leaves these optional, yet every document python-sbom writes carries them and the checks below read them _REQUIRED: Final[tuple[tuple[str, ...], ...]] = ( ("metadata", "timestamp"), @@ -36,8 +36,8 @@ MAX_COMPONENTS: Final[int] = 100_000 # the modules of the validate extra _EXTRA: Final[tuple[str, ...]] = ("jsonschema_rs", "license_expression", "spdx_tools") -# an exception's message can quote the hostile value that caused it -_MAX_TEXT: Final[int] = 200 +# an exception's message, or a schema error's, can quote the hostile value that caused it, which runs to megabytes +MAX_TEXT: Final[int] = 200 # the schema validator recurses once per level and crashes the process past a few thousand levels on a small thread # stack; the SBOMs python-sbom and other tools write nest about a dozen levels deep _MAX_DEPTH: Final[int] = 128 @@ -59,7 +59,7 @@ def validate_document( # the JSON parser recurses once per level, how deep it gets depends on the Python version return [f"nests more than {_MAX_DEPTH} levels deep, too deep to check"] except Exception as error: # ruff: ignore[blind-except] # a hostile document may trip any check, see above - return [f"could not be checked, {type(error).__name__}: {str(error)[:_MAX_TEXT]}"] + return [f"could not be checked, {type(error).__name__}: {str(error)[:MAX_TEXT]}"] def _problems(raw: str, allowed_licenses: frozenset[str] | None, max_components: int) -> list[str]: @@ -98,13 +98,13 @@ def _problems(raw: str, allowed_licenses: frozenset[str] | None, max_components: return problems -def _census_problems(document: _Json, max_components: int) -> list[str]: +def _census_problems(document: Json, max_components: int) -> list[str]: # counts the components and bom-refs and measures the depth in one linear pass before the schema vouches for any # shape; iterative, so deep nesting cannot exhaust the stack. Nested components count too: the schema checks every # components array, and one array may hold them all, so only the total bounds its work components = 0 refs: Final[Counter[str]] = Counter() - pending: Final[list[tuple[_Json, int]]] = [(document, 1)] + pending: Final[list[tuple[Json, int]]] = [(document, 1)] while pending: node, depth = pending.pop() if depth > _MAX_DEPTH: @@ -131,7 +131,7 @@ def _census_problems(document: _Json, max_components: int) -> list[str]: return [] -def _has(value: _Json, path: tuple[str, ...]) -> bool: +def _has(value: Json, path: tuple[str, ...]) -> bool: for key in path: if not isinstance(value, dict) or key not in value: return False @@ -187,6 +187,8 @@ def _walk(components: list[Component]) -> list[Component]: __all__ = [ "MAX_COMPONENTS", + "MAX_TEXT", + "Json", "is_document", "validate_document", ] diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index ef713e9..398c818 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -2,13 +2,15 @@ import json from importlib.resources import files -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import TYPE_CHECKING, Final from jsonschema_rs import Draft7Validator, Registry from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document +from ._document import MAX_TEXT + if TYPE_CHECKING: from importlib.resources.abc import Traversable @@ -16,13 +18,10 @@ from python_sbom._formats._spdx import Spdx -# a JSON value, as json.loads hands it over -_Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" -# the message quotes the offending value, which for a document root runs to megabytes -_MAX_TEXT: Final[int] = 200 + from ._document import Json -def schema_error(document: _Json) -> str | None: +def schema_error(document: Json) -> str | None: detail: Final[ValidationError | None] = next(_VALIDATOR.iter_errors(document), None) if detail is None: return None @@ -30,7 +29,7 @@ def schema_error(document: _Json) -> str | None: f"[{step}]" if isinstance(step, int) else f".{step}" if step.isidentifier() else f"[{step!r}]" for step in detail.instance_path ) - return f"{detail.message[:_MAX_TEXT]} at {f'${steps}'[:_MAX_TEXT]}" + return f"{detail.message[:MAX_TEXT]} at {f'${steps}'[:MAX_TEXT]}" def spdx_problems(spdx: Spdx) -> list[str]: @@ -46,7 +45,7 @@ def spdx_problems(spdx: Spdx) -> list[str]: # references serves the newest; Apache-2.0, see the LICENSE beside them. Format assertion is on, as in CycloneDX's own # Draft 7 validation, and offline, so a reference missing from the registry fails instead of reaching the network _SCHEMAS: Final[Traversable] = files(__package__) / "schemas" -_REFERENCED: Final[list[dict[str, _Json]]] = [ +_REFERENCED: Final[list[dict[str, Json]]] = [ json.loads((_SCHEMAS / name).read_text(encoding="utf-8")) for name in ("spdx.schema.json", "jsf-0.82.schema.json") ] _VALIDATOR: Final[Draft7Validator] = Draft7Validator( diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py index 66ea9a2..555820f 100644 --- a/tests/_validate/test_document.py +++ b/tests/_validate/test_document.py @@ -2,7 +2,7 @@ import dataclasses import json -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import TYPE_CHECKING, Final import pytest @@ -20,9 +20,8 @@ from python_sbom import Component, License from python_sbom._model._inputs import BuildFacts from python_sbom._model._types import Document + from python_sbom._validate._document import Json -# a JSON value, as json.loads hands it over -_Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" # what auditwheel writes: CycloneDX 1.4 conventions, a legacy tools array, no timestamp _FOREIGN: Final[str] = json.dumps({ "bomFormat": "CycloneDX", @@ -171,7 +170,7 @@ def test_component_without_bom_ref(describe: Describe) -> None: ), ], ) -def test_schema_valid_variations_pass(describe: Describe, mutate: Callable[[dict[str, _Json]], None]) -> None: +def test_schema_valid_variations_pass(describe: Describe, mutate: Callable[[dict[str, Json]], None]) -> None: loaded = json.loads(dumps(describe())) mutate(loaded) assert validate_document(json.dumps(loaded), None) == [] @@ -277,5 +276,5 @@ def test_unexpected_error_is_reported(describe: Describe, mocker: MockerFixture) pytest.param([], False, id="not-an-object"), ], ) -def test_is_document(value: _Json, expected: bool) -> None: +def test_is_document(value: Json, expected: bool) -> None: assert is_document(value) is expected diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index d46f825..f930168 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -3,7 +3,7 @@ import hashlib import sys from importlib.resources import files -from typing import TypeAlias +from typing import TYPE_CHECKING import pytest @@ -11,8 +11,8 @@ from python_sbom._validate._document import validate_document from python_sbom._validate._schemas import schema_error -# a JSON value, as json.loads hands it over -_Json: TypeAlias = "str | int | float | bool | list[_Json] | dict[str, _Json] | None" +if TYPE_CHECKING: + from python_sbom._validate._document import Json @pytest.mark.parametrize( @@ -65,12 +65,12 @@ def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch, module: str) -> def test_schema_error_names_the_path() -> None: - document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"} + document: Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"} assert schema_error(document) == '"one" is not of type "integer" at $.version' def test_schema_error_quotes_keys_that_are_not_names() -> None: - document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "metadata": {"component": {"bom-ref": ""}}} + document: Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "metadata": {"component": {"bom-ref": ""}}} assert schema_error(document) == '"type" is a required property at $.metadata.component' document = { "bomFormat": "CycloneDX", @@ -81,11 +81,11 @@ def test_schema_error_quotes_keys_that_are_not_names() -> None: def test_schema_error_clips_the_offending_value() -> None: - document: _Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "x" * 1000} + document: Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "x" * 1000} assert schema_error(document) == f'"{"x" * 199} at $.version' -def _url(value: str) -> dict[str, _Json]: +def _url(value: str) -> dict[str, Json]: return {"externalReferences": [{"type": "website", "url": value}]} @@ -112,7 +112,7 @@ def _url(value: str) -> dict[str, _Json]: pytest.param({"serialNumber": "urn:uuid:bde8d34d-2da7-5045-bd69-644208002436\n"}, False, id="pattern-newline"), ], ) -def test_formats_follow_their_rfcs(fields: dict[str, _Json], valid: bool) -> None: +def test_formats_follow_their_rfcs(fields: dict[str, Json], valid: bool) -> None: assert (schema_error({"bomFormat": "CycloneDX", "specVersion": "1.6", **fields}) is None) is valid @@ -125,13 +125,13 @@ def test_formats_follow_their_rfcs(fields: dict[str, _Json], valid: bool) -> Non pytest.param({"version": "1"}, {"version": "2"}, True, id="different"), ], ) -def test_unique_items(first: dict[str, _Json], second: dict[str, _Json], unique: bool) -> None: - components: list[_Json] = [{"type": "library", "name": "a", **first}, {"type": "library", "name": "a", **second}] +def test_unique_items(first: dict[str, Json], second: dict[str, Json], unique: bool) -> None: + components: list[Json] = [{"type": "library", "name": "a", **first}, {"type": "library", "name": "a", **second}] assert (schema_error({"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components}) is None) is unique def test_many_unique_components_pass() -> None: - components: list[_Json] = [ + components: list[Json] = [ {"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(20_000) ] assert schema_error({"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components}) is None From 7648ccbae79e78238f6fd05244b02d533030b4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:54:55 -0800 Subject: [PATCH 072/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20blame=20the=20JSO?= =?UTF-8?q?N=20parser=20for=20deep=20nesting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A RecursionError comes from json.loads near the interpreter's recursion limit, about 1,000 levels, yet the message claimed the 128-level depth check refused the document. Say the JSON parser could not read it. --- src/python_sbom/_validate/_document.py | 5 +++-- tests/_validate/test_document.py | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 7e49a47..e1f2249 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -56,8 +56,9 @@ def validate_document( except json.JSONDecodeError as error: return [f"is not JSON: {error}"] except RecursionError: - # the JSON parser recurses once per level, how deep it gets depends on the Python version - return [f"nests more than {_MAX_DEPTH} levels deep, too deep to check"] + # the JSON parser recurses once per level and gives up near the recursion limit, long before the depth check + # below sees the document + return ["nests too deep for the JSON parser to read"] except Exception as error: # ruff: ignore[blind-except] # a hostile document may trip any check, see above return [f"could not be checked, {type(error).__name__}: {str(error)[:MAX_TEXT]}"] diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py index 555820f..62d9126 100644 --- a/tests/_validate/test_document.py +++ b/tests/_validate/test_document.py @@ -240,9 +240,7 @@ def test_deep_nesting_is_reported(describe: Describe) -> None: def test_nesting_too_deep_to_parse_is_reported() -> None: - assert validate_document("[" * 1_000_000 + "]" * 1_000_000, None) == [ - "nests more than 128 levels deep, too deep to check" - ] + assert validate_document("[" * 1_000_000 + "]" * 1_000_000, None) == ["nests too deep for the JSON parser to read"] @pytest.mark.parametrize( From ab339fb30f996122bf2a589918b287bfc35dadbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:55:28 -0800 Subject: [PATCH 073/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20report=20a=20brok?= =?UTF-8?q?en=20validate=20extra=20as=20missing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit find_spec checks only the extra's top-level modules, so a missing dependency of theirs, such as spdx-tools' beartype or uritools, raised ImportError inside the checks and every SBOM came back as "could not be checked". Raise MissingExtraError from the failed import instead. --- src/python_sbom/_validate/_document.py | 14 +++++++++----- tests/_validate/test_schemas.py | 8 ++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index e1f2249..a05f142 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -65,16 +65,20 @@ def validate_document( def _problems(raw: str, allowed_licenses: frozenset[str] | None, max_components: int) -> list[str]: # every document must conform to the CycloneDX 1.6 schema; one python-sbom wrote must also carry the urn:uuid serial - # actions/attest needs, resolvable and unique bom-refs and a valid SPDX rendering, and meet the license policy a + # actions/attest needs, resolvable and unique bom-refs and a valid SPDX rendering, and meet the license policy. A # missing extra fails every document alike, while importing it, half a second, waits until the cheap census passes + missing_extra: Final[str] = "validation needs the validate extra: pip install 'python-sbom[validate]'" if any(find_spec(name) is None for name in _EXTRA): - msg = "validation needs the validate extra: pip install 'python-sbom[validate]'" - raise MissingExtraError(msg) + raise MissingExtraError(missing_extra) loaded: Final = json.loads(raw) if census := _census_problems(loaded, max_components): return census - from ._licenses import license_problems # ruff: ignore[import-outside-top-level] # the optional extra, see above - from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # the optional extra, see above + # find_spec sees only the top-level modules, not a dependency of theirs that is missing or broken + try: + from ._licenses import license_problems # ruff: ignore[import-outside-top-level] # the optional extra, see above + from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # the optional extra, see above + except ImportError as error: + raise MissingExtraError(missing_extra) from error if error := schema_error(loaded): return [f"does not conform to the CycloneDX 1.6 schema: {error}"] diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index f930168..9a435a8 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -64,6 +64,14 @@ def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch, module: str) -> validate_document("not JSON", None) +def test_validate_with_broken_extra(monkeypatch: pytest.MonkeyPatch) -> None: + # find_spec finds the extra's modules, yet one of their dependencies, such as spdx-tools' beartype, fails to import + monkeypatch.setitem(sys.modules, "python_sbom._validate._schemas", None) + with pytest.raises(MissingExtraError, match=r"pip install 'python-sbom\[validate\]'") as raised: + validate_document('{"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1}', None) + assert isinstance(raised.value.__cause__, ImportError) + + def test_schema_error_names_the_path() -> None: document: Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"} assert schema_error(document) == '"one" is not of type "integer" at $.version' From 760c7afdaf62ce464f3c01888534af8815a8d43b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:58:06 -0800 Subject: [PATCH 074/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20tie=20the=20SBOM?= =?UTF-8?q?=20size=20limit=20to=20the=20component=20cap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate_document refused anything over 16 Mi characters before counting components, while the archive reader allowed 64 MiB members and decoded every sboms/* member into one dict first. At about 540 bytes per component the size check fired near 30,000 components, so a higher --max-components never took effect, and a 2.4 MB wheel of compressible members reached 2.5 GB of memory. byte_limit derives one limit from the cap, 1 MiB plus 1 KiB per component, 103,448,576 bytes at the default 100,000. validate_document takes bytes and checks it first; the reader stops within a chunk past it, and validates the members one at a time, reporting an oversized one as its own problem instead of aborting the run. The docs name both the size and the depth limit. --- docs/usage.rst | 5 ++ src/python_sbom/_validate/_artifact.py | 46 +++++++----- src/python_sbom/_validate/_document.py | 24 +++++-- tests/_validate/test_artifact.py | 51 ++++++++++++-- tests/_validate/test_document.py | 97 +++++++++++++++++--------- tests/_validate/test_schemas.py | 4 +- 6 files changed, 162 insertions(+), 65 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index d2c7b38..92885d7 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -168,3 +168,8 @@ about 5,200. Raise the cap for an SBOM you expect to be larger, or lower it to r .. code-block:: console $ python-sbom validate dist/ --max-components 500000 + +The cap also sets the size limit: 1 KiB per component plus 1 MiB, 103,448,576 bytes at the default, about twice the 54 +MB python-sbom writes for 100,000 components. ``validate`` refuses a larger SBOM before parsing it, and reads the SBOMs +of a wheel one at a time, so memory holds only one however many the wheel packs. It also refuses an SBOM nested more +than 128 levels deep; the SBOMs python-sbom and other tools write nest about a dozen levels. diff --git a/src/python_sbom/_validate/_artifact.py b/src/python_sbom/_validate/_artifact.py index 556e753..a43038d 100644 --- a/src/python_sbom/_validate/_artifact.py +++ b/src/python_sbom/_validate/_artifact.py @@ -1,36 +1,44 @@ from __future__ import annotations import zipfile -from typing import TYPE_CHECKING, Final +from typing import IO, TYPE_CHECKING, Final -from python_sbom._wheel._archive import open_archive, read_member +from python_sbom._wheel._archive import open_archive -from ._document import MAX_COMPONENTS, validate_document +from ._document import MAX_COMPONENTS, byte_limit, validate_document if TYPE_CHECKING: from pathlib import Path -# an SBOM lists every file of its wheel at a few hundred bytes each, so even a wheel with a hundred thousand files stays -# well under this, while one archive member can make validation hold no more than this in memory -_SBOM_LIMIT: Final[int] = 64 * 1024 * 1024 +# what one read may decompress, so a compressed member cannot run far past the limit before the reader notices +_CHUNK: Final[int] = 1024 * 1024 def validate(path: Path, allowed_licenses: frozenset[str] | None, max_components: int = MAX_COMPONENTS) -> list[str]: + limit: Final[int] = byte_limit(max_components) if not zipfile.is_zipfile(path): - return validate_document(path.read_text(encoding="utf-8"), allowed_licenses, max_components) + with path.open("rb") as file: + return validate_document(_read(file, limit), allowed_licenses, max_components) + problems: Final[list[str]] = [] with open_archive(path, path.name) as archive: - found: Final[dict[str, str]] = { - name: read_member(archive, name, path.name, _SBOM_LIMIT).decode("utf-8") - for name in sorted(archive.namelist()) - if _is_sbom(name) - } - if not found: - return [f"no SBOM found in {path.name}"] - return [ - f"{name}: {problem}" - for name, raw in found.items() - for problem in validate_document(raw, allowed_licenses, max_components) - ] + if not (names := sorted(name for name in archive.namelist() if _is_sbom(name))): + return [f"no SBOM found in {path.name}"] + # one at a time, so memory holds a single SBOM however many the archive packs + for name in names: + with archive.open(name) as member: + raw = _read(member, limit) + problems.extend( + f"{name}: {problem}" for problem in validate_document(raw, allowed_licenses, max_components) + ) + return problems + + +def _read(source: IO[bytes], limit: int) -> bytes: + # stops within a chunk past the limit: enough for validate_document to refuse the SBOM without reading all of it + content: Final[bytearray] = bytearray() + while len(content) <= limit and (chunk := source.read(_CHUNK)): + content.extend(chunk) + return bytes(content) def _is_sbom(name: str) -> bool: diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index a05f142..105cfb0 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -29,8 +29,6 @@ ("dependencies",), ) _SERIAL: Final[re.Pattern[str]] = re.compile(r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") -# the archive reader stops at the same size -_MAX_CHARACTERS: Final[int] = 16 * 1024 * 1024 # every check takes time in proportion to the count, so the cap bounds the work an untrusted document can ask for, far # above the few thousand components a wheel with its build tools' files lists MAX_COMPONENTS: Final[int] = 100_000 @@ -43,11 +41,22 @@ _MAX_DEPTH: Final[int] = 128 +def byte_limit(max_components: int) -> int: + # python-sbom writes about 540 bytes per component, since every file of the wheel and of its build tools is one; a + # KiB each leaves room for documents with fuller components, plus a MiB for the metadata, dependencies and + # formulation beside them, while tying what the JSON parser may be asked to hold to the component cap + return 1024 * 1024 + max_components * 1024 + + def validate_document( - raw: str, allowed_licenses: frozenset[str] | None, max_components: int = MAX_COMPONENTS + raw: bytes, allowed_licenses: frozenset[str] | None, max_components: int = MAX_COMPONENTS ) -> list[str]: - if len(raw) > _MAX_CHARACTERS: - return [f"is {len(raw)} characters long, over the {_MAX_CHARACTERS} character limit"] + if len(raw) > (limit := byte_limit(max_components)): + problem = ( + f"is over {limit} bytes, the size limit for {max_components} components;" + " pass a higher --max-components to check it" + ) + return [problem] # one document's problem never stops the others from being checked, so whatever the checks raise is reported try: return _problems(raw, allowed_licenses, max_components) @@ -63,14 +72,14 @@ def validate_document( return [f"could not be checked, {type(error).__name__}: {str(error)[:MAX_TEXT]}"] -def _problems(raw: str, allowed_licenses: frozenset[str] | None, max_components: int) -> list[str]: +def _problems(raw: bytes, allowed_licenses: frozenset[str] | None, max_components: int) -> list[str]: # every document must conform to the CycloneDX 1.6 schema; one python-sbom wrote must also carry the urn:uuid serial # actions/attest needs, resolvable and unique bom-refs and a valid SPDX rendering, and meet the license policy. A # missing extra fails every document alike, while importing it, half a second, waits until the cheap census passes missing_extra: Final[str] = "validation needs the validate extra: pip install 'python-sbom[validate]'" if any(find_spec(name) is None for name in _EXTRA): raise MissingExtraError(missing_extra) - loaded: Final = json.loads(raw) + loaded: Final = json.loads(raw.decode("utf-8")) if census := _census_problems(loaded, max_components): return census # find_spec sees only the top-level modules, not a dependency of theirs that is missing or broken @@ -194,6 +203,7 @@ def _walk(components: list[Component]) -> list[Component]: "MAX_COMPONENTS", "MAX_TEXT", "Json", + "byte_limit", "is_document", "validate_document", ] diff --git a/tests/_validate/test_artifact.py b/tests/_validate/test_artifact.py index 96f17fa..5bd570e 100644 --- a/tests/_validate/test_artifact.py +++ b/tests/_validate/test_artifact.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import tracemalloc import zipfile from typing import TYPE_CHECKING, Final @@ -11,6 +12,7 @@ from python_sbom._validate._artifact import validate if TYPE_CHECKING: + from collections.abc import Callable from pathlib import Path from conftest import Describe, WriteWheel @@ -64,11 +66,50 @@ def test_sbom_of_every_dist_info_is_checked(tmp_path: Path, write_wheel: WriteWh ] -def test_oversized_sbom(tmp_path: Path, write_wheel: WriteWheel) -> None: - sbom = b" " * (64 * 1024 * 1024 + 1) - wheel = write_wheel(tmp_path / "example-1.0-py3-none-any.whl", {**_METADATA, "example.cdx.json": sbom}) - with pytest.raises(WheelError, match=r"example\.cdx\.json is over the 67108864 byte limit"): - validate(wheel, None) +def test_oversized_sbom_is_reported_and_the_rest_checked(tmp_path: Path, write_wheel: WriteWheel) -> None: + # 64 MiB of whitespace compresses to a few KiB; reading stops past the 1,058,816 bytes ten components allow + sboms = { + "example-1.0.dist-info/sboms/big.cdx.json": b" " * (64 * 1024 * 1024), + "example-1.0.dist-info/sboms/small.cdx.json": b"not JSON", + } + wheel = write_wheel(tmp_path / "example-1.0-py3-none-any.whl", {**_METADATA, **sboms}) + problems, peak = traced(lambda: validate(wheel, None, max_components=10)) + assert problems == [ + ( + "example-1.0.dist-info/sboms/big.cdx.json: is over 1058816 bytes, the size limit for 10 components;" + " pass a higher --max-components to check it" + ), + "example-1.0.dist-info/sboms/small.cdx.json: is not JSON: Expecting value: line 1 column 1 (char 0)", + ] + assert peak < 8 * 1024 * 1024 + + +def traced(run: Callable[[], list[str]]) -> tuple[list[str], int]: + tracemalloc.start() + try: + result = run() + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + return result, peak + + +def test_sboms_are_held_one_at_a_time(tmp_path: Path, write_wheel: WriteWheel) -> None: + # each compresses to a few KiB, so a small wheel packs many SBOMs that together decompress to 64 MiB + sboms = {f"example-1.0.dist-info/sboms/x{index:02}.cdx.json": b" " * (2 * 1024 * 1024) for index in range(32)} + wheel = write_wheel(tmp_path / "example-1.0-py3-none-any.whl", {**_METADATA, **sboms}, record=False) + problems, peak = traced(lambda: validate(wheel, None)) + assert problems == [ + f"{name}: is not JSON: Expecting value: line 1 column 2097153 (char 2097152)" for name in sorted(sboms) + ] + assert peak < 16 * 1024 * 1024 + + +def test_oversized_json_file(tmp_path: Path) -> None: + (sbom := tmp_path / "example.cdx.json").write_bytes(b" " * 1_058_817) + assert validate(sbom, None, max_components=10) == [ + "is over 1058816 bytes, the size limit for 10 components; pass a higher --max-components to check it" + ] def test_archive_with_duplicate_member(tmp_path: Path) -> None: diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py index 62d9126..f873314 100644 --- a/tests/_validate/test_document.py +++ b/tests/_validate/test_document.py @@ -23,7 +23,7 @@ from python_sbom._validate._document import Json # what auditwheel writes: CycloneDX 1.4 conventions, a legacy tools array, no timestamp -_FOREIGN: Final[str] = json.dumps({ +_FOREIGN: Final[bytes] = json.dumps({ "bomFormat": "CycloneDX", "specVersion": "1.4", "version": 1, @@ -33,19 +33,21 @@ }, "components": [{"type": "library", "bom-ref": "pkg:rpm/almalinux/libfoo@1", "name": "libfoo", "version": "1"}], "dependencies": [{"ref": "pkg:pypi/plain@1.0", "dependsOn": ["pkg:rpm/almalinux/libfoo@1"]}], -}) +}).encode() def test_valid(describe: Describe) -> None: - assert validate_document(dumps(describe("Requires-Dist: click\nLicense-Expression: MIT\n")).decode(), None) == [] + assert validate_document(dumps(describe("Requires-Dist: click\nLicense-Expression: MIT\n")), None) == [] def test_not_json() -> None: - assert validate_document("this is not json", None) == ["is not JSON: Expecting value: line 1 column 1 (char 0)"] + assert validate_document(b"this is not json", None) == ["is not JSON: Expecting value: line 1 column 1 (char 0)"] def test_schema_violation() -> None: - (problem,) = validate_document(json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"}), None) + (problem,) = validate_document( + json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"}).encode(), None + ) assert problem == 'does not conform to the CycloneDX 1.6 schema: "one" is not of type "integer" at $.version' @@ -53,9 +55,22 @@ def test_other_tools_documents_get_the_schema_check_alone() -> None: assert validate_document(_FOREIGN, None) == [] -def test_oversized_document_is_not_parsed() -> None: - assert validate_document(" " * (16 * 1024 * 1024 + 1), None) == [ - "is 16777217 characters long, over the 16777216 character limit" +@pytest.mark.parametrize( + ("max_components", "limit"), + [pytest.param(100_000, 103_448_576, id="default"), pytest.param(10, 1_058_816, id="lowered")], +) +def test_oversized_document_is_not_parsed(max_components: int, limit: int) -> None: + assert validate_document(b" " * (limit + 1), None, max_components) == [ + ( + f"is over {limit} bytes, the size limit for {max_components} components; pass a higher --max-components to" + " check it" + ) + ] + + +def test_document_at_the_size_limit_is_parsed() -> None: + assert validate_document(b" " * 1_058_816, None, 10) == [ + "is not JSON: Expecting value: line 1 column 1058817 (char 1058816)" ] @@ -65,7 +80,7 @@ def test_component_cap_counts_nested_components() -> None: ] -def many_components(groups: int) -> str: +def many_components(groups: int) -> bytes: # groups of 100 parts each components = [ { @@ -75,7 +90,7 @@ def many_components(groups: int) -> str: } for group in range(groups) ] - return json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "components": components}) + return json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "components": components}).encode() def test_component_cap_can_be_raised() -> None: @@ -88,24 +103,38 @@ def test_component_cap_can_be_lowered() -> None: ] -def test_sbom_the_size_of_virtualenvs_passes_by_default(describe: Describe, facts: BuildFacts) -> None: - # virtualenv's wheel lists 5,207 components and its zipapp 5,452, nearly all of them files of its build tools +@pytest.mark.parametrize( + ("files", "expected"), + [ + # virtualenv's wheel lists 5,207 components and its zipapp 5,452, nearly all of them files of its build tools + pytest.param(5_500, [], id="virtualenv"), + # the interpreter and the tool add two, so these reach the default cap, at about 54 MB, and one more file + # passes it; the size limit must leave both to the component count + pytest.param(99_998, [], id="at-the-cap"), + pytest.param( + 99_999, + ["lists more than 100000 components, nested ones included; pass a higher --max-components to check it"], + id="over-the-cap", + ), + ], +) +def test_sbom_the_default_cap_admits(describe: Describe, facts: BuildFacts, files: int, expected: list[str]) -> None: rows = [ RecordRow(path=f"tool/module_{index}.py", algorithm="SHA-256", digest="0" * 64, size="1") - for index in range(5_500) + for index in range(files) ] tool = InstalledDistribution( name="tool", metadata=parse_metadata("Name: tool\nVersion: 1.0\n"), files=rows, depends_on=[] ) document = describe(facts=dataclasses.replace(facts, distributions=[tool])) - assert validate_document(dumps(document).decode(), None) == [] + assert validate_document(dumps(document), None) == expected def test_missing_fields(describe: Describe) -> None: # the schema makes the timestamp optional, so the loaded JSON, not the typed document, loses it loaded = json.loads(dumps(describe())) del loaded["metadata"]["timestamp"] - assert validate_document(json.dumps(loaded), None) == [ + assert validate_document(json.dumps(loaded).encode(), None) == [ "lacks metadata.timestamp, which every document python-sbom writes carries" ] @@ -114,7 +143,9 @@ def test_serial_is_required(describe: Describe) -> None: # the schema makes it optional, so the loaded JSON, not the typed document, loses it loaded = json.loads(dumps(describe())) del loaded["serialNumber"] - assert validate_document(json.dumps(loaded), None) == ["serialNumber must be a urn:uuid: RFC 4122 UUID, got ''"] + assert validate_document(json.dumps(loaded).encode(), None) == [ + "serialNumber must be a urn:uuid: RFC 4122 UUID, got ''" + ] def test_duplicate_refs(describe: Describe) -> None: @@ -124,23 +155,25 @@ def test_duplicate_refs(describe: Describe) -> None: ] -def with_components(document: Document, *components: Component) -> str: +def with_components(document: Document, *components: Component) -> bytes: document["components"].extend(components) document["dependencies"][0].setdefault("dependsOn", []).extend(component["bom-ref"] for component in components) - return dumps(document).decode() + return dumps(document) def test_duplicate_refs_in_other_tools_documents() -> None: foreign = json.loads(_FOREIGN) foreign["components"].append({"type": "library", "bom-ref": "pkg:pypi/plain@1.0", "name": "copy"}) - assert validate_document(json.dumps(foreign), None) == ["bom-ref values are not unique: ['pkg:pypi/plain@1.0']"] + assert validate_document(json.dumps(foreign).encode(), None) == [ + "bom-ref values are not unique: ['pkg:pypi/plain@1.0']" + ] def test_component_without_bom_ref(describe: Describe) -> None: # the schema makes the bom-ref optional, so the loaded JSON, not the typed document, loses it loaded = json.loads(dumps(describe())) loaded["components"].append({"type": "library", "name": "nameless"}) - assert validate_document(json.dumps(loaded), None) == ["components lack a bom-ref: ['nameless']"] + assert validate_document(json.dumps(loaded).encode(), None) == ["components lack a bom-ref: ['nameless']"] @pytest.mark.parametrize( @@ -173,13 +206,13 @@ def test_component_without_bom_ref(describe: Describe) -> None: def test_schema_valid_variations_pass(describe: Describe, mutate: Callable[[dict[str, Json]], None]) -> None: loaded = json.loads(dumps(describe())) mutate(loaded) - assert validate_document(json.dumps(loaded), None) == [] + assert validate_document(json.dumps(loaded).encode(), None) == [] def test_unknown_dependency_refs(describe: Describe) -> None: document = describe() document["dependencies"].append({"ref": "ghost", "dependsOn": ["phantom"]}) - assert validate_document(dumps(document).decode(), None) == [ + assert validate_document(dumps(document), None) == [ "dependencies reference bom-refs that do not exist: ['ghost', 'phantom']" ] @@ -187,15 +220,13 @@ def test_unknown_dependency_refs(describe: Describe) -> None: def test_root_must_depend_on_components(describe: Describe) -> None: document = describe("Requires-Dist: click\n") document["dependencies"][0]["dependsOn"] = [] - assert validate_document(dumps(document).decode(), None) == [ - "root dependsOn [] != components ['requires-dist:click']" - ] + assert validate_document(dumps(document), None) == ["root dependsOn [] != components ['requires-dist:click']"] def test_workflow_must_reference_tools(describe: Describe) -> None: document = describe() document.get("formulation", [])[0]["workflows"][0]["resourceReferences"].append({"ref": "pkg:pypi/example@1.0"}) - assert validate_document(dumps(document).decode(), None) == [ + assert validate_document(dumps(document), None) == [ "workflow wheel-build references unknown tools: ['pkg:pypi/example@1.0']" ] @@ -203,14 +234,14 @@ def test_workflow_must_reference_tools(describe: Describe) -> None: def test_formula_without_workflows(describe: Describe) -> None: loaded = json.loads(dumps(describe())) loaded["formulation"] = [{"bom-ref": "formula:other"}] - assert validate_document(json.dumps(loaded), None) == [] + assert validate_document(json.dumps(loaded).encode(), None) == [] def test_spdx_rendering_must_validate(describe: Describe) -> None: document = describe() properties = document["metadata"].get("properties", []) properties.append({"name": "example:sbom:repository", "value": "https://example.org/a#fragment"}) - (problem,) = validate_document(dumps(document).decode(), None) + (problem,) = validate_document(dumps(document), None) assert problem.startswith("SPDX rendering invalid: ") @@ -235,12 +266,14 @@ def test_deep_nesting_is_reported(describe: Describe) -> None: f'{{"type": "library", "bom-ref": "deep-{level}", "name": "deep", "components": [' for level in range(100) ) nested += "]}" * 100 - raw = json.dumps(describe()).replace('"components": []', f'"components": [{nested}]', 1) + raw = json.dumps(describe()).replace('"components": []', f'"components": [{nested}]', 1).encode() assert validate_document(raw, None) == ["nests more than 128 levels deep, too deep to check"] def test_nesting_too_deep_to_parse_is_reported() -> None: - assert validate_document("[" * 1_000_000 + "]" * 1_000_000, None) == ["nests too deep for the JSON parser to read"] + assert validate_document(b"[" * 1_000_000 + b"]" * 1_000_000, None) == [ + "nests too deep for the JSON parser to read" + ] @pytest.mark.parametrize( @@ -248,7 +281,7 @@ def test_nesting_too_deep_to_parse_is_reported() -> None: [pytest.param("NaN", id="nan"), pytest.param("Infinity", id="infinity"), pytest.param("-Infinity", id="minus")], ) def test_non_finite_numbers_are_rejected(constant: str) -> None: - raw = f'{{"bomFormat": "CycloneDX", "specVersion": "1.6", "version": {constant}}}' + raw = f'{{"bomFormat": "CycloneDX", "specVersion": "1.6", "version": {constant}}}'.encode() assert validate_document(raw, None) == [f"holds {float(constant)}, which is not a JSON number"] @@ -261,7 +294,7 @@ def test_license_policy(describe: Describe) -> None: def test_unexpected_error_is_reported(describe: Describe, mocker: MockerFixture) -> None: mocker.patch("python_sbom._validate._document.render_spdx", autospec=True, side_effect=RuntimeError("x" * 300)) - assert validate_document(dumps(describe()).decode(), None) == [f"could not be checked, RuntimeError: {'x' * 200}"] + assert validate_document(dumps(describe()), None) == [f"could not be checked, RuntimeError: {'x' * 200}"] @pytest.mark.parametrize( diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index 9a435a8..dda9913 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -61,14 +61,14 @@ def test_validate_without_extra(monkeypatch: pytest.MonkeyPatch, module: str) -> # would reject reports the missing extra monkeypatch.setitem(sys.modules, module, None) with pytest.raises(MissingExtraError, match=r"pip install 'python-sbom\[validate\]'"): - validate_document("not JSON", None) + validate_document(b"not JSON", None) def test_validate_with_broken_extra(monkeypatch: pytest.MonkeyPatch) -> None: # find_spec finds the extra's modules, yet one of their dependencies, such as spdx-tools' beartype, fails to import monkeypatch.setitem(sys.modules, "python_sbom._validate._schemas", None) with pytest.raises(MissingExtraError, match=r"pip install 'python-sbom\[validate\]'") as raised: - validate_document('{"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1}', None) + validate_document(b'{"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1}', None) assert isinstance(raised.value.__cause__, ImportError) From bba79562ae4e80a673afa6e567358e05f5e1f3ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 07:58:56 -0800 Subject: [PATCH 075/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20report=20an=20SBO?= =?UTF-8?q?M=20that=20is=20not=20UTF-8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decoding a member happened outside the per-document try, so a non-UTF-8 SBOM ended validate with a UnicodeDecodeError traceback. The bytes now decode inside it and the SBOM is reported as not UTF-8, with the reason and offset, while the others are still checked. --- src/python_sbom/_validate/_document.py | 2 ++ tests/_validate/test_artifact.py | 12 ++++++++++++ tests/_validate/test_document.py | 4 ++++ 3 files changed, 18 insertions(+) diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 105cfb0..389ac50 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -62,6 +62,8 @@ def validate_document( return _problems(raw, allowed_licenses, max_components) except MissingExtraError: raise + except UnicodeDecodeError as error: + return [f"is not UTF-8: {error.reason} at byte {error.start}"] except json.JSONDecodeError as error: return [f"is not JSON: {error}"] except RecursionError: diff --git a/tests/_validate/test_artifact.py b/tests/_validate/test_artifact.py index 5bd570e..a0ce0cb 100644 --- a/tests/_validate/test_artifact.py +++ b/tests/_validate/test_artifact.py @@ -112,6 +112,18 @@ def test_oversized_json_file(tmp_path: Path) -> None: ] +def test_sbom_not_utf8_is_reported_and_the_rest_checked(tmp_path: Path, write_wheel: WriteWheel) -> None: + sboms = { + "example-1.0.dist-info/sboms/utf16.cdx.json": b"\xff\xfe{}", + "example-1.0.dist-info/sboms/other.cdx.json": b"not JSON", + } + wheel = write_wheel(tmp_path / "example-1.0-py3-none-any.whl", {**_METADATA, **sboms}) + assert validate(wheel, None) == [ + "example-1.0.dist-info/sboms/other.cdx.json: is not JSON: Expecting value: line 1 column 1 (char 0)", + "example-1.0.dist-info/sboms/utf16.cdx.json: is not UTF-8: invalid start byte at byte 0", + ] + + def test_archive_with_duplicate_member(tmp_path: Path) -> None: with zipfile.ZipFile(wheel := tmp_path / "example-1.0-py3-none-any.whl", "w") as archive: archive.writestr("example.cdx.json", b"{}") diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py index f873314..eb302e3 100644 --- a/tests/_validate/test_document.py +++ b/tests/_validate/test_document.py @@ -44,6 +44,10 @@ def test_not_json() -> None: assert validate_document(b"this is not json", None) == ["is not JSON: Expecting value: line 1 column 1 (char 0)"] +def test_not_utf8() -> None: + assert validate_document(b'{"name": "\xe9t\xe9"}', None) == ["is not UTF-8: invalid continuation byte at byte 10"] + + def test_schema_violation() -> None: (problem,) = validate_document( json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"}).encode(), None From 9a611255684496030733f9b3274aa8b03eaa5644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:02:45 -0800 Subject: [PATCH 076/115] =?UTF-8?q?=E2=9A=A1=20perf:=20check=20the=20SPDX?= =?UTF-8?q?=20rendering=20in=20linear=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit spdx-tools' validate_full_spdx_document looks every relationship end and every package's relationships up in lists, and lists the LicenseRef ids for each license expression, so its time grows with the square of the elements: 0.6 s for 1,000 packages, 6.9 s for 4,000 and 100 s for 16,000, putting the 100,000 cap at about an hour. Keep its parser and its validators of each element, and do the lookups across elements with sets: relationship ends, duplicate ids and license references. The rendering writes no files, snippets or annotations and always a DESCRIBES relationship, so those checks drop out. A test pins the messages to spdx-tools' own on mutated renderings; 100,000 packages now take about five seconds. The docs give the measured times. --- docs/usage.rst | 9 ++-- src/python_sbom/_validate/_schemas.py | 69 +++++++++++++++++++++++++-- tests/_validate/test_schemas.py | 67 +++++++++++++++++++++++++- 3 files changed, 137 insertions(+), 8 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 92885d7..780d02b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -160,10 +160,11 @@ Each ``--allow-license`` adds an SPDX id that bundled wheels may use. With any g ids do not allow fails the check, and so does one that declares a license only by name. Before anything else, ``validate`` counts the components an SBOM lists, nested ones included, and fails an SBOM that -lists more than 100,000 without checking it further. Every check takes time in proportion to the count, about a tenth of -a second for 100,000 components, so the cap bounds what an untrusted SBOM can ask of a CI job while staying far above -real SBOMs: every file of the wheel, of each bundled wheel and of each build tool counts, and virtualenv's wheel lists -about 5,200. Raise the cap for an SBOM you expect to be larger, or lower it to refuse more: +lists more than 100,000 without checking it further. Every check takes time in proportion to the count: 100,000 files +take about a second, and 100,000 packages, each also checked in the SPDX rendering, about six. The cap thus bounds what +an untrusted SBOM can ask of a CI job while staying far above real SBOMs: every file of the wheel, of each bundled wheel +and of each build tool counts, and virtualenv's wheel lists about 5,200. Raise the cap for an SBOM you expect to be +larger, or lower it to refuse more: .. code-block:: console diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index 398c818..9174ce4 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -1,13 +1,19 @@ from __future__ import annotations import json +from collections import Counter from importlib.resources import files from typing import TYPE_CHECKING, Final from jsonschema_rs import Draft7Validator, Registry +from license_expression import ExpressionError, ExpressionParseError, LicenseExpression +from spdx_tools.common.spdx_licensing import spdx_licensing from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser -from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document +from spdx_tools.spdx.validation.creation_info_validator import validate_creation_info +from spdx_tools.spdx.validation.extracted_licensing_info_validator import validate_extracted_licensing_infos +from spdx_tools.spdx.validation.package_validator import validate_package +from spdx_tools.spdx.validation.spdx_id_validators import validate_spdx_id from ._document import MAX_TEXT @@ -15,6 +21,7 @@ from importlib.resources.abc import Traversable from jsonschema_rs import ValidationError + from spdx_tools.spdx.model import Document, SpdxNoAssertion, SpdxNone from python_sbom._formats._spdx import Spdx @@ -34,10 +41,66 @@ def schema_error(document: Json) -> str | None: def spdx_problems(spdx: Spdx) -> list[str]: try: - parsed: Final = JsonLikeDictParser().parse(dict(spdx)) + parsed: Final[Document] = JsonLikeDictParser().parse(dict(spdx)) except SPDXParsingError as error: return [f"SPDX rendering does not parse: {message}" for message in error.get_messages()] - return [f"SPDX rendering invalid: {message.validation_message}" for message in validate_full_spdx_document(parsed)] + return [f"SPDX rendering invalid: {message}" for message in _invalid(parsed)] + + +def _invalid(document: Document) -> list[str]: + # spdx-tools' validate_full_spdx_document looks every relationship end, and every package's own relationships, up + # in lists, so its time grows with the square of the elements: 16,000 took a minute and a half. This runs its + # validators of each element and does the lookups across elements in sets. The rendering writes no files, snippets + # or annotations, and always the DESCRIBES relationship, so their checks are left out + version: Final[str] = document.creation_info.spdx_version + messages: Final[list[str]] = [ + message.validation_message for message in validate_creation_info(document.creation_info, version) + ] + license_refs: Final[frozenset[str | None]] = frozenset( + info.license_id for info in document.extracted_licensing_info + ) + for package in document.packages: + messages.extend(validate_spdx_id(package.spdx_id, document)) + messages.extend(_license_problems(package.license_concluded, license_refs)) + messages.extend(_license_problems(package.license_declared, license_refs)) + messages.extend(message.validation_message for message in validate_package(package, version)) + ids: Final[Counter[str]] = Counter(package.spdx_id for package in document.packages) + ids[document.creation_info.spdx_id] += 1 + for relationship in document.relationships: + ends = (relationship.spdx_element_id, relationship.related_spdx_element_id) + messages.extend( + f'did not find the referenced spdx_id "{end}" in the SPDX document' for end in ends if end not in ids + ) + messages.extend( + message.validation_message for message in validate_extracted_licensing_infos(document.extracted_licensing_info) + ) + if duplicates := sorted(spdx_id for spdx_id, count in ids.items() if count > 1): + messages.append( + f"every spdx_id must be unique within the document, but found the following duplicates: {duplicates}" + ) + return messages + + +def _license_problems( + expression: LicenseExpression | SpdxNoAssertion | SpdxNone | None, license_refs: frozenset[str | None] +) -> list[str]: + # spdx-tools' validate_license_expression, with the LicenseRef ids it lists for every call gathered once + if not isinstance(expression, LicenseExpression): + return [] + problems: Final[list[str]] = [ + f"Unrecognized license reference: {symbol}. license_expression must only use IDs from the license list or" + f" extracted licensing info, but is: {expression}" + for symbol in spdx_licensing.validate(expression).invalid_symbols + if symbol not in license_refs + ] + try: + spdx_licensing.parse(str(expression), validate=True, strict=True) + except ExpressionParseError as error: + # an exception used as a license or the other way around; unknown symbols raise ExpressionError, listed above + problems.append(f"{error}. for license_expression: {expression}") + except ExpressionError: + pass + return problems # the official CycloneDX schema files, unchanged: bom-1.6 and the jsf-0.82 it references from the specification's 1.6.2 diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index dda9913..e443a0b 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -6,12 +6,21 @@ from typing import TYPE_CHECKING import pytest +from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import JsonLikeDictParser +from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document from python_sbom._errors import MissingExtraError +from python_sbom._formats._spdx import render_spdx from python_sbom._validate._document import validate_document -from python_sbom._validate._schemas import schema_error +from python_sbom._validate._schemas import schema_error, spdx_problems if TYPE_CHECKING: + from collections.abc import Callable + + from conftest import Describe + + from python_sbom import Component + from python_sbom._formats._spdx import Spdx from python_sbom._validate._document import Json @@ -143,3 +152,59 @@ def test_many_unique_components_pass() -> None: {"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(20_000) ] assert schema_error({"bomFormat": "CycloneDX", "specVersion": "1.6", "components": components}) is None + + +def _declare(spdx: Spdx, expression: str, text: str | None = None) -> None: + spdx["packages"][0]["licenseDeclared"] = expression + if text is not None: + spdx["hasExtractedLicensingInfos"] = [ + {"licenseId": "LicenseRef-Custom", "extractedText": text, "name": "custom"} + ] + + +@pytest.mark.parametrize( + ("mutate", "valid"), + [ + pytest.param(lambda _: None, True, id="as-rendered"), + pytest.param( + lambda spdx: spdx.update(documentNamespace="https://example.org/a#fragment"), False, id="namespace" + ), + pytest.param( + lambda spdx: spdx["relationships"].append({ + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-ghost", + }), + False, + id="unknown-relationship-end", + ), + pytest.param(lambda spdx: spdx["packages"].append(spdx["packages"][0]), False, id="duplicate-id"), + pytest.param( + lambda spdx: spdx["packages"][0].update(checksums=[{"algorithm": "SHA256", "checksumValue": "ab"}]), + False, + id="short-checksum", + ), + pytest.param(lambda spdx: _declare(spdx, "MIT AND Foo-Bar"), False, id="unknown-license"), + pytest.param(lambda spdx: _declare(spdx, "GPL-2.0-only WITH MIT"), False, id="license-as-exception"), + pytest.param(lambda spdx: _declare(spdx, "MIT OR LicenseRef-Custom", "custom"), True, id="license-ref"), + pytest.param(lambda spdx: _declare(spdx, "MIT OR LicenseRef-Custom", ""), False, id="license-ref-without-text"), + ], +) +def test_spdx_problems_match_spdx_tools(describe: Describe, mutate: Callable[[Spdx], None], valid: bool) -> None: + # the full validation of spdx-tools, whose time grows with the square of the elements, is the reference + spdx = render_spdx(describe("Requires-Dist: click\n"), "1") + mutate(spdx) + reference = validate_full_spdx_document(JsonLikeDictParser().parse(dict(spdx))) + assert spdx_problems(spdx) == [f"SPDX rendering invalid: {message.validation_message}" for message in reference] + assert (not reference) is valid + + +def test_many_packages_pass(describe: Describe) -> None: + # spdx-tools' full validation took a minute and a half for 16,000 packages, so this guards the linear checks + document = describe() + libraries: list[Component] = [ + {"type": "library", "bom-ref": f"r{index}", "name": f"n{index}"} for index in range(20_000) + ] + document["components"].extend(libraries) + document["dependencies"][0].setdefault("dependsOn", []).extend(f"r{index}" for index in range(20_000)) + assert spdx_problems(render_spdx(document, "1")) == [] From 6e12241440333572807d8bef7ed07e17b9cd9578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:03:13 -0800 Subject: [PATCH 077/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(model):?= =?UTF-8?q?=20share=20one=20component=20walker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vendored_dependencies walked the tree breadth first with list.pop(0), quadratic in the component count, and the SPDX renderer kept its own walker. Both now use walk_components from _model._document, depth first in document order, so a bundled wheel's vendored dependencies list in the order its component appears. --- src/python_sbom/_formats/_spdx.py | 20 +++++++------------- src/python_sbom/_model/_components.py | 19 +++++++------------ src/python_sbom/_model/_document.py | 11 ++++++----- tests/_model/test_components.py | 18 ++++++++++++++++++ 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/python_sbom/_formats/_spdx.py b/src/python_sbom/_formats/_spdx.py index e0944a9..3967258 100644 --- a/src/python_sbom/_formats/_spdx.py +++ b/src/python_sbom/_formats/_spdx.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Final, NotRequired, TypedDict +from python_sbom._model._document import walk_components from python_sbom._model._names import SBOM_FILE, SBOM_REPOSITORY, property_prefix, sbom_property from python_sbom.version import __version__ @@ -125,7 +126,12 @@ def render_spdx(bom: Document, tool_version: str) -> Spdx: repository: Final[str] = recorded.get(sbom_property(project, SBOM_REPOSITORY), "https://spdx.org/spdxdocs") sbom_file: Final[str] = recorded.get(sbom_property(project, SBOM_FILE), "CycloneDX JSON") tools, tool_creators = _tools(metadata) - components: Final[list[Component]] = _walk([root, *bom.get("components", []), *tools]) + # file components mirror wheel RECORD entries, which carry no SHA-1 and so cannot become SPDX files + components: Final[list[Component]] = [ + component + for component in walk_components([root, *bom.get("components", []), *tools]) + if component["type"] != "file" + ] packages: Final[list[_Package]] = [_package(component, f"{project}:") for component in components] spdx: Final[Spdx] = { "spdxVersion": "SPDX-2.3", @@ -167,18 +173,6 @@ def _creator(name: str, version: str | None) -> str: return f"Tool: {name}" if version is None else f"Tool: {name}-{version}" -def _walk(components: list[Component]) -> list[Component]: - # iterative, so a deeply nested hostile document cannot exhaust the stack; file components mirror wheel RECORD - # entries, which carry no SHA-1 and so cannot become SPDX files - found: Final[list[Component]] = [] - pending: Final[list[Component]] = list(reversed(components)) - while pending: - if (component := pending.pop())["type"] != "file": - found.append(component) - pending.extend(reversed(component.get("components", []))) - return found - - def _package(component: Component, property_prefix: str) -> _Package: # a component carries either one SPDX expression, or the free-form names from License/classifier metadata plus the # SPDX ids concluded from them diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index 371520a..1ca1b8a 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Final from urllib.parse import urlsplit +from ._document import walk_components from ._licenses import licenses from ._names import property_prefix, purl, pypi_url, reference_type, tool_ref from ._types import Dependency @@ -131,19 +132,13 @@ def file_component(parent_ref: str, row: RecordRow) -> Component: def vendored_dependencies(components: list[Component]) -> list[Dependency]: # each bundled wheel depends on the packages it vendors, wherever it sits in the document found: Final[list[Dependency]] = [] - pending = list(components) - while pending: - component = pending.pop(0) - children = component.get("components", []) - if component["bom-ref"].startswith("bundled:") and ( - vendored := [ - child["bom-ref"] - for child in children - if child["type"] == "library" and "#vendored/" in child["bom-ref"] - ] - ): + for component in walk_components(components): + if not component["bom-ref"].startswith("bundled:"): + continue + # a file's ref holds its path, which may itself hold "#vendored/" + libraries = (child["bom-ref"] for child in component.get("components", []) if child["type"] == "library") + if vendored := [ref for ref in libraries if "#vendored/" in ref]: found.append(Dependency(ref=component["bom-ref"], dependsOn=vendored)) - pending.extend(children) return found diff --git a/src/python_sbom/_model/_document.py b/src/python_sbom/_model/_document.py index 390be2c..30e1335 100644 --- a/src/python_sbom/_model/_document.py +++ b/src/python_sbom/_model/_document.py @@ -19,7 +19,7 @@ def with_header(body: Body, namespace: uuid.UUID) -> Document: # a provider contributes refs of its own, so the document is checked before it is written rather than by validate components: Final[list[Component]] = [ - *_walk_components([ + *walk_components([ body["metadata"]["component"], *body["components"], *body["metadata"]["tools"]["components"], @@ -56,9 +56,9 @@ def with_header(body: Body, namespace: uuid.UUID) -> Document: } -def _walk_components(components: Iterable[Component]) -> Iterator[Component]: - # depth first, each component before the ones it nests, in document order; iterative, so deep nesting cannot exhaust - # the stack +def walk_components(components: Iterable[Component]) -> Iterator[Component]: + # depth first, each component before the ones it nests, in document order; iterative, so a deeply nested document + # cannot exhaust the stack pending: Final[list[Component]] = list(reversed(list(components))) while pending: component = pending.pop() @@ -69,7 +69,7 @@ def _walk_components(components: Iterable[Component]) -> Iterator[Component]: def checked_contributions(contributions: Contributions, provider: ContentProvider) -> Contributions: # what a provider adds is checked for the fields everything else here reads, so a mistake names the provider rather # than surfacing as a KeyError deep in the build - for component in _walk_components([*contributions.components, *contributions.tools]): + for component in walk_components([*contributions.components, *contributions.tools]): if missing := [key for key in ("type", "bom-ref", "name") if key not in component]: name = type(provider).__qualname__ msg = f"content provider {name} added a component without {', '.join(missing)}: {component!r:.200}" @@ -114,5 +114,6 @@ def timestamp(facts: BuildFacts) -> str: "contributed_properties", "formulation", "timestamp", + "walk_components", "with_header", ] diff --git a/tests/_model/test_components.py b/tests/_model/test_components.py index 0edf0d4..c6439c4 100644 --- a/tests/_model/test_components.py +++ b/tests/_model/test_components.py @@ -9,6 +9,7 @@ bundled_wheel_component, component_from_metadata, script_tool, + vendored_dependencies, ) from python_sbom._model._inputs import BundledWheel, RecordRow from python_sbom._model._metadata import parse_metadata @@ -345,3 +346,20 @@ def test_vendored_spellings_of_one_package_merge() -> None: def test_pypi_purl_keeps_dots() -> None: component = component_from_metadata(parse_metadata("Name: zope.interface\nVersion: 7.0\n")) assert component["purl"] == "pkg:pypi/zope.interface@7.0" + + +def test_vendored_dependencies_at_any_depth_in_document_order() -> None: + def bundled(ref: str, *children: Component) -> Component: + return {"type": "library", "bom-ref": ref, "name": ref, "components": list(children)} + + vendored: Component = {"type": "library", "bom-ref": "bundled:b.whl#vendored/pkg:pypi/six@1", "name": "six"} + # a file whose path holds the marker is no vendored package + file: Component = {"type": "file", "bom-ref": "bundled:a.whl#x#vendored/y", "name": "x#vendored/y"} + components = [ + bundled("pkg:pypi/app@1", bundled("bundled:a.whl", file), bundled("bundled:b.whl", vendored)), + bundled("bundled:c.whl", {**vendored, "bom-ref": "bundled:c.whl#vendored/pkg:pypi/six@1"}), + ] + assert vendored_dependencies(components) == [ + {"ref": "bundled:b.whl", "dependsOn": ["bundled:b.whl#vendored/pkg:pypi/six@1"]}, + {"ref": "bundled:c.whl", "dependsOn": ["bundled:c.whl#vendored/pkg:pypi/six@1"]}, + ] From f1ae1ea498b01926addaf63046f0fed31979ca9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:03:28 -0800 Subject: [PATCH 078/115] =?UTF-8?q?=E2=9C=85=20test(content):=20check=20th?= =?UTF-8?q?e=20falsy=20provider=20is=20falsy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/_collect/test_content.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/_collect/test_content.py b/tests/_collect/test_content.py index 62e779e..ea7b472 100644 --- a/tests/_collect/test_content.py +++ b/tests/_collect/test_content.py @@ -68,6 +68,7 @@ def __len__(self) -> int: return 0 given = Empty() + assert not given configured = dataclasses.replace(settings, content_provider="not_a_module_anywhere:Content") assert resolve_provider(given, configured, tmp_path) == (given, ()) From 316278cacbbd528627183d6c3fd2004036487e2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:04:27 -0800 Subject: [PATCH 079/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(model):?= =?UTF-8?q?=20rename=20the=20namespace=20params?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit namespace meant a property prefix in bundled_wheel_component and _declared_dependency, and a UUID in with_header. They are now property_prefix and serial_namespace. with_header also checked the document's refs; check_refs does that now, so with_header only derives the serial and adds the header. The ref checks use loops in place of the nested comprehensions. --- src/python_sbom/_model/_bom.py | 46 ++++++++++++++------------- src/python_sbom/_model/_components.py | 12 ++++--- src/python_sbom/_model/_document.py | 36 +++++++++++---------- tests/_model/test_document.py | 24 ++++++++------ 4 files changed, 65 insertions(+), 53 deletions(-) diff --git a/src/python_sbom/_model/_bom.py b/src/python_sbom/_model/_bom.py index ec3d74c..431a272 100644 --- a/src/python_sbom/_model/_bom.py +++ b/src/python_sbom/_model/_bom.py @@ -7,7 +7,7 @@ from packaging.requirements import Requirement from ._components import bundled_wheel_component, component_from_metadata, script_tool, vendored_dependencies -from ._document import contributed_properties, formulation, timestamp, with_header +from ._document import check_refs, contributed_properties, formulation, timestamp, with_header from ._identity import project_identity from ._inputs import Artifact from ._markers import applies_without_extras @@ -19,7 +19,7 @@ from ._identity import ProjectIdentity from ._inputs import BuildFacts, BundledWheelInfo, Contributions, SbomRequest, Settings, SourceTree - from ._types import Component, Document, ExternalReference, Metadata, Property + from ._types import Body, Component, Document, ExternalReference, Metadata, Property def describe( @@ -71,7 +71,7 @@ def describe( return assemble(artifact, project, settings, facts, contributions) -def _declared_dependency(text: str, requirement: Requirement, namespace: str) -> Component: +def _declared_dependency(text: str, requirement: Requirement, property_prefix: str) -> Component: component: Final[Component] = { "type": "library", # the same distribution can be declared more than once with different markers, so the purl is not unique @@ -79,10 +79,13 @@ def _declared_dependency(text: str, requirement: Requirement, namespace: str) -> "name": requirement.name, "purl": purl(requirement.name), "externalReferences": [{"type": "distribution", "url": pypi_url(requirement.name)}], - "properties": [{"name": f"{namespace}:requires-dist", "value": text}], + "properties": [{"name": f"{property_prefix}:requires-dist", "value": text}], } if requirement.marker is not None: - component["properties"].append({"name": f"{namespace}:environment-marker", "value": str(requirement.marker)}) + component["properties"].append({ + "name": f"{property_prefix}:environment-marker", + "value": str(requirement.marker), + }) return component @@ -113,23 +116,22 @@ def assemble( *contributions.tools, ] uid, name = artifact.workflow - return with_header( - { - "metadata": _metadata(tools, artifact, project, settings, facts), - "components": components, - "dependencies": [ - {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in components]}, - *artifact.dependencies, - *tool_dependencies, - *contributions.dependencies, - ], - # what the artifact needs at run time beyond what it ships, an interpreter and the dependencies an installer - # resolves, is not listed - "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], - "formulation": [formulation(uid, name, tools, root["bom-ref"], facts)], - }, - project.serial_namespace, - ) + body: Final[Body] = { + "metadata": _metadata(tools, artifact, project, settings, facts), + "components": components, + "dependencies": [ + {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in components]}, + *artifact.dependencies, + *tool_dependencies, + *contributions.dependencies, + ], + # what the artifact needs at run time beyond what it ships, an interpreter and the dependencies an installer + # resolves, is not listed + "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], + "formulation": [formulation(uid, name, tools, root["bom-ref"], facts)], + } + check_refs(body) + return with_header(body, project.serial_namespace) def _provenance( diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index 1ca1b8a..cfc41d5 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -6,9 +6,11 @@ from typing import TYPE_CHECKING, Final from urllib.parse import urlsplit +from packaging.utils import canonicalize_name + from ._document import walk_components from ._licenses import licenses -from ._names import property_prefix, purl, pypi_url, reference_type, tool_ref +from ._names import purl, pypi_url, reference_type, tool_ref from ._types import Dependency if TYPE_CHECKING: @@ -20,7 +22,7 @@ def bundled_wheel_component( - wheel: BundledWheel, namespace: str, properties: list[Property], supplier: OrganizationalEntity | None + wheel: BundledWheel, property_prefix: str, properties: list[Property], supplier: OrganizationalEntity | None ) -> Component: component = component_from_metadata(wheel.metadata) # the purl names the package; the path tells apart copies of one release bundled in two places @@ -33,7 +35,7 @@ def bundled_wheel_component( {"type": "distribution", "url": pypi_url(wheel.metadata.name, wheel.metadata.version)}, ] component["properties"] = [ - {"name": f"{namespace}:bundled-wheel", "value": wheel.path}, + {"name": f"{property_prefix}:bundled-wheel", "value": wheel.path}, *properties, *component["properties"], ] @@ -47,11 +49,11 @@ def bundled_wheel_component( for source, metadata in wheel.vendored: child = component_from_metadata(metadata) child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['bom-ref']}" - child["properties"] = [*child["properties"], {"name": f"{namespace}:vendored-manifest", "value": source}] + child["properties"] = [*child["properties"], {"name": f"{property_prefix}:vendored-manifest", "value": source}] child["evidence"] = {"identity": [_evidence("purl", "manifest-analysis", source)]} # a vendored .dist-info comes after pip's manifest line naming the same package, so its metadata wins; keyed on # the PEP 503 name, since the manifest and the .dist-info may spell it differently - vendored[f"{property_prefix(child['name'])}=={child['version']}"] = child + vendored[f"{canonicalize_name(child['name'])}=={child['version']}"] = child component["components"] = [ *(file_component(component["bom-ref"], row) for row in wheel.files), *(vendored[key] for key in sorted(vendored)), diff --git a/src/python_sbom/_model/_document.py b/src/python_sbom/_model/_document.py index 30e1335..dad5c83 100644 --- a/src/python_sbom/_model/_document.py +++ b/src/python_sbom/_model/_document.py @@ -16,7 +16,7 @@ from ._types import Body, Component, Document, Formula, Property -def with_header(body: Body, namespace: uuid.UUID) -> Document: +def check_refs(body: Body) -> None: # a provider contributes refs of its own, so the document is checked before it is written rather than by validate components: Final[list[Component]] = [ *walk_components([ @@ -27,11 +27,8 @@ def with_header(body: Body, namespace: uuid.UUID) -> Document: ] refs: Final[Counter[str]] = Counter(component["bom-ref"] for component in components) # formulas and workflows share the bom-ref namespace, so a component cannot take theirs - refs.update( - ref - for formula in body.get("formulation", []) - for ref in (formula["bom-ref"], *(workflow["bom-ref"] for workflow in formula["workflows"])) - ) + for formula in body.get("formulation", []): + refs.update([formula["bom-ref"], *(workflow["bom-ref"] for workflow in formula["workflows"])]) if repeated := sorted(ref for ref, count in refs.items() if count > 1): msg = f"bom-ref values repeat: {', '.join(repeated)}" raise SbomError(msg) @@ -41,19 +38,12 @@ def with_header(body: Body, namespace: uuid.UUID) -> Document: if repeated := sorted(ref for ref, count in entries.items() if count > 1): msg = f"dependencies list more than one entry for: {', '.join(repeated)}" raise SbomError(msg) - known: Final[set[str]] = {component["bom-ref"] for component in components} - named = {ref for entry in body["dependencies"] for ref in (entry["ref"], *entry.get("dependsOn", []))} - if unknown := sorted(named - known): + named: Final[set[str]] = set(entries) + for entry in body["dependencies"]: + named.update(entry.get("dependsOn", [])) + if unknown := sorted(named - {component["bom-ref"] for component in components}): msg = f"dependencies name bom-refs no component has: {', '.join(unknown)}" raise SbomError(msg) - return { - "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", - "bomFormat": "CycloneDX", - "specVersion": "1.6", - "serialNumber": f"urn:uuid:{uuid.uuid5(namespace, json.dumps(body, sort_keys=True))}", - "version": 1, - **body, - } def walk_components(components: Iterable[Component]) -> Iterator[Component]: @@ -66,6 +56,17 @@ def walk_components(components: Iterable[Component]) -> Iterator[Component]: pending.extend(reversed(component.get("components", []))) +def with_header(body: Body, serial_namespace: uuid.UUID) -> Document: + return { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "serialNumber": f"urn:uuid:{uuid.uuid5(serial_namespace, json.dumps(body, sort_keys=True))}", + "version": 1, + **body, + } + + def checked_contributions(contributions: Contributions, provider: ContentProvider) -> Contributions: # what a provider adds is checked for the fields everything else here reads, so a mistake names the provider rather # than surfacing as a KeyError deep in the build @@ -110,6 +111,7 @@ def timestamp(facts: BuildFacts) -> str: __all__ = [ + "check_refs", "checked_contributions", "contributed_properties", "formulation", diff --git a/tests/_model/test_document.py b/tests/_model/test_document.py index 641707a..260a087 100644 --- a/tests/_model/test_document.py +++ b/tests/_model/test_document.py @@ -8,7 +8,13 @@ from python_sbom import ContentProvider, Contributions from python_sbom._errors import SbomError -from python_sbom._model._document import checked_contributions, contributed_properties, formulation, with_header +from python_sbom._model._document import ( + check_refs, + checked_contributions, + contributed_properties, + formulation, + with_header, +) if TYPE_CHECKING: from python_sbom import Component, Dependency, Property @@ -38,7 +44,7 @@ def test_document_serial_derives_from_body() -> None: } -def test_header_refuses_repeated_refs() -> None: +def test_check_refs_refuses_repeated_refs() -> None: body: Body = { "metadata": { "timestamp": "2023-11-14T22:13:20+00:00", @@ -50,10 +56,10 @@ def test_header_refuses_repeated_refs() -> None: "dependencies": [], } with pytest.raises(SbomError, match="bom-ref values repeat: app"): - with_header(body, uuid.uuid4()) + check_refs(body) -def test_header_refuses_unknown_dependency() -> None: +def test_check_refs_refuses_unknown_dependency() -> None: body: Body = { "metadata": { "timestamp": "2023-11-14T22:13:20+00:00", @@ -65,7 +71,7 @@ def test_header_refuses_unknown_dependency() -> None: "dependencies": [{"ref": "app", "dependsOn": ["ghost"]}], } with pytest.raises(SbomError, match="dependencies name bom-refs no component has: ghost"): - with_header(body, uuid.uuid4()) + check_refs(body) def test_contributed_properties_keep_their_own_prefix() -> None: @@ -83,11 +89,11 @@ def _partial(**fields: str | list[dict[str, str]]) -> Component: return cast("Component", fields) -def test_header_refuses_a_second_dependency_entry_for_a_ref() -> None: +def test_check_refs_refuses_a_second_dependency_entry_for_a_ref() -> None: # as when a provider adds an entry for the root, whose entry python-sbom writes body = _body([], [{"ref": "app", "dependsOn": []}, {"ref": "app", "dependsOn": []}], []) with pytest.raises(SbomError, match="dependencies list more than one entry for: app"): - with_header(body, uuid.uuid4()) + check_refs(body) def _body(components: list[Component], dependencies: list[Dependency], formulation: list[Formula]) -> Body: @@ -107,11 +113,11 @@ def _body(components: list[Component], dependencies: list[Dependency], formulati @pytest.mark.parametrize( "ref", [pytest.param("formula:build", id="formula"), pytest.param("workflow:build", id="workflow")] ) -def test_header_refuses_a_component_taking_a_formulation_ref(facts: BuildFacts, ref: str) -> None: +def test_check_refs_refuses_a_component_taking_a_formulation_ref(facts: BuildFacts, ref: str) -> None: formula = formulation("build", "build the app", [], "app", facts) body = _body([{"type": "library", "bom-ref": ref, "name": "clash"}], [], [formula]) with pytest.raises(SbomError, match=f"bom-ref values repeat: {ref}"): - with_header(body, uuid.uuid4()) + check_refs(body) @pytest.mark.parametrize( From e3f91fe9f5fab22b34718807783c19c5da5145c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:05:27 -0800 Subject: [PATCH 080/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20move?= =?UTF-8?q?=20revision=5Freader=20to=20=5Fcollect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit revision_reader lived in _generate._embed, so the archive pipeline and the CLI imported the embed pipeline to get it. It now sits beside default_revision in _collect._revision. git_revision and commit_revision were exported for tests alone; they are private now and the tests go through default_revision and revision_reader. --- src/python_sbom/_collect/_revision.py | 35 ++++++++++++++++----------- src/python_sbom/_generate/_embed.py | 11 +-------- tests/_collect/test_revision.py | 35 ++++++++++++++++++--------- tests/_generate/test_archive.py | 2 +- 4 files changed, 47 insertions(+), 36 deletions(-) diff --git a/src/python_sbom/_collect/_revision.py b/src/python_sbom/_collect/_revision.py index 24a7933..3efdf70 100644 --- a/src/python_sbom/_collect/_revision.py +++ b/src/python_sbom/_collect/_revision.py @@ -22,15 +22,23 @@ class RevisionReader(Protocol): def __call__(self, root: Path, /) -> Revision | None: ... +def revision_reader(commit: str | None) -> RevisionReader: + # a commit given on the command line stands for the checkout, so git and the files go unread + if commit is None: + return default_revision + revision = _commit_revision(commit, "--commit") + return lambda _root: revision + + def default_revision(root: Path, /) -> Revision | None: # the variable exists for builds from an sdist, so it wins over whatever checkout the sdist was unpacked into if commit := os.environ.get("PYTHON_SBOM_COMMIT"): - return commit_revision(commit, "PYTHON_SBOM_COMMIT") - if (found := git_revision(root)) is not None: + return _commit_revision(commit, "PYTHON_SBOM_COMMIT") + if (found := _git_revision(root)) is not None: return found # the sdist build wrote it from the revision it resolved, so it is at least as exact as an archival file if (recorded := root / REVISION_FILE).is_file(): - return commit_revision(recorded.read_text(encoding="utf-8").strip().removeprefix("node: "), REVISION_FILE) + return _commit_revision(recorded.read_text(encoding="utf-8").strip().removeprefix("node: "), REVISION_FILE) archival = root / ".git_archival.txt" if archival.is_file() and ( match := re.search(rf"^node: ({_COMMIT.pattern})$", archival.read_text(encoding="utf-8"), re.MULTILINE) @@ -39,15 +47,7 @@ def default_revision(root: Path, /) -> Revision | None: return None -def commit_revision(commit: str, origin: str) -> Revision: - # the commit lands in the source link, so anything but a full hash could point the link elsewhere - if not _COMMIT.fullmatch(commit): - msg = f"{origin} must be a full lowercase hex commit hash of 40 or 64 characters, got {commit!r}" - raise SettingsError(msg) - return Revision(commit=commit, dirty=False) - - -def git_revision(root: Path, /) -> Revision | None: +def _git_revision(root: Path, /) -> Revision | None: resolved = root.resolve() if (git := shutil.which("git")) is None or not any( (directory / ".git").exists() for directory in (resolved, *resolved.parents) @@ -77,10 +77,17 @@ def _git(git: str, root: Path, *arguments: str) -> str: ).stdout.strip() +def _commit_revision(commit: str, origin: str) -> Revision: + # the commit lands in the source link, so anything but a full hash could point the link elsewhere + if not _COMMIT.fullmatch(commit): + msg = f"{origin} must be a full lowercase hex commit hash of 40 or 64 characters, got {commit!r}" + raise SettingsError(msg) + return Revision(commit=commit, dirty=False) + + __all__ = [ "REVISION_FILE", "RevisionReader", - "commit_revision", "default_revision", - "git_revision", + "revision_reader", ] diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py index 3e7f5ae..65386a9 100644 --- a/src/python_sbom/_generate/_embed.py +++ b/src/python_sbom/_generate/_embed.py @@ -6,7 +6,7 @@ from python_sbom._collect._archive import member_reader from python_sbom._collect._content import resolve_provider from python_sbom._collect._facts import collect_facts -from python_sbom._collect._revision import commit_revision, default_revision +from python_sbom._collect._revision import revision_reader from python_sbom._collect._settings import settings_from_pyproject from python_sbom._collect._source import read_source_tree, wheel_request from python_sbom._model._content import ProviderContext @@ -19,7 +19,6 @@ from collections.abc import Sequence from pathlib import Path - from python_sbom._collect._revision import RevisionReader from python_sbom._model._content import ContentProvider from python_sbom._model._inputs import BuildFacts, SbomRequest @@ -60,15 +59,7 @@ def embed_into(wheel: Path, request: SbomRequest, content: ContentProvider, fact return write_sbom(wheel, sbom.file_name, sbom.content, facts.epoch) -def revision_reader(commit: str | None) -> RevisionReader: - if commit is None: - return default_revision - revision = commit_revision(commit, "--commit") - return lambda _root: revision - - __all__ = [ "embed_into", "embed_sbom", - "revision_reader", ] diff --git a/tests/_collect/test_revision.py b/tests/_collect/test_revision.py index c996509..ea420d5 100644 --- a/tests/_collect/test_revision.py +++ b/tests/_collect/test_revision.py @@ -4,7 +4,7 @@ import pytest -from python_sbom._collect._revision import REVISION_FILE, default_revision, git_revision +from python_sbom._collect._revision import REVISION_FILE, default_revision, revision_reader from python_sbom._errors import SettingsError from python_sbom._model._inputs import Revision @@ -68,20 +68,20 @@ def test_git_revision(checkout: Path, git: Git) -> None: def test_git_revision_dirty(checkout: Path, git: Git) -> None: (checkout / "pyproject.toml").write_text("[project]\nname = 'changed'\n", encoding="utf-8") - assert git_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=True) + assert default_revision(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=True) def test_untracked_file_marks_revision_dirty(checkout: Path) -> None: # the build packages it, so the commit alone no longer describes the source (checkout / "backdoor.py").write_text("import os\n", encoding="utf-8") - assert (revision := git_revision(checkout)) is not None + assert (revision := default_revision(checkout)) is not None assert revision.dirty def test_ignored_file_leaves_revision_clean(checkout: Path) -> None: (checkout / "build").mkdir() (checkout / "build" / "output.txt").write_text("new\n", encoding="utf-8") - assert (revision := git_revision(checkout)) is not None + assert (revision := default_revision(checkout)) is not None assert not revision.dirty @@ -96,13 +96,13 @@ def monorepo(tmp_path: Path, git: Git) -> Path: def test_git_revision_of_project_in_subdirectory(monorepo: Path, git: Git) -> None: - assert git_revision(monorepo / "one") == Revision(commit=git("rev-parse", "HEAD"), dirty=False) + assert default_revision(monorepo / "one") == Revision(commit=git("rev-parse", "HEAD"), dirty=False) def test_sibling_changes_leave_revision_clean(monorepo: Path) -> None: (monorepo / "two" / "pyproject.toml").write_text("[project]\nname = 'changed'\n", encoding="utf-8") (monorepo / "two" / "new.py").write_text("", encoding="utf-8") - assert (revision := git_revision(monorepo / "one")) is not None + assert (revision := default_revision(monorepo / "one")) is not None assert not revision.dirty @@ -122,23 +122,23 @@ def test_git_runs_no_fsmonitor_command(tmp_path: Path, checkout: Path, git: Git, git("config", "core.fsmonitor", f"touch {(marker := tmp_path / 'ran').as_posix()}; false") (root := checkout / project).mkdir(parents=True, exist_ok=True) (root / "pyproject.toml").touch() - git_revision(root) + default_revision(root) assert not marker.exists() def test_git_revision_before_first_commit(tmp_path: Path, git: Git) -> None: (tmp_path / "checkout" / "pyproject.toml").write_text("[project]\n", encoding="utf-8") git("add", "pyproject.toml") - assert git_revision(tmp_path / "checkout") is None + assert default_revision(tmp_path / "checkout") is None def test_git_revision_outside_checkout(tmp_path: Path) -> None: - assert git_revision(tmp_path) is None + assert default_revision(tmp_path) is None def test_git_revision_without_git(checkout: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("PATH", "") - assert git_revision(checkout) is None + assert default_revision(checkout) is None @pytest.mark.parametrize( @@ -164,7 +164,7 @@ def test_git_status_failure_leaves_revision_unknown(checkout: Path, git: Git) -> (stored := checkout / ".git" / "objects" / tree[:2] / tree[2:]).chmod(0o644) stored.unlink() with pytest.warns(UserWarning, match="git cannot read the source revision in .*: error: bad tree object HEAD"): - assert git_revision(checkout) is None + assert default_revision(checkout) is None @pytest.mark.parametrize( @@ -182,3 +182,16 @@ def test_git_archival(tmp_path: Path, archival: str, expected: Revision | None) def test_no_revision_source(tmp_path: Path) -> None: assert default_revision(tmp_path) is None + + +def test_given_commit_wins_over_checkout(checkout: Path) -> None: + assert revision_reader("a" * 40)(checkout) == Revision(commit="a" * 40, dirty=False) + + +def test_given_commit_must_be_full_hash() -> None: + with pytest.raises(SettingsError, match="--commit must be a full lowercase hex commit hash"): + revision_reader("c0ffee") + + +def test_no_given_commit_reads_checkout(checkout: Path, git: Git) -> None: + assert revision_reader(None)(checkout) == Revision(commit=git("rev-parse", "HEAD"), dirty=False) diff --git a/tests/_generate/test_archive.py b/tests/_generate/test_archive.py index ca3e3c1..e467bec 100644 --- a/tests/_generate/test_archive.py +++ b/tests/_generate/test_archive.py @@ -11,9 +11,9 @@ from python_sbom import BundledWheelInfo, ContentProvider, Contributions, ProviderContext from python_sbom._collect._facts import collect_facts +from python_sbom._collect._revision import revision_reader from python_sbom._errors import WheelError from python_sbom._generate._archive import describe_archive -from python_sbom._generate._embed import revision_reader from python_sbom._model._inputs import Revision from python_sbom._validate._artifact import validate From c5d5ea5ad5008fae1b3fb81f6541e5061f7b4741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:06:12 -0800 Subject: [PATCH 081/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20drop?= =?UTF-8?q?=20parameters=20callers=20fill=20alike?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit settings_from_pyproject took overrides that production always passed as {}, and render_spdx took the tool version that every caller passed as __version__. Both parameters are gone. --- src/python_sbom/_collect/_settings.py | 5 ++-- src/python_sbom/_formats/_spdx.py | 10 +++---- src/python_sbom/_generate/_archive.py | 2 +- src/python_sbom/_generate/_embed.py | 2 +- tests/_collect/test_settings.py | 17 ++++++----- tests/_formats/test_spdx.py | 41 ++++++++++++++------------- 6 files changed, 38 insertions(+), 39 deletions(-) diff --git a/src/python_sbom/_collect/_settings.py b/src/python_sbom/_collect/_settings.py index b90ac18..a0df191 100644 --- a/src/python_sbom/_collect/_settings.py +++ b/src/python_sbom/_collect/_settings.py @@ -18,7 +18,7 @@ Toml: TypeAlias = "str | int | float | bool | datetime | date | time | list[Toml] | dict[str, Toml]" -def settings_from_pyproject(root: Path, overrides: Mapping[str, Toml]) -> Settings: +def settings_from_pyproject(root: Path) -> Settings: pyproject = root / "pyproject.toml" try: loaded = tomllib.loads(pyproject.read_text(encoding="utf-8")) @@ -28,8 +28,7 @@ def settings_from_pyproject(root: Path, overrides: Mapping[str, Toml]) -> Settin except tomllib.TOMLDecodeError as error: msg = f"{pyproject} is not valid TOML: {error}" raise SettingsError(msg) from error - table = loaded.get("tool", {}).get("python-sbom", {}) - return settings_from_table({**table, **overrides}, f"[tool.python-sbom] of {pyproject}") + return settings_from_table(loaded.get("tool", {}).get("python-sbom", {}), f"[tool.python-sbom] of {pyproject}") def settings_from_table(table: Mapping[str, Toml], where: str) -> Settings: diff --git a/src/python_sbom/_formats/_spdx.py b/src/python_sbom/_formats/_spdx.py index 3967258..05e7497 100644 --- a/src/python_sbom/_formats/_spdx.py +++ b/src/python_sbom/_formats/_spdx.py @@ -108,10 +108,10 @@ class Spdx(TypedDict): def to_spdx(document: Document) -> str: - return dumps(render_spdx(document, __version__)).decode() + return dumps(render_spdx(document)).decode() -def render_spdx(bom: Document, tool_version: str) -> Spdx: +def render_spdx(bom: Document) -> Spdx: metadata: Final[Metadata] = bom["metadata"] root: Final[Component] = metadata["component"] project: Final[str] = property_prefix(root["name"]) @@ -141,7 +141,7 @@ def render_spdx(bom: Document, tool_version: str) -> Spdx: "documentNamespace": f"{repository}/sboms/{name}-{source.removeprefix('urn:uuid:')}", "comment": f"Rendered from the CycloneDX SBOM {source} ({sbom_file}), which also carries per-file hashes and" " the build formulation.", - "creationInfo": _creation_info(metadata, tool_creators, tool_version), + "creationInfo": _creation_info(metadata, tool_creators), "packages": packages, "relationships": _relationships(bom, components, tools), } @@ -231,7 +231,7 @@ def _spdx_id(bom_ref: str) -> str: return f"SPDXRef-{readable}-{hashlib.sha256(bom_ref.encode()).hexdigest()[:8]}" -def _creation_info(metadata: Metadata, tool_creators: list[str], tool_version: str) -> _CreationInfo: +def _creation_info(metadata: Metadata, tool_creators: list[str]) -> _CreationInfo: timestamp: Final[str | None] = metadata.get("timestamp") # SPDX requires a creation time; without the CycloneDX one, the time of this rendering is the true one created: Final[datetime] = datetime.now(UTC) if timestamp is None else datetime.fromisoformat(timestamp) @@ -239,7 +239,7 @@ def _creation_info(metadata: Metadata, tool_creators: list[str], tool_version: s organization: Final[str] = "" if supplier is None else supplier.get("name", "") creators: Final[list[str]] = [f"Organization: {organization}"] if organization else [] creators.extend(tool_creators) - creators.append(f"Tool: python-sbom-{tool_version}") + creators.append(f"Tool: python-sbom-{__version__}") return {"created": created.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "creators": creators} diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index e4da50e..2925169 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -44,7 +44,7 @@ def describe_archive( # the archive's top-level .dist-info names the project; every other .dist-info marks a wheel bundled unpacked into # its parent directory and every .whl member a bundled wheel; other files belong to the deepest distribution that # holds them, or, at the top level, to the archive - settings = settings_from_pyproject(root, {}) + settings = settings_from_pyproject(root) content, provider_scripts = resolve_provider(provider, settings, root) tree = read_archive(archive) project = project_identity(tree.metadata[""], settings) diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py index 65386a9..b0e6dcc 100644 --- a/src/python_sbom/_generate/_embed.py +++ b/src/python_sbom/_generate/_embed.py @@ -31,7 +31,7 @@ def embed_sbom( commit: str | None, build_env: Sequence[str] | None, ) -> str: - settings = settings_from_pyproject(root, {}) + settings = settings_from_pyproject(root) content, scripts = resolve_provider(provider, settings, root) return embed_into( wheel, diff --git a/tests/_collect/test_settings.py b/tests/_collect/test_settings.py index fa085b1..37a1a72 100644 --- a/tests/_collect/test_settings.py +++ b/tests/_collect/test_settings.py @@ -16,35 +16,34 @@ def test_missing_pyproject(tmp_path: Path) -> None: with pytest.raises(SettingsError, match=r"has no pyproject\.toml to read"): - settings_from_pyproject(tmp_path, {}) + settings_from_pyproject(tmp_path) def test_invalid_pyproject(tmp_path: Path) -> None: (tmp_path / "pyproject.toml").write_text("[tool\n", encoding="utf-8") with pytest.raises(SettingsError, match=r"pyproject\.toml is not valid TOML"): - settings_from_pyproject(tmp_path, {}) + settings_from_pyproject(tmp_path) def test_defaults(tmp_path: Path, settings: Settings) -> None: (tmp_path / "pyproject.toml").write_text("[project]\nname = 'a'\n", encoding="utf-8") - assert settings_from_pyproject(tmp_path, {}) == settings + assert settings_from_pyproject(tmp_path) == settings -def test_table_with_overrides(tmp_path: Path) -> None: +def test_table(tmp_path: Path) -> None: (tmp_path / "pyproject.toml").write_text( "[tool.python-sbom]\n" "repository = 'https://github.com/example/example/'\n" "default-branch = 'dev'\n" "bundled-wheels = ['src/*.whl']\n" - "supplier = { name = 'Example Org' }\n", + "supplier = { name = 'Example Org' }\n" + "content-provider = 'sbom:Content'\n", encoding="utf-8", ) - assert settings_from_pyproject( - tmp_path, {"default-branch": "trunk", "content-provider": "sbom:Content"} - ) == Settings( + assert settings_from_pyproject(tmp_path) == Settings( supplier={"name": "Example Org", "url": []}, repository="https://github.com/example/example", - default_branch="trunk", + default_branch="dev", bundled_wheels=("src/*.whl",), content_provider="sbom:Content", ) diff --git a/tests/_formats/test_spdx.py b/tests/_formats/test_spdx.py index 4078f57..fd5909f 100644 --- a/tests/_formats/test_spdx.py +++ b/tests/_formats/test_spdx.py @@ -11,6 +11,7 @@ from python_sbom._formats._json import dumps from python_sbom._formats._spdx import render_spdx, to_spdx from python_sbom._validate._schemas import spdx_problems +from python_sbom.version import __version__ if TYPE_CHECKING: from conftest import Describe @@ -38,7 +39,7 @@ def test_minimal(describe: Describe) -> None: document = describe() - spdx = render_spdx(document, "1.2.3") + spdx = render_spdx(document) ids = {package["name"]: package["SPDXID"] for package in spdx["packages"]} serial = document["serialNumber"] assert spdx == { @@ -49,7 +50,7 @@ def test_minimal(describe: Describe) -> None: "documentNamespace": f"https://spdx.org/spdxdocs/sboms/Example-1.0-{serial.removeprefix('urn:uuid:')}", "comment": f"Rendered from the CycloneDX SBOM {serial} (example.cdx.json), which also carries per-file hashes" " and the build formulation.", - "creationInfo": {"created": "2023-11-14T22:13:20Z", "creators": ["Tool: python-sbom-1.2.3"]}, + "creationInfo": {"created": "2023-11-14T22:13:20Z", "creators": [f"Tool: python-sbom-{__version__}"]}, "packages": [ { "SPDXID": ids["Example"], @@ -103,14 +104,14 @@ def test_minimal(describe: Describe) -> None: def test_spdx_ids_are_valid_and_tell_similar_refs_apart(describe: Describe) -> None: document = describe("Requires-Dist: click; os_name == 'nt'\nRequires-Dist: click; os_name != 'nt'\n") - ids = [package["SPDXID"] for package in render_spdx(document, "1")["packages"]] + ids = [package["SPDXID"] for package in render_spdx(document)["packages"]] assert all(re.fullmatch(r"SPDXRef-[A-Za-z0-9.-]+", spdx_id) for spdx_id in ids) assert len(set(ids)) == len(ids) def test_repository_names_the_namespace(describe: Describe, settings: Settings) -> None: document = describe(settings=dataclasses.replace(settings, repository="https://github.com/example/example")) - namespace = render_spdx(document, "1")["documentNamespace"] + namespace = render_spdx(document)["documentNamespace"] assert namespace.startswith("https://github.com/example/example/sboms/Example-1.0-") @@ -120,18 +121,18 @@ def test_supplier_and_application_tools_are_creators(describe: Describe, setting {"type": "application", "bom-ref": "tool:hatchling", "name": "hatchling", "version": "1.28"}, {"type": "application", "bom-ref": "tool:script", "name": "script"}, ]) - assert render_spdx(document, "1")["creationInfo"]["creators"] == [ + assert render_spdx(document)["creationInfo"]["creators"] == [ "Organization: Example Org", "Tool: hatchling-1.28", "Tool: script", - "Tool: python-sbom-1", + f"Tool: python-sbom-{__version__}", ] def test_versionless_root(describe: Describe) -> None: document = describe() del document["metadata"]["component"]["version"] - spdx = render_spdx(document, "1") + spdx = render_spdx(document) assert spdx["name"] == "Example" assert "versionInfo" not in spdx["packages"][0] @@ -155,7 +156,7 @@ def test_package_fields(describe: Describe) -> None: "components": [{"type": "file", "bom-ref": "bundled:a.whl#a.py", "name": "a.py"}], } document["components"].append(component) - (package,) = (package for package in render_spdx(document, "1")["packages"] if package["name"] == "a") + (package,) = (package for package in render_spdx(document)["packages"] if package["name"] == "a") assert {key: value for key, value in package.items() if key != "SPDXID"} == { "name": "a", "downloadLocation": "NOASSERTION", @@ -175,7 +176,7 @@ def test_package_fields(describe: Describe) -> None: def test_license_refs_carry_extracted_text(describe: Describe) -> None: document = describe("License-Expression: MIT AND LicenseRef-Vendor\n") - assert render_spdx(document, "1").get("hasExtractedLicensingInfos") == [ + assert render_spdx(document).get("hasExtractedLicensingInfos") == [ { "licenseId": "LicenseRef-Vendor", "extractedText": "LicenseRef-Vendor, as named in package metadata", @@ -192,7 +193,7 @@ def test_nested_libraries_are_contained(describe: Describe) -> None: "name": "outer", "components": [{"type": "library", "bom-ref": "inner", "name": "inner"}], }) - spdx = render_spdx(document, "1") + spdx = render_spdx(document) ids = {package["name"]: package["SPDXID"] for package in spdx["packages"]} assert {"spdxElementId": ids["outer"], "relationshipType": "CONTAINS", "relatedSpdxElement": ids["inner"]} in spdx[ "relationships" @@ -210,7 +211,7 @@ def test_nested_libraries_are_contained(describe: Describe) -> None: def test_package_purpose(describe: Describe, kind: str, purpose: str) -> None: document = describe() document["components"].append({"type": kind, "bom-ref": "extra", "name": "extra"}) - (package,) = (package for package in render_spdx(document, "1")["packages"] if package["name"] == "extra") + (package,) = (package for package in render_spdx(document)["packages"] if package["name"] == "extra") assert package["primaryPackagePurpose"] == purpose @@ -220,28 +221,28 @@ def test_to_spdx_text(describe: Describe) -> None: def test_other_tools_document_renders_valid_spdx() -> None: - assert spdx_problems(render_spdx(json.loads(_FOREIGN), "1")) == [] + assert spdx_problems(render_spdx(json.loads(_FOREIGN))) == [] def test_legacy_tools_become_creators() -> None: - assert render_spdx(json.loads(_FOREIGN), "1")["creationInfo"]["creators"] == [ + assert render_spdx(json.loads(_FOREIGN))["creationInfo"]["creators"] == [ "Tool: auditwheel-6.4.0", "Tool: repair", - "Tool: python-sbom-1", + f"Tool: python-sbom-{__version__}", ] def test_document_without_tools() -> None: foreign = json.loads(_FOREIGN) del foreign["metadata"]["tools"] - assert render_spdx(foreign, "1")["creationInfo"]["creators"] == ["Tool: python-sbom-1"] + assert render_spdx(foreign)["creationInfo"]["creators"] == [f"Tool: python-sbom-{__version__}"] def test_namespace_without_serial_names_the_content() -> None: changed = json.loads(_FOREIGN) changed["version"] = 2 - namespaces = [render_spdx(document, "1")["documentNamespace"] for document in (json.loads(_FOREIGN),) * 2] - assert (namespaces[0] == namespaces[1], namespaces[0] == render_spdx(changed, "1")["documentNamespace"]) == ( + namespaces = [render_spdx(document)["documentNamespace"] for document in (json.loads(_FOREIGN),) * 2] + assert (namespaces[0] == namespaces[1], namespaces[0] == render_spdx(changed)["documentNamespace"]) == ( True, False, ) @@ -249,7 +250,7 @@ def test_namespace_without_serial_names_the_content() -> None: def test_creation_time_without_timestamp_is_the_rendering_time() -> None: before = datetime.now(UTC).replace(microsecond=0) - created = datetime.fromisoformat(render_spdx(json.loads(_FOREIGN), "1")["creationInfo"]["created"]) + created = datetime.fromisoformat(render_spdx(json.loads(_FOREIGN))["creationInfo"]["created"]) assert before <= created <= datetime.now(UTC) @@ -257,7 +258,7 @@ def test_supplier_without_name_is_no_creator(describe: Describe) -> None: # the schema makes the name optional, so the loaded JSON, not the typed document, loses it loaded = json.loads(dumps(describe())) loaded["metadata"]["supplier"] = {"url": ["https://example.org"]} - assert render_spdx(loaded, "1")["creationInfo"]["creators"] == ["Tool: python-sbom-1"] + assert render_spdx(loaded)["creationInfo"]["creators"] == [f"Tool: python-sbom-{__version__}"] @pytest.mark.parametrize( @@ -278,7 +279,7 @@ def test_hash_algorithms_take_spdx_names(algorithm: str, expected: str) -> None: def packages_of(fields: dict[str, _Json]) -> list[_Package]: document = json.loads(_FOREIGN) document["components"] = [{"type": "library", "bom-ref": "lib", "name": "lib", **fields}] - return [package for package in render_spdx(document, "1")["packages"] if package["name"] == "lib"] + return [package for package in render_spdx(document)["packages"] if package["name"] == "lib"] def test_hash_algorithm_spdx_lacks_is_dropped() -> None: From 5035b7aa55f4fd1027b049df5199de552e5ac354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:07:19 -0800 Subject: [PATCH 082/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20unnest?= =?UTF-8?q?=20the=20double-loop=20comprehensions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SPDX license references and relationships, the bundled-wheel glob, the archive's wheel order, and a component's authors and header properties each packed two loops into one comprehension. Each now names its inner piece or loops over the outer one. --- src/python_sbom/_collect/_archive.py | 4 +++- src/python_sbom/_collect/_source.py | 11 +++-------- src/python_sbom/_formats/_spdx.py | 28 +++++++++++++-------------- src/python_sbom/_model/_components.py | 28 +++++++++++++-------------- 4 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/python_sbom/_collect/_archive.py b/src/python_sbom/_collect/_archive.py index 427521d..044d91d 100644 --- a/src/python_sbom/_collect/_archive.py +++ b/src/python_sbom/_collect/_archive.py @@ -57,7 +57,9 @@ def read_archive(archive: Path) -> ArchiveTree: else: loose.append(name) # in the order the components list them, so the first broken wheel is the one reported - wheels: Final[list[str]] = [name for prefix in sorted(metadata) for name in owned[prefix] if name.endswith(".whl")] + wheels: Final[list[str]] = [] + for prefix in sorted(metadata): + wheels.extend(name for name in owned[prefix] if name.endswith(".whl")) return ArchiveTree( metadata=metadata, owned=owned, diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index 46e6c82..8c7d6a6 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -32,21 +32,16 @@ def read_source_tree(request: SbomRequest) -> SourceTree: root = request.root + wheels: Final[set[str]] = set() for pattern in request.settings.bundled_wheels: # a wheel outside the project is none the project ships, and pathlib refuses an absolute pattern outright if (path := PurePath(pattern)).anchor or ".." in path.parts: msg = f"`bundled-wheels` pattern {pattern!r} must be relative to the project, without `..`" raise SettingsError(msg) + wheels.update(wheel.relative_to(root).as_posix() for wheel in root.glob(pattern)) return SourceTree( copyright=copyright_notice(root, request.metadata.values("license_files")), - bundled=read_bundled_wheels( - sorted({ - wheel.relative_to(root).as_posix() - for pattern in request.settings.bundled_wheels - for wheel in root.glob(pattern) - }), - lambda path: (root / path).open("rb"), - ), + bundled=read_bundled_wheels(sorted(wheels), lambda path: (root / path).open("rb")), # str, so a Windows drive in an absolute path survives the join with the native root build_scripts=[ BuildScript(path=script, sha256=hashlib.sha256((root / str(script)).read_bytes()).hexdigest()) diff --git a/src/python_sbom/_formats/_spdx.py b/src/python_sbom/_formats/_spdx.py index 05e7497..14c8aa7 100644 --- a/src/python_sbom/_formats/_spdx.py +++ b/src/python_sbom/_formats/_spdx.py @@ -146,13 +146,13 @@ def render_spdx(bom: Document) -> Spdx: "relationships": _relationships(bom, components, tools), } # SPDX requires the text of every LicenseRef a package declares; the metadata names them, it holds no text - references: Final[list[str]] = sorted({ - token for package in packages for token in _LICENSE_REF.findall(package["licenseDeclared"]) - }) + references: Final[set[str]] = set() + for package in packages: + references.update(_LICENSE_REF.findall(package["licenseDeclared"])) if references: spdx["hasExtractedLicensingInfos"] = [ {"licenseId": token, "extractedText": f"{token}, as named in package metadata", "name": token} - for token in references + for token in sorted(references) ] return spdx @@ -246,17 +246,15 @@ def _creation_info(metadata: Metadata, tool_creators: list[str]) -> _CreationInf def _relationships(bom: Document, components: list[Component], tools: list[Component]) -> list[_Relationship]: root_id: Final[str] = _spdx_id(bom["metadata"]["component"]["bom-ref"]) relationships: Final[list[_Relationship]] = [_relationship("SPDXRef-DOCUMENT", "DESCRIBES", root_id)] - relationships.extend( - _relationship(_spdx_id(parent["bom-ref"]), "CONTAINS", _spdx_id(child["bom-ref"])) - for parent in components - for child in parent.get("components", []) - if child["type"] != "file" - ) - relationships.extend( - _relationship(_spdx_id(entry["ref"]), "DEPENDS_ON", _spdx_id(dependency)) - for entry in bom.get("dependencies", []) - for dependency in entry.get("dependsOn", []) - ) + for parent in components: + parent_id = _spdx_id(parent["bom-ref"]) + children = (child for child in parent.get("components", []) if child["type"] != "file") + relationships.extend(_relationship(parent_id, "CONTAINS", _spdx_id(child["bom-ref"])) for child in children) + for entry in bom.get("dependencies", []): + dependent_id = _spdx_id(entry["ref"]) + relationships.extend( + _relationship(dependent_id, "DEPENDS_ON", _spdx_id(dependency)) for dependency in entry.get("dependsOn", []) + ) relationships.extend(_relationship(_spdx_id(tool["bom-ref"]), "BUILD_TOOL_OF", root_id) for tool in tools) return relationships diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index cfc41d5..85be170 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -69,27 +69,27 @@ def component_from_metadata(metadata: CoreMetadata) -> Component: component["description"] = summary # maintainers are who users reach today, so they win over the original authors when both are named for names, addresses in (("maintainer", "maintainer_email"), ("author", "author_email")): - named = (_contact(name, "") for name in metadata.values(names)) - if people := [ - contact for contact in (*named, *starmap(_contact, getaddresses(metadata.values(addresses)))) if contact - ]: + contacts = [ + *(_contact(name, "") for name in metadata.values(names)), + *starmap(_contact, getaddresses(metadata.values(addresses))), + ] + if people := [contact for contact in contacts if contact]: component["authors"] = people break if declared := licenses(metadata): component["licenses"] = declared component["externalReferences"] = _project_references(metadata) # core metadata headers copied verbatim, so a reader sees what the distribution declared - component["properties"] = [ - {"name": f"python:{header}", "value": value} - for header, values in ( - ("requires-python", metadata.values("requires_python")), - ("requires-dist", metadata.values("requires_dist")), - ("provides-extra", metadata.values("provides_extra")), - ("classifier", metadata.values("classifiers")), - ("keywords", metadata.keywords()), - ) - for value in values + headers: Final[list[tuple[str, list[str]]]] = [ + ("requires-python", metadata.values("requires_python")), + ("requires-dist", metadata.values("requires_dist")), + ("provides-extra", metadata.values("provides_extra")), + ("classifier", metadata.values("classifiers")), + ("keywords", metadata.keywords()), ] + component["properties"] = [] + for header, values in headers: + component["properties"].extend({"name": f"python:{header}", "value": value} for value in values) return component From 9bc21af407d7ebd1b6be287b5b04a33ca7a9c763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:09:52 -0800 Subject: [PATCH 083/115] =?UTF-8?q?=F0=9F=8E=A8=20style:=20keep=20suppress?= =?UTF-8?q?ions=20within=20120=20columns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ContentProvider's two no-op defaults each carried the same two-rule ruff suppression past 160 columns; one disable/enable range now covers both. The zipremove repack line drops under 120 columns by mapping remove over the stale members. --- src/python_sbom/_model/_content.py | 7 +++++-- src/python_sbom/_wheel/_archive.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/python_sbom/_model/_content.py b/src/python_sbom/_model/_content.py index 4be1e4e..83dee50 100644 --- a/src/python_sbom/_model/_content.py +++ b/src/python_sbom/_model/_content.py @@ -53,7 +53,8 @@ class ContentProvider: """ - def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: # ruff: ignore[unused-method-argument, no-self-use] # the no-op default + # ruff: disable[unused-method-argument, no-self-use] # the defaults add nothing, whatever they are given + def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: """ Describe what the project does with a wheel it bundles, as a file or unpacked into a directory. @@ -64,7 +65,7 @@ def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> Bundl """ return BundledWheelInfo() - def contributions(self, context: ProviderContext) -> Contributions: # ruff: ignore[unused-method-argument, no-self-use] # the no-op default + def contributions(self, context: ProviderContext) -> Contributions: """ Add components, their dependencies, root properties and build tools, before references and the serial derive. @@ -74,6 +75,8 @@ def contributions(self, context: ProviderContext) -> Contributions: # ruff: ign """ return Contributions() + # ruff: enable[unused-method-argument, no-self-use] + __all__ = [ "ContentProvider", diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index 9113314..4c62acc 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -60,7 +60,7 @@ def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: # here; both go last, RECORD after the SBOM with zipremove.ZipFile(temporary, "a") as archive: stale = [info for info in archive.infolist() if info.filename in {record, sbom}] - archive.repack([archive.remove(info) for info in stale]) # ty: ignore[unresolved-attribute] # zipremove adds both + archive.repack(list(map(archive.remove, stale))) # ty: ignore[unresolved-attribute] # zipremove adds both archive.writestr(entry, content) archive.writestr(record_info, rewritten) Path(temporary).chmod(stat.S_IMODE(wheel.stat().st_mode)) From cf5d589beb15c11de6d6c39b24c8c42d70626407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:09:52 -0800 Subject: [PATCH 084/115] =?UTF-8?q?=E2=9C=85=20test(bom):=20parametrize=20?= =?UTF-8?q?the=20bundled=20supplier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provider chose its supplier with a conditional and lived in the test body behind a suppression. A module-level provider supplies the wheels the project names as its own, and the test takes it as a parameter. --- tests/_model/test_bom.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/_model/test_bom.py b/tests/_model/test_bom.py index 7fc51dd..75627c7 100644 --- a/tests/_model/test_bom.py +++ b/tests/_model/test_bom.py @@ -341,34 +341,41 @@ def contributions(self, context: ProviderContext) -> Contributions: _ORG: Final[OrganizationalEntity] = {"name": "Example Org", "url": []} +_PIP: Final[str] = "src/pip-1.0-py3-none-any.whl" + + +class _ProjectSupplies(ContentProvider): + # the wheels the project builds itself, which its own supplier supplies + def __init__(self, *own: str) -> None: + self._own = frozenset(own) + + def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: + if str(wheel) in self._own: + return BundledWheelInfo(supplier=context.settings.supplier) + return BundledWheelInfo() @pytest.mark.parametrize( - ("supplied", "expected"), - [pytest.param(True, _ORG, id="by-project"), pytest.param(False, None, id="unnamed")], + ("content", "expected"), + [ + pytest.param(_ProjectSupplies(_PIP), _ORG, id="by-project"), + pytest.param(_ProjectSupplies(), None, id="unnamed"), + ], ) def test_bundled_wheel_supplier_is_the_projects_choice( - describe: Describe, settings: Settings, supplied: bool, expected: OrganizationalEntity | None + describe: Describe, settings: Settings, content: ContentProvider, expected: OrganizationalEntity | None ) -> None: wheel = BundledWheel( - path="src/pip-1.0-py3-none-any.whl", + path=_PIP, sha256="ab" * 32, metadata_name="pip-1.0.dist-info/METADATA", metadata=parse_metadata("Name: pip\nVersion: 1.0\nProject-URL: Source, https://github.com/example/pip\n"), files=[], vendored=[], ) - - class Supplied(ContentProvider): - def __init__(self, supplied: bool) -> None: - self._supplied = supplied - - def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: # ruff: ignore[unused-method-argument] # the base class's signature - return BundledWheelInfo(supplier=context.settings.supplier if self._supplied else None) - document = describe( settings=dataclasses.replace(settings, supplier=_ORG), source=SourceTree(copyright=None, bundled=[wheel], build_scripts=[]), - content=Supplied(supplied), + content=content, ) assert document["components"][0].get("supplier") == expected From 16d9ad73221e21038eddcbea6c49cff2e65c759b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:09:53 -0800 Subject: [PATCH 085/115] =?UTF-8?q?=E2=9C=85=20test(embed):=20pin=20the=20?= =?UTF-8?q?Python=20version=20with=20mocker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/_generate/test_embed.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/_generate/test_embed.py b/tests/_generate/test_embed.py index 3386cf5..6882e6f 100644 --- a/tests/_generate/test_embed.py +++ b/tests/_generate/test_embed.py @@ -2,7 +2,6 @@ import hashlib import json -import platform import zipfile from pathlib import Path from typing import TYPE_CHECKING, Final @@ -15,6 +14,7 @@ import pytest from conftest import Install, WriteWheel + from pytest_mock import MockerFixture _PINNED: Final[Path] = Path(__file__).parent / "pinned" @@ -56,12 +56,12 @@ def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> Bundl def test_embed_output_is_pinned( - tmp_path: Path, write_wheel: WriteWheel, install: Install, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, write_wheel: WriteWheel, install: Install, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture ) -> None: # every input fixed, down to the interpreter's version, so the SBOM and the wheel's members must not change by a # byte; a change that means to alter them updates the pinned files with it monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") - monkeypatch.setattr(platform, "python_version", lambda: "3.14.0") + mocker.patch("platform.python_version", autospec=True, return_value="3.14.0") metadata = b"""\ Metadata-Version: 2.4 Name: Pkg.Name From 126318036139e1e8ca9cb342129f513d76daa14b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:11:24 -0800 Subject: [PATCH 086/115] =?UTF-8?q?=F0=9F=90=9B=20fix(model):=20link=20fil?= =?UTF-8?q?es=20at=20the=20built=20commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The license file and build-script links always named the default branch, so they drifted from the built artifact as the branch moved, even when the SBOM records the exact commit. A clean checkout's commit now names the tree; an unknown revision or one with uncommitted changes still falls back to the default branch. The pinned embed SBOM changes in its license link alone, which moves its serial number and the SBOM's and RECORD's digests in the members list. --- docs/explanation.rst | 3 +- src/python_sbom/_model/_bom.py | 14 ++++++--- src/python_sbom/_model/_inputs.py | 3 +- tests/_generate/pinned/embed-members.json | 4 +-- tests/_generate/pinned/embed.cdx.json | 4 +-- tests/_generate/test_archive.py | 2 +- tests/_model/test_bom.py | 35 +++++++++++++++++------ 7 files changed, 46 insertions(+), 19 deletions(-) diff --git a/docs/explanation.rst b/docs/explanation.rst index cbb4e6d..dda598e 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -65,7 +65,8 @@ What the SBOM records The document's root component is the project, described from the wheel's ``METADATA``: name, version, summary, license expression, maintainers, project URLs, classifiers and keywords. Links to the PyPI release, the wheel's attestation and, with a repository, the license file follow. A build from a git checkout adds the commit and a link to that exact -revision. +revision. Links to the license file and build scripts point at that commit when the checkout is clean, and at the +default branch otherwise. The components below the root are the wheels the project bundles and the runtime dependencies it declares. A bundled wheel carries its SHA-256, its license, a component per file from its ``RECORD``, and the packages it vendors. A diff --git a/src/python_sbom/_model/_bom.py b/src/python_sbom/_model/_bom.py index 431a272..0a25d74 100644 --- a/src/python_sbom/_model/_bom.py +++ b/src/python_sbom/_model/_bom.py @@ -109,7 +109,7 @@ def assemble( environment, tool_dependencies = tool_components(facts) tools: Final[list[Component]] = [ *( - script_tool(script.path, script.sha256, artifact.version, _file_link(project, settings, script.path)) + script_tool(script.path, script.sha256, artifact.version, _file_link(project, settings, facts, script.path)) for script in artifact.build_scripts ), *environment, @@ -149,7 +149,7 @@ def _provenance( {"type": "advisories", "url": f"{project.repository}/security/advisories"}, ]) if artifact.license_file is not None and ( - link := _file_link(project, settings, PurePosixPath(artifact.license_file)) + link := _file_link(project, settings, facts, PurePosixPath(artifact.license_file)) ): references.append({"type": "license", "url": link}) properties: Final[list[Property]] = [ @@ -172,8 +172,14 @@ def _provenance( return root -def _file_link(project: ProjectIdentity, settings: Settings, path: PurePosixPath) -> str | None: - return None if project.repository is None else f"{project.repository}/blob/{settings.default_branch}/{path}" +def _file_link(project: ProjectIdentity, settings: Settings, facts: BuildFacts, path: PurePosixPath) -> str | None: + if project.repository is None: + return None + # the commit of a clean checkout holds the file as it was built; with no commit, or uncommitted changes, the default + # branch is where the file most likely still lives + revision = facts.revision + tree = settings.default_branch if revision is None or revision.dirty else revision.commit + return f"{project.repository}/blob/{tree}/{path}" def _metadata( diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py index 0fb4d76..14d03d7 100644 --- a/src/python_sbom/_model/_inputs.py +++ b/src/python_sbom/_model/_inputs.py @@ -56,7 +56,8 @@ class Settings: :param supplier: the organization that supplies the project, or ``None`` to name none :param repository: the repository URL, or ``None`` to take the first ``Source``, ``Repository`` or ``GitHub`` project URL - :param default_branch: the branch in links to files in the repository + :param default_branch: the branch in links to files in the repository when the build's commit is unknown or has + uncommitted changes :param bundled_wheels: glob patterns, relative to the project directory, of wheels the project ships as data files :param content_provider: ``module:attribute`` of the content provider, or ``None`` diff --git a/tests/_generate/pinned/embed-members.json b/tests/_generate/pinned/embed-members.json index f7d3139..1ebd6d2 100644 --- a/tests/_generate/pinned/embed-members.json +++ b/tests/_generate/pinned/embed-members.json @@ -81,7 +81,7 @@ ], 8, 2175008768, - "63503a8e31088e81123f5b7861da0411bc423174c7353aa74d2d1f9391ce23e4" + "be78512aff2db98c3570bf08ada840be7ad95ab0035b962c4595b31059383a55" ], [ "pkg_name-1.0.dist-info/RECORD", @@ -95,6 +95,6 @@ ], 0, 25165824, - "a932f8430d0fec72d6bdfc1aa0afa1700ede4c4b6eec6a825e245e9835ed0a82" + "ab2b152f6aa0db7eda9723b860ff7b3ff3fa06598fa0bf4304d636e66f6219dc" ] ] diff --git a/tests/_generate/pinned/embed.cdx.json b/tests/_generate/pinned/embed.cdx.json index 2df0c7b..e4ea1fc 100644 --- a/tests/_generate/pinned/embed.cdx.json +++ b/tests/_generate/pinned/embed.cdx.json @@ -2,7 +2,7 @@ "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", "bomFormat": "CycloneDX", "specVersion": "1.6", - "serialNumber": "urn:uuid:d3cce085-f38d-5e88-a7b1-b1b95274c6f3", + "serialNumber": "urn:uuid:0c382715-4ac3-55f8-b2e5-bb3920847bd3", "version": 1, "metadata": { "timestamp": "2023-11-14T22:13:20+00:00", @@ -170,7 +170,7 @@ }, { "type": "license", - "url": "https://github.com/example/pkg/blob/main/LICENSE" + "url": "https://github.com/example/pkg/blob/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/LICENSE" }, { "type": "vcs", diff --git a/tests/_generate/test_archive.py b/tests/_generate/test_archive.py index e467bec..ebfb396 100644 --- a/tests/_generate/test_archive.py +++ b/tests/_generate/test_archive.py @@ -178,7 +178,7 @@ def test_archive_tools_and_metadata(document: Document) -> None: script = metadata["tools"]["components"][0] assert (script["name"], script.get("externalReferences"), metadata.get("properties"), metadata.get("supplier")) == ( "build_app.py", - [{"type": "vcs", "url": "https://github.com/example/app/blob/main/build_app.py"}], + [{"type": "vcs", "url": f"https://github.com/example/app/blob/{_COMMIT}/build_app.py"}], [ { "name": "app:sbom:scope", diff --git a/tests/_model/test_bom.py b/tests/_model/test_bom.py index 75627c7..e21b087 100644 --- a/tests/_model/test_bom.py +++ b/tests/_model/test_bom.py @@ -113,18 +113,23 @@ def test_serial_follows_content(describe: Describe) -> None: @pytest.mark.parametrize( ("dirty", "trail"), [ - pytest.param(False, ("exact source revision", []), id="clean"), + pytest.param(False, ("c0ffee", "exact source revision", []), id="clean"), pytest.param( True, - ("source revision, with uncommitted changes", [{"name": "example:vcs-dirty", "value": "true"}]), + ("main", "source revision, with uncommitted changes", [{"name": "example:vcs-dirty", "value": "true"}]), id="dirty", ), ], ) def test_root_component( - describe: Describe, settings: Settings, facts: BuildFacts, dirty: bool, trail: tuple[str, list[dict[str, str]]] + describe: Describe, + settings: Settings, + facts: BuildFacts, + dirty: bool, + trail: tuple[str, str, list[dict[str, str]]], ) -> None: - comment, dirty_properties = trail + # a dirty checkout's files may differ from its commit's, so its links fall back to the default branch + tree, comment, dirty_properties = trail supplier: OrganizationalEntity = {"name": "Example Org", "url": ["https://example.org"]} document = describe( "License-File: LICENSE\nProject-URL: Source, https://github.com/example/example/\n", @@ -150,7 +155,7 @@ def test_root_component( }, {"type": "security-contact", "url": f"{repository}/security/policy"}, {"type": "advisories", "url": f"{repository}/security/advisories"}, - {"type": "license", "url": f"{repository}/blob/main/LICENSE"}, + {"type": "license", "url": f"{repository}/blob/{tree}/LICENSE"}, {"type": "vcs", "url": f"{repository}/tree/c0ffee", "comment": comment}, ], "properties": [{"name": "example:vcs-commit", "value": "c0ffee"}, *dirty_properties], @@ -269,14 +274,26 @@ def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> Bundl ] -def test_build_script_is_a_linked_tool(describe: Describe, settings: Settings) -> None: +@pytest.mark.parametrize( + ("revision", "tree"), + [ + pytest.param(None, "main", id="unknown"), + pytest.param(Revision(commit="c0ffee", dirty=False), "c0ffee", id="clean"), + pytest.param(Revision(commit="c0ffee", dirty=True), "main", id="dirty"), + ], +) +def test_build_script_is_a_linked_tool( + describe: Describe, settings: Settings, facts: BuildFacts, revision: Revision | None, tree: str +) -> None: source = SourceTree( copyright=None, bundled=[], build_scripts=[BuildScript(path=PurePosixPath("hatch_build.py"), sha256="ef" * 32)], ) document = describe( - settings=dataclasses.replace(settings, repository="https://github.com/example/example"), source=source + settings=dataclasses.replace(settings, repository="https://github.com/example/example"), + facts=dataclasses.replace(facts, revision=revision), + source=source, ) assert document["metadata"]["tools"]["components"][0] == { "type": "application", @@ -284,7 +301,9 @@ def test_build_script_is_a_linked_tool(describe: Describe, settings: Settings) - "name": "hatch_build.py", "version": "1.0", "hashes": [{"alg": "SHA-256", "content": "ef" * 32}], - "externalReferences": [{"type": "vcs", "url": "https://github.com/example/example/blob/main/hatch_build.py"}], + "externalReferences": [ + {"type": "vcs", "url": f"https://github.com/example/example/blob/{tree}/hatch_build.py"} + ], } From eb3cd2bb3161455845d483b11bd97de4b030a260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:15:01 -0800 Subject: [PATCH 087/115] =?UTF-8?q?=F0=9F=90=9B=20fix(model):=20encode=20U?= =?UTF-8?q?RLs=20validate=20would=20reject?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Project-URL or Home-page went into externalReferences as written, so an scp-style git address, a space or a brace built an SBOM that embed wrote without complaint and validate then rejected, since CycloneDX requires an IRI reference. iri_reference percent-encodes each character RFC 3987 allows nowhere in its part of the reference, and gives up on a scheme, IP literal or port no encoding mends. An encoded URL is linked; one it gives up on is left out, and a python:project-url or python:home-page property keeps the header's text either way. The repository taken from a Project-URL skips an unusable one for the next. --- docs/explanation.rst | 4 ++ src/python_sbom/_model/_components.py | 30 ++++++--- src/python_sbom/_model/_identity.py | 11 ++- src/python_sbom/_model/_iri.py | 96 +++++++++++++++++++++++++++ tests/_model/test_components.py | 52 +++++++++++++++ tests/_model/test_identity.py | 13 ++++ tests/_model/test_iri.py | 47 +++++++++++++ 7 files changed, 239 insertions(+), 14 deletions(-) create mode 100644 src/python_sbom/_model/_iri.py create mode 100644 tests/_model/test_iri.py diff --git a/docs/explanation.rst b/docs/explanation.rst index dda598e..8870e42 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -68,6 +68,10 @@ with a repository, the license file follow. A build from a git checkout adds the revision. Links to the license file and build scripts point at that commit when the checkout is clean, and at the default branch otherwise. +CycloneDX requires each URL to be an IRI reference. A project URL or home page that is not one gets percent-encoded, +or left out when encoding cannot mend it, as with an scp-style ``git@host:path`` address; either way a +``python:project-url`` or ``python:home-page`` property keeps the text as written. + The components below the root are the wheels the project bundles and the runtime dependencies it declares. A bundled wheel carries its SHA-256, its license, a component per file from its ``RECORD``, and the packages it vendors. A declared dependency carries no version, because the installer picks it at install time; the document marks the root's diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index 85be170..0895457 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -4,11 +4,11 @@ from email.utils import getaddresses from itertools import starmap from typing import TYPE_CHECKING, Final -from urllib.parse import urlsplit from packaging.utils import canonicalize_name from ._document import walk_components +from ._iri import iri_reference from ._licenses import licenses from ._names import purl, pypi_url, reference_type, tool_ref from ._types import Dependency @@ -78,7 +78,7 @@ def component_from_metadata(metadata: CoreMetadata) -> Component: break if declared := licenses(metadata): component["licenses"] = declared - component["externalReferences"] = _project_references(metadata) + component["externalReferences"], unlinked = _project_references(metadata) # core metadata headers copied verbatim, so a reader sees what the distribution declared headers: Final[list[tuple[str, list[str]]]] = [ ("requires-python", metadata.values("requires_python")), @@ -90,6 +90,7 @@ def component_from_metadata(metadata: CoreMetadata) -> Component: component["properties"] = [] for header, values in headers: component["properties"].extend({"name": f"python:{header}", "value": value} for value in values) + component["properties"].extend(unlinked) return component @@ -105,16 +106,23 @@ def _contact(name: str, email: str) -> Contact: return contact -def _project_references(metadata: CoreMetadata) -> list[ExternalReference]: - # a home page that is not an http(s) URL, such as UNKNOWN, is left out +def _project_references(metadata: CoreMetadata) -> tuple[list[ExternalReference], list[Property]]: + # a URL CycloneDX would reject is percent-encoded where that keeps what it points at and left out where it cannot + # be, as an scp-style git address; either way the header's text is kept verbatim as a property references: Final[list[ExternalReference]] = [] - if (home_page := metadata.first("home_page")) and urlsplit(home_page).scheme in {"http", "https"}: - references.append({"type": "website", "url": home_page}) - references.extend( - {"type": reference_type(label), "url": url, "comment": f"Project-URL: {label}"} - for label, url in metadata.project_urls() - ) - return references + unlinked: Final[list[Property]] = [] + # a home page that is not an http(s) URL, such as UNKNOWN, is left out + if (home_page := metadata.first("home_page")) and re.match(r"https?:", home_page, re.IGNORECASE): + if (url := iri_reference(home_page)) is not None: + references.append({"type": "website", "url": url}) + if url != home_page: + unlinked.append({"name": "python:home-page", "value": home_page}) + for label, link in metadata.project_urls(): + if (url := iri_reference(link)) is not None: + references.append({"type": reference_type(label), "url": url, "comment": f"Project-URL: {label}"}) + if url != link: + unlinked.append({"name": "python:project-url", "value": f"{label}, {link}"}) + return references, unlinked def _evidence(field: str, technique: str, value: str) -> Identity: diff --git a/src/python_sbom/_model/_identity.py b/src/python_sbom/_model/_identity.py index c470c8f..732a1f7 100644 --- a/src/python_sbom/_model/_identity.py +++ b/src/python_sbom/_model/_identity.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING +from ._iri import iri_reference from ._names import property_prefix, reference_type if TYPE_CHECKING: @@ -13,9 +14,13 @@ def project_identity(metadata: CoreMetadata, settings: Settings) -> ProjectIdentity: project = property_prefix(metadata.name) - repository = settings.repository or next( - (url.rstrip("/") for label, url in metadata.project_urls() if reference_type(label) == "vcs"), None - ) + repository = settings.repository + # links into the repository extend its URL, so one that cannot be an IRI reference, as an scp-style git address, is + # passed over for the next + for label, url in [] if repository else metadata.project_urls(): + if reference_type(label) == "vcs" and (link := iri_reference(url)) is not None: + repository = link.rstrip("/") + break base = repository or f"https://pypi.org/project/{project}" return ProjectIdentity( property_prefix=project, diff --git a/src/python_sbom/_model/_iri.py b/src/python_sbom/_model/_iri.py new file mode 100644 index 0000000..572226d --- /dev/null +++ b/src/python_sbom/_model/_iri.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import ipaddress +import re +import string +from typing import Final, cast +from urllib.parse import quote + +# RFC 3986 appendix B: splits any text into scheme, authority, path, query and fragment +_PARTS: Final[re.Pattern[str]] = re.compile( + r"(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?", re.DOTALL +) +_SCHEME: Final[re.Pattern[str]] = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*") +# what may follow the host: nothing, or a colon and the port's digits +_PORT: Final[re.Pattern[str]] = re.compile(r"(?::[0-9]*)?") +_IP_FUTURE: Final[re.Pattern[str]] = re.compile(r"v[0-9A-Fa-f]+\.[A-Za-z0-9._~!$&'()*+,;=:-]+") +# a valid percent-encoding, or any one character +_TOKEN: Final[re.Pattern[str]] = re.compile(r"%[0-9A-Fa-f]{2}|.", re.DOTALL) +_REG_NAME: Final[str] = f"{string.ascii_letters}{string.digits}-._~!$&'()*+,;=" +_USERINFO: Final[str] = f"{_REG_NAME}:" +_PATH: Final[str] = f"{_USERINFO}@/" +# the query and the fragment allow the same characters +_QUERY: Final[str] = f"{_PATH}?" +# the characters beyond ASCII an IRI holds unencoded, RFC 3987's ucschar +_UCSCHAR: Final[tuple[tuple[int, int], ...]] = ( + (0xA0, 0xD7FF), + (0xF900, 0xFDCF), + (0xFDF0, 0xFFEF), + *((plane << 16, plane << 16 | 0xFFFD) for plane in range(1, 14)), + (0xE1000, 0xEFFFD), +) + + +def iri_reference(text: str) -> str | None: + # the form CycloneDX requires of a URL: a character no part of an IRI reference allows is percent-encoded, which + # keeps what the text points at; a scheme, IP literal or port no encoding mends, as in the scp-style + # git@github.com:org/repo.git, leaves None; the pattern matches any text, every part of it optional + match: Final = cast("re.Match[str]", _PARTS.fullmatch(text)) + scheme, authority, path, query, fragment = match.groups() + if scheme is not None and not _SCHEME.fullmatch(scheme): + return None + parts: Final[list[str]] = [] if scheme is None else [f"{scheme}:"] + if authority is not None: + if (encoded := _authority(authority)) is None: + return None + parts.append(f"//{encoded}") + parts.append(_encoded(path, _PATH)) + if query is not None: + parts.append(f"?{_encoded(query, _QUERY)}") + if fragment is not None: + parts.append(f"#{_encoded(fragment, _QUERY)}") + return "".join(parts) + + +def _authority(authority: str) -> str | None: + userinfo, at, host_port = authority.rpartition("@") + if host_port.startswith("["): + literal, _, port = host_port[1:].partition("]") + if not _ip_literal(literal): + return None + host = f"[{literal}]" + else: + name, colon, number = host_port.partition(":") + host, port = _encoded(name, _REG_NAME), f"{colon}{number}" + if not _PORT.fullmatch(port): + return None + return f"{_encoded(userinfo, _USERINFO)}{at}{host}{port}" + + +def _ip_literal(literal: str) -> bool: + if _IP_FUTURE.fullmatch(literal): + return True + # a zone index has no place in a URI's IP literal + if "%" in literal: + return False + try: + ipaddress.IPv6Address(literal) + except ValueError: + return False + return True + + +def _encoded(text: str, allowed: str) -> str: + return "".join(token if _kept(token, allowed) else quote(token, safe="") for token in _TOKEN.findall(text)) + + +def _kept(token: str, allowed: str) -> bool: + # a token longer than a character is a valid percent-encoding + if len(token) > 1 or token in allowed: + return True + return any(low <= ord(token) <= high for low, high in _UCSCHAR) + + +__all__ = [ + "iri_reference", +] diff --git a/tests/_model/test_components.py b/tests/_model/test_components.py index c6439c4..df86209 100644 --- a/tests/_model/test_components.py +++ b/tests/_model/test_components.py @@ -363,3 +363,55 @@ def bundled(ref: str, *children: Component) -> Component: {"ref": "bundled:b.whl", "dependsOn": ["bundled:b.whl#vendored/pkg:pypi/six@1"]}, {"ref": "bundled:c.whl", "dependsOn": ["bundled:c.whl#vendored/pkg:pypi/six@1"]}, ] + + +@pytest.mark.parametrize( + ("fields", "references", "kept"), + [ + pytest.param( + "Project-URL: Source, git@github.com:example/pkg.git\n", + [], + [{"name": "python:project-url", "value": "Source, git@github.com:example/pkg.git"}], + id="scp-style-git", + ), + pytest.param( + "Project-URL: Docs, https://example.org/my docs\n", + [{"type": "documentation", "url": "https://example.org/my%20docs", "comment": "Project-URL: Docs"}], + [{"name": "python:project-url", "value": "Docs, https://example.org/my docs"}], + id="space", + ), + pytest.param( + "Project-URL: Template, https://example.org/{name}/x\n", + [{"type": "other", "url": "https://example.org/%7Bname%7D/x", "comment": "Project-URL: Template"}], + [{"name": "python:project-url", "value": "Template, https://example.org/{name}/x"}], + id="braces", + ), + pytest.param( + "Home-page: https://example.org/a b\n", + [{"type": "website", "url": "https://example.org/a%20b"}], + [{"name": "python:home-page", "value": "https://example.org/a b"}], + id="home-page-space", + ), + pytest.param( + "Home-page: https://example.org:port/\n", + [], + [{"name": "python:home-page", "value": "https://example.org:port/"}], + id="home-page-bad-port", + ), + pytest.param( + "Home-page: https://example.org/\nProject-URL: Source, https://github.com/example/pkg\n", + [ + {"type": "website", "url": "https://example.org/"}, + {"type": "vcs", "url": "https://github.com/example/pkg", "comment": "Project-URL: Source"}, + ], + [], + id="valid-kept-as-is", + ), + ], +) +def test_url_no_iri_reference_is_encoded_or_kept_as_property( + fields: str, references: list[dict[str, str]], kept: list[dict[str, str]] +) -> None: + # validation requires an IRI reference, so a URL copied as written would build an SBOM validate rejects + component = component_from_metadata(parse_metadata(f"Name: a\nVersion: 1\n{fields}")) + assert (component["externalReferences"], component["properties"]) == (references, kept) diff --git a/tests/_model/test_identity.py b/tests/_model/test_identity.py index 0a14415..3abc30d 100644 --- a/tests/_model/test_identity.py +++ b/tests/_model/test_identity.py @@ -29,6 +29,19 @@ "https://git.example.org/example", id="setting-wins", ), + pytest.param( + None, + "Project-URL: Source, git@github.com:example/example.git\n" + "Project-URL: Repository, https://github.com/example/example\n", + "https://github.com/example/example", + id="scp-style-passed-over", + ), + pytest.param( + None, + "Project-URL: Source, https://git.example.org/{team}/example\n", + "https://git.example.org/%7Bteam%7D/example", + id="encoded", + ), ], ) def test_project_identity_repository( diff --git a/tests/_model/test_iri.py b/tests/_model/test_iri.py new file mode 100644 index 0000000..ef2b6fa --- /dev/null +++ b/tests/_model/test_iri.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from jsonschema_rs import Draft7Validator + +from python_sbom._model._iri import iri_reference + +_IRI_REFERENCE: Final[Draft7Validator] = Draft7Validator( + {"type": "string", "format": "iri-reference"}, validate_formats=True +) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + pytest.param("https://example.org/a?b=1#c", "https://example.org/a?b=1#c", id="valid"), + pytest.param("https://example.org/รผ๐Ÿ˜€", "https://example.org/รผ๐Ÿ˜€", id="unicode"), + pytest.param("https://example.org/%e2%82", "https://example.org/%e2%82", id="percent-encoded"), + pytest.param("UNKNOWN", "UNKNOWN", id="relative"), + pytest.param("//example.org/a", "//example.org/a", id="network-path"), + pytest.param("ssh://git@github.com/org/repo", "ssh://git@github.com/org/repo", id="userinfo"), + pytest.param("http://[::1]:80/", "http://[::1]:80/", id="ipv6"), + pytest.param("http://[v1.x]/", "http://[v1.x]/", id="ip-future"), + pytest.param("https://example.org:/", "https://example.org:/", id="empty-port"), + pytest.param("https://example.org/a b", "https://example.org/a%20b", id="space"), + pytest.param("https://example.org/{a}", "https://example.org/%7Ba%7D", id="braces"), + pytest.param('https://example.org/|^`\\"', "https://example.org/%3Ca%3E%7C%5E%60%5C%22", id="delimiters"), + pytest.param("https://example.org/a[1]?b[2]", "https://example.org/a%5B1%5D?b%5B2%5D", id="brackets"), + pytest.param("https://example.org/a#b#c", "https://example.org/a#b%23c", id="second-hash"), + pytest.param("https://example.org/%zz", "https://example.org/%25zz", id="bare-percent"), + pytest.param("https://example.org/a\tb", "https://example.org/a%09b", id="control"), + pytest.param("https://example.org/\x85\ue000", "https://example.org/%C2%85%EE%80%80", id="no-ucschar"), + pytest.param("https://ex ample.org/", "https://ex%20ample.org/", id="host-space"), + pytest.param("https://a@b@example.org/", "https://a%40b@example.org/", id="userinfo-at"), + pytest.param("git@github.com:org/repo.git", None, id="scp-style"), + pytest.param("https://example.org:port/", None, id="port"), + pytest.param("https://example.org:\uff18\uff10/", None, id="non-ascii-port"), + pytest.param("http://[bad/", None, id="unclosed-literal"), + pytest.param("http://[fe80::1%eth0]/", None, id="zone-index"), + pytest.param("http://[::1]x/", None, id="after-literal"), + ], +) +def test_iri_reference(text: str, expected: str | None) -> None: + assert iri_reference(text) == expected + assert expected is None or _IRI_REFERENCE.is_valid(expected) From 43311391533a53f644bc4b4d0187159b189a3399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:18:13 -0800 Subject: [PATCH 088/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20walk?= =?UTF-8?q?=20components=20the=20same=20way=20in=20validate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The core cleanup moved revision_reader to _collect, dropped render_spdx's version argument and made walk_components the one tree walker; validate and the archive command now use them too. --- docs/usage.rst | 3 ++- src/python_sbom/_cli/_commands.py | 2 +- src/python_sbom/_validate/_document.py | 17 +++-------------- src/python_sbom/_validate/_licenses.py | 20 +++++++++----------- tests/_validate/test_schemas.py | 4 ++-- 5 files changed, 17 insertions(+), 29 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 780d02b..363a3b4 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -42,7 +42,8 @@ fails the build with a pointer to ``[tool.python-sbom]``. repository = 'https://gitlab.com/example/project' ``default-branch`` - The branch in links to files of the repository, ``main`` by default. + The branch in links to files of the repository when the build's commit is unknown or the checkout has + uncommitted changes, ``main`` by default; otherwise the links name the commit. .. code-block:: toml diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py index f0b35c4..c376f20 100644 --- a/src/python_sbom/_cli/_commands.py +++ b/src/python_sbom/_cli/_commands.py @@ -87,9 +87,9 @@ def add_archive(commands: _SubParsersAction[ArgumentParser]) -> None: def _archive(options: Options) -> bool: from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # each command loads only what it runs + from python_sbom._collect._revision import revision_reader # ruff: ignore[import-outside-top-level] # each command loads only what it runs from python_sbom._formats._json import dumps # ruff: ignore[import-outside-top-level] # each command loads only what it runs from python_sbom._generate._archive import describe_archive # ruff: ignore[import-outside-top-level] # each command loads only what it runs - from python_sbom._generate._embed import revision_reader # ruff: ignore[import-outside-top-level] # each command loads only what it runs from python_sbom._wheel._archive import append_member # ruff: ignore[import-outside-top-level] # each command loads only what it runs root = Path.cwd() if options.root is None else options.root diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 389ac50..c9d258f 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -9,8 +9,8 @@ from python_sbom._errors import MissingExtraError from python_sbom._formats._spdx import render_spdx +from python_sbom._model._document import walk_components from python_sbom._model._names import SBOM_FILE -from python_sbom.version import __version__ if TYPE_CHECKING: from typing import TypeIs # new in 3.13; annotations here are never evaluated @@ -108,7 +108,7 @@ def _problems(raw: bytes, allowed_licenses: frozenset[str] | None, max_component if not _SERIAL.fullmatch(serial := loaded.get("serialNumber", "")): return [f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {serial!r}"] # the SPDX rendering of a document whose references are broken only repeats those problems - problems: Final[list[str]] = _reference_problems(loaded) or spdx_problems(render_spdx(loaded, __version__)) + problems: Final[list[str]] = _reference_problems(loaded) or spdx_problems(render_spdx(loaded)) if allowed_licenses is not None: problems.extend(license_problems(loaded["components"], allowed_licenses)) return problems @@ -163,7 +163,7 @@ def is_document(value: object) -> TypeIs[Document]: def _reference_problems(document: Document) -> list[str]: root: Final[Component] = document["metadata"]["component"] tools: Final[list[Component]] = document["metadata"]["tools"]["components"] - components: Final[list[Component]] = _walk([root, *document["components"], *tools]) + components: Final[list[Component]] = list(walk_components([root, *document["components"], *tools])) if nameless := sorted(component["name"] for component in components if "bom-ref" not in component): return [f"components lack a bom-ref: {nameless}"] problems: Final[list[str]] = [] @@ -190,17 +190,6 @@ def _reference_problems(document: Document) -> list[str]: return problems -def _walk(components: list[Component]) -> list[Component]: - # iterative, so a deeply nested hostile document cannot exhaust the stack - found: Final[list[Component]] = [] - pending: Final[list[Component]] = list(components) - while pending: - component = pending.pop() - found.append(component) - pending.extend(component.get("components", [])) - return found - - __all__ = [ "MAX_COMPONENTS", "MAX_TEXT", diff --git a/src/python_sbom/_validate/_licenses.py b/src/python_sbom/_validate/_licenses.py index a4b22f8..5f16461 100644 --- a/src/python_sbom/_validate/_licenses.py +++ b/src/python_sbom/_validate/_licenses.py @@ -11,6 +11,8 @@ get_license_index, ) +from python_sbom._model._document import walk_components + if TYPE_CHECKING: from license_expression import LicenseExpression @@ -38,17 +40,13 @@ def license_problems(components: list[Component], allowed: frozenset[str]) -> li def _shipped(components: list[Component]) -> list[Component]: # only what the artifact carries bytes for is judged: a declared requirement resolves at install time, and a library - # a bundled wheel vendors ships under that wheel's terms; iterative, so deep nesting cannot exhaust the stack - found: Final[list[Component]] = [] - pending: Final[list[Component]] = list(reversed(components)) - while pending: - component = pending.pop() - if component["type"] == "library" and ( - "hashes" in component or any(child["type"] == "file" for child in component.get("components", [])) - ): - found.append(component) - pending.extend(reversed(component.get("components", []))) - return found + # a bundled wheel vendors ships under that wheel's terms + return [ + component + for component in walk_components(components) + if component["type"] == "library" + and ("hashes" in component or any(child["type"] == "file" for child in component.get("components", []))) + ] def _judge(component: Component, allowed: frozenset[str], folded: frozenset[str]) -> str | None: diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index e443a0b..8a026ff 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -192,7 +192,7 @@ def _declare(spdx: Spdx, expression: str, text: str | None = None) -> None: ) def test_spdx_problems_match_spdx_tools(describe: Describe, mutate: Callable[[Spdx], None], valid: bool) -> None: # the full validation of spdx-tools, whose time grows with the square of the elements, is the reference - spdx = render_spdx(describe("Requires-Dist: click\n"), "1") + spdx = render_spdx(describe("Requires-Dist: click\n")) mutate(spdx) reference = validate_full_spdx_document(JsonLikeDictParser().parse(dict(spdx))) assert spdx_problems(spdx) == [f"SPDX rendering invalid: {message.validation_message}" for message in reference] @@ -207,4 +207,4 @@ def test_many_packages_pass(describe: Describe) -> None: ] document["components"].extend(libraries) document["dependencies"][0].setdefault("dependsOn", []).extend(f"r{index}" for index in range(20_000)) - assert spdx_problems(render_spdx(document, "1")) == [] + assert spdx_problems(render_spdx(document)) == [] From 76676dde95e261454297afe8e1f9537e8b80fc8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:09:53 -0800 Subject: [PATCH 089/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20give?= =?UTF-8?q?=20each=20command=20its=20own=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each command lives in its own module with top-level imports, and the parser registers its handler as a module:function name that main resolves with pkgutil.resolve_name. --help, --version and every command import only what they run, without the ten import-outside-top-level suppressions. Each command reads its options through its own protocol instead of one namespace declaring every command's fields. --- src/python_sbom/_cli/_archive.py | 44 +++++ src/python_sbom/_cli/_commands.py | 194 --------------------- src/python_sbom/_cli/_embed.py | 28 +++ src/python_sbom/_cli/_main.py | 6 +- src/python_sbom/_cli/_options.py | 32 ---- src/python_sbom/_cli/_parser.py | 120 ++++++++++++- src/python_sbom/_cli/_spdx.py | 36 ++++ src/python_sbom/_cli/_validate.py | 40 +++++ tests/_cli/conftest.py | 25 +++ tests/_cli/test_archive.py | 97 +++++++++++ tests/_cli/test_commands.py | 278 ------------------------------ tests/_cli/test_embed.py | 77 +++++++++ tests/_cli/test_main.py | 49 ++++++ tests/_cli/test_spdx.py | 37 ++++ tests/_cli/test_validate.py | 63 +++++++ 15 files changed, 614 insertions(+), 512 deletions(-) create mode 100644 src/python_sbom/_cli/_archive.py delete mode 100644 src/python_sbom/_cli/_commands.py create mode 100644 src/python_sbom/_cli/_embed.py delete mode 100644 src/python_sbom/_cli/_options.py create mode 100644 src/python_sbom/_cli/_spdx.py create mode 100644 src/python_sbom/_cli/_validate.py create mode 100644 tests/_cli/conftest.py create mode 100644 tests/_cli/test_archive.py delete mode 100644 tests/_cli/test_commands.py create mode 100644 tests/_cli/test_embed.py create mode 100644 tests/_cli/test_main.py create mode 100644 tests/_cli/test_spdx.py create mode 100644 tests/_cli/test_validate.py diff --git a/src/python_sbom/_cli/_archive.py b/src/python_sbom/_cli/_archive.py new file mode 100644 index 0000000..7a4ab84 --- /dev/null +++ b/src/python_sbom/_cli/_archive.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import sys +from pathlib import Path, PurePosixPath +from typing import Protocol + +from python_sbom._collect._facts import collect_facts +from python_sbom._collect._revision import revision_reader +from python_sbom._formats._json import dumps +from python_sbom._generate._archive import describe_archive +from python_sbom._wheel._archive import append_member + +from ._parser import BuildOptions + + +class _Options(BuildOptions, Protocol): + archive: Path + build_scripts: list[str] | None + output: Path | None + embed: bool + + +def run(options: _Options) -> bool: + root = Path.cwd() if options.root is None else options.root + facts = collect_facts(root, revision_reader(options.commit), path=options.build_env) + document = describe_archive( + root, + options.archive, + facts=facts, + provider=options.content_provider, + build_scripts=tuple(PurePosixPath(script) for script in options.build_scripts or []), + ) + name = f"{options.archive.name}.cdx.json" + (target := options.output or options.archive.with_name(name)).write_bytes(content := dumps(document)) + sys.stdout.write(f"wrote {target}\n") + if options.embed: + append_member(options.archive, name, content, facts.epoch) + sys.stdout.write(f"wrote {name} into {options.archive.name}\n") + return True + + +__all__ = [ + "run", # simplint: ignore[SMP402] # the parser names it for pkgutil.resolve_name +] diff --git a/src/python_sbom/_cli/_commands.py b/src/python_sbom/_cli/_commands.py deleted file mode 100644 index c376f20..0000000 --- a/src/python_sbom/_cli/_commands.py +++ /dev/null @@ -1,194 +0,0 @@ -from __future__ import annotations - -import json -import sys -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING - -from python_sbom._errors import SbomError -from python_sbom._validate._document import MAX_COMPONENTS - -if TYPE_CHECKING: - from argparse import ArgumentParser, _SubParsersAction - - from ._options import Options - - -def add_embed(commands: _SubParsersAction[ArgumentParser]) -> None: - parser = commands.add_parser( - "embed", - help="write the SBOM into wheels another build backend built", - description="Write the SBOM the hatchling hook writes into wheels another build backend built, such as" - " meson-python. Options come from the [tool.python-sbom] table of the project's pyproject.toml.", - ) - parser.set_defaults(handler=_embed) - parser.add_argument("wheels", type=Path, nargs="+", metavar="WHEEL", help="wheel to rewrite in place") - _add_build_options(parser) - - -def _embed(options: Options) -> bool: - from python_sbom._generate._embed import embed_sbom # ruff: ignore[import-outside-top-level] # each command loads only what it runs - - root = Path.cwd() if options.root is None else options.root - for wheel in options.wheels: - written = embed_sbom( - wheel, root, provider=options.content_provider, commit=options.commit, build_env=options.build_env - ) - sys.stdout.write(f"wrote {written} into {wheel.name}\n") - return True - - -def _add_build_options(parser: ArgumentParser) -> None: - parser.add_argument( - "--root", type=Path, metavar="DIR", help="project directory of the artifact (default: current directory)" - ) - parser.add_argument( - "--commit", metavar="SHA", help="source revision to record, for an artifact built from an sdist" - ) - parser.add_argument( - "--build-env", - action="append", - metavar="DIR", - help="site-packages directory of the environment that built the artifact; repeat it per directory" - " (default: this interpreter's)", - ) - parser.add_argument( - "--content-provider", - metavar="MODULE:ATTRIBUTE", - help="content provider, overriding the content-provider option", - ) - - -def add_archive(commands: _SubParsersAction[ArgumentParser]) -> None: - parser = commands.add_parser( - "archive", - help="write the SBOM of a zip archive that is not a wheel, such as a zipapp", - description="Describe a zip archive that holds the project's .dist-info at its top level, such as a zipapp:" - " the archive's own files, the project's distribution, and every wheel it bundles, as a file or unpacked," - " described as the wheel's SBOM describes them. Options come from [tool.python-sbom].", - ) - parser.set_defaults(handler=_archive) - parser.add_argument("archive", type=Path, metavar="ARCHIVE", help="archive to describe") - _add_build_options(parser) - parser.add_argument( - "--build-script", - dest="build_scripts", - action="append", - metavar="PATH", - help="script, relative to the project, that built the archive; repeat it per script", - ) - parser.add_argument( - "--output", type=Path, metavar="FILE", help="SBOM file to write (default: .cdx.json next to it)" - ) - parser.add_argument( - "--embed", action="store_true", help="also add the SBOM to the archive, at its top level as .cdx.json" - ) - - -def _archive(options: Options) -> bool: - from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # each command loads only what it runs - from python_sbom._collect._revision import revision_reader # ruff: ignore[import-outside-top-level] # each command loads only what it runs - from python_sbom._formats._json import dumps # ruff: ignore[import-outside-top-level] # each command loads only what it runs - from python_sbom._generate._archive import describe_archive # ruff: ignore[import-outside-top-level] # each command loads only what it runs - from python_sbom._wheel._archive import append_member # ruff: ignore[import-outside-top-level] # each command loads only what it runs - - root = Path.cwd() if options.root is None else options.root - facts = collect_facts(root, revision_reader(options.commit), path=options.build_env) - document = describe_archive( - root, - options.archive, - facts=facts, - provider=options.content_provider, - build_scripts=tuple(PurePosixPath(script) for script in options.build_scripts or []), - ) - name = f"{options.archive.name}.cdx.json" - (target := options.output or options.archive.with_name(name)).write_bytes(content := dumps(document)) - sys.stdout.write(f"wrote {target}\n") - if options.embed: - append_member(options.archive, name, content, facts.epoch) - sys.stdout.write(f"wrote {name} into {options.archive.name}\n") - return True - - -def add_spdx(commands: _SubParsersAction[ArgumentParser]) -> None: - parser = commands.add_parser("spdx", help="render a CycloneDX SBOM as SPDX 2.3 JSON") - parser.set_defaults(handler=_spdx) - parser.add_argument("source", type=Path, metavar="SOURCE", help="CycloneDX JSON file to read") - parser.add_argument("target", type=Path, metavar="TARGET", help="SPDX JSON file to write") - - -def _spdx(options: Options) -> bool: - from python_sbom._formats._spdx import to_spdx # ruff: ignore[import-outside-top-level] # each command loads only what it runs - from python_sbom._validate._document import is_document # ruff: ignore[import-outside-top-level] # each command loads only what it runs - - if not is_document(loaded := json.loads(options.source.read_text(encoding="utf-8"))): - msg = f"{options.source} is not a CycloneDX document python-sbom can render" - raise SbomError(msg) - try: - rendered = to_spdx(loaded) - except (KeyError, TypeError) as error: - # the CLI renders without the schema check, which needs the validate extra - msg = f"{options.source} is not a CycloneDX document python-sbom can render: {error!r}" - raise SbomError(msg) from error - # bytes, so the line endings are the same on every operating system - options.target.write_bytes(rendered.encode()) - return True - - -def add_validate(commands: _SubParsersAction[ArgumentParser]) -> None: - parser = commands.add_parser( - "validate", - help="validate the SBOMs in wheels and archives", - description="Check every CycloneDX SBOM a wheel or archive holds against the CycloneDX 1.6 schema; for one" - " python-sbom wrote, also its serial number, references, SPDX rendering and, with --allow-license, the licenses" - " of what it ships. Needs the validate extra: python-sbom[validate].", - ) - parser.set_defaults(handler=_validate) - parser.add_argument( - "paths", type=Path, nargs="+", metavar="PATH", help="wheel, archive, or directory holding wheels" - ) - parser.add_argument( - "--allow-license", - dest="allowed_licenses", - action="append", - metavar="SPDX-ID", - help="SPDX license id shipped components may use; repeat it per id, leave it out to skip the license policy", - ) - parser.add_argument( - "--max-components", - type=int, - default=MAX_COMPONENTS, - metavar="N", - help="reject an SBOM listing more than N components, nested ones included, without checking it further, which" - f" bounds the work an untrusted SBOM can ask for (default: {MAX_COMPONENTS})", - ) - - -def _validate(options: Options) -> bool: - from python_sbom._validate._artifact import validate as validate_artifact # ruff: ignore[import-outside-top-level] # each command loads only what it runs - - allowed = None if options.allowed_licenses is None else frozenset(options.allowed_licenses) - if not ( - artifacts := sorted( - artifact for path in options.paths for artifact in (path.glob("*.whl") if path.is_dir() else [path]) - ) - ): - sys.stderr.write(f"no wheel found in {', '.join(map(str, options.paths))}\n") - return False - valid = True - for artifact in artifacts: - for problem in (problems := validate_artifact(artifact, allowed, options.max_components)): - sys.stderr.write(f"SBOM invalid: {artifact.name}: {problem}\n") - if problems: - valid = False - else: - sys.stdout.write(f"SBOM in {artifact.name} is valid\n") - return valid - - -__all__ = [ - "add_archive", - "add_embed", - "add_spdx", - "add_validate", -] diff --git a/src/python_sbom/_cli/_embed.py b/src/python_sbom/_cli/_embed.py new file mode 100644 index 0000000..ec7c1db --- /dev/null +++ b/src/python_sbom/_cli/_embed.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Protocol + +from python_sbom._generate._embed import embed_sbom + +from ._parser import BuildOptions + + +class _Options(BuildOptions, Protocol): + wheels: list[Path] + + +def run(options: _Options) -> bool: + root = Path.cwd() if options.root is None else options.root + for wheel in options.wheels: + written = embed_sbom( + wheel, root, provider=options.content_provider, commit=options.commit, build_env=options.build_env + ) + sys.stdout.write(f"wrote {written} into {wheel.name}\n") + return True + + +__all__ = [ + "run", # simplint: ignore[SMP402] # the parser names it for pkgutil.resolve_name +] diff --git a/src/python_sbom/_cli/_main.py b/src/python_sbom/_cli/_main.py index 2c7d71b..8888134 100644 --- a/src/python_sbom/_cli/_main.py +++ b/src/python_sbom/_cli/_main.py @@ -1,11 +1,11 @@ from __future__ import annotations +import pkgutil import sys from typing import TYPE_CHECKING from python_sbom._errors import SbomError -from ._options import Options from ._parser import build_parser if TYPE_CHECKING: @@ -13,9 +13,9 @@ def main(args: Sequence[str] | None = None) -> None: - options = build_parser().parse_args(args, namespace=Options()) + options = build_parser().parse_args(args) try: - succeeded = options.handler(options) + succeeded = pkgutil.resolve_name(options.handler)(options) except SbomError as error: sys.stderr.write(f"python-sbom: {error}\n") succeeded = False diff --git a/src/python_sbom/_cli/_options.py b/src/python_sbom/_cli/_options.py deleted file mode 100644 index 81d0afc..0000000 --- a/src/python_sbom/_cli/_options.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -from argparse import Namespace -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Callable - from pathlib import Path - - -class Options(Namespace): - # each subcommand sets only its own fields; its handler reads only those - handler: Callable[[Options], bool] - wheels: list[Path] - archive: Path - root: Path | None - commit: str | None - build_env: list[str] | None - build_scripts: list[str] | None - content_provider: str | None - output: Path | None - embed: bool - source: Path - target: Path - paths: list[Path] - allowed_licenses: list[str] | None - max_components: int - - -__all__ = [ - "Options", -] diff --git a/src/python_sbom/_cli/_parser.py b/src/python_sbom/_cli/_parser.py index 2f1576f..b79ec4d 100644 --- a/src/python_sbom/_cli/_parser.py +++ b/src/python_sbom/_cli/_parser.py @@ -1,21 +1,131 @@ from __future__ import annotations -from argparse import ArgumentParser +from argparse import ArgumentParser, _SubParsersAction +from pathlib import Path +from typing import Protocol +from python_sbom._validate._document import MAX_COMPONENTS from python_sbom.version import __version__ -from ._commands import add_archive, add_embed, add_spdx, add_validate - def build_parser() -> ArgumentParser: parser = ArgumentParser(prog="python-sbom", description="Write, convert and check the CycloneDX SBOMs of wheels.") parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") commands = parser.add_subparsers(required=True) - for add in (add_embed, add_archive, add_spdx, add_validate): - add(commands) + _add_embed(commands) + _add_archive(commands) + _add_spdx(commands) + _add_validate(commands) return parser +def _add_embed(commands: _SubParsersAction[ArgumentParser]) -> None: + parser = commands.add_parser( + "embed", + help="write the SBOM into wheels another build backend built", + description="Write the SBOM the hatchling hook writes into wheels another build backend built, such as" + " meson-python. Options come from the [tool.python-sbom] table of the project's pyproject.toml.", + ) + # a name, not the function, so each command imports only what it runs and --help imports none of it + parser.set_defaults(handler="python_sbom._cli._embed:run") + parser.add_argument("wheels", type=Path, nargs="+", metavar="WHEEL", help="wheel to rewrite in place") + _add_build_options(parser) + + +def _add_build_options(parser: ArgumentParser) -> None: + parser.add_argument( + "--root", type=Path, metavar="DIR", help="project directory of the artifact (default: current directory)" + ) + parser.add_argument( + "--commit", metavar="SHA", help="source revision to record, for an artifact built from an sdist" + ) + parser.add_argument( + "--build-env", + action="append", + metavar="DIR", + help="site-packages directory of the environment that built the artifact; repeat it per directory" + " (default: this interpreter's)", + ) + parser.add_argument( + "--content-provider", + metavar="MODULE:ATTRIBUTE", + help="content provider, overriding the content-provider option", + ) + + +def _add_archive(commands: _SubParsersAction[ArgumentParser]) -> None: + parser = commands.add_parser( + "archive", + help="write the SBOM of a zip archive that is not a wheel, such as a zipapp", + description="Describe a zip archive that holds the project's .dist-info at its top level, such as a zipapp:" + " the archive's own files, the project's distribution, and every wheel it bundles, as a file or unpacked," + " described as the wheel's SBOM describes them. Options come from [tool.python-sbom].", + ) + parser.set_defaults(handler="python_sbom._cli._archive:run") + parser.add_argument("archive", type=Path, metavar="ARCHIVE", help="archive to describe") + _add_build_options(parser) + parser.add_argument( + "--build-script", + dest="build_scripts", + action="append", + metavar="PATH", + help="script, relative to the project, that built the archive; repeat it per script", + ) + parser.add_argument( + "--output", type=Path, metavar="FILE", help="SBOM file to write (default: .cdx.json next to it)" + ) + parser.add_argument( + "--embed", action="store_true", help="also add the SBOM to the archive, at its top level as .cdx.json" + ) + + +def _add_spdx(commands: _SubParsersAction[ArgumentParser]) -> None: + parser = commands.add_parser( + "spdx", + help="render a CycloneDX SBOM as SPDX 2.3 JSON", + ) + parser.set_defaults(handler="python_sbom._cli._spdx:run") + parser.add_argument("source", type=Path, metavar="SOURCE", help="CycloneDX JSON file to read") + parser.add_argument("target", type=Path, metavar="TARGET", help="SPDX JSON file to write") + + +def _add_validate(commands: _SubParsersAction[ArgumentParser]) -> None: + parser = commands.add_parser( + "validate", + help="validate the SBOMs in wheels and archives", + description="Check every CycloneDX SBOM a wheel or archive holds against the CycloneDX 1.6 schema; for one" + " python-sbom wrote, also its serial number, references, SPDX rendering and, with --allow-license, the licenses" + " of what it ships. Needs the validate extra: python-sbom[validate].", + ) + parser.set_defaults(handler="python_sbom._cli._validate:run") + parser.add_argument( + "paths", type=Path, nargs="+", metavar="PATH", help="wheel, archive, or directory holding wheels" + ) + parser.add_argument( + "--allow-license", + dest="allowed_licenses", + action="append", + metavar="SPDX-ID", + help="SPDX license id shipped components may use; repeat it per id, leave it out to skip the license policy", + ) + parser.add_argument( + "--max-components", + type=int, + default=MAX_COMPONENTS, + metavar="N", + help="reject an SBOM listing more than N components, nested ones included, without checking it further, which" + f" bounds the work an untrusted SBOM can ask for (default: {MAX_COMPONENTS})", + ) + + +class BuildOptions(Protocol): + root: Path | None + commit: str | None + build_env: list[str] | None + content_provider: str | None + + __all__ = [ + "BuildOptions", "build_parser", ] diff --git a/src/python_sbom/_cli/_spdx.py b/src/python_sbom/_cli/_spdx.py new file mode 100644 index 0000000..f4dcec8 --- /dev/null +++ b/src/python_sbom/_cli/_spdx.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Protocol + +from python_sbom._errors import SbomError +from python_sbom._formats._spdx import to_spdx +from python_sbom._validate._document import is_document + +if TYPE_CHECKING: + from pathlib import Path + + +class _Options(Protocol): + source: Path + target: Path + + +def run(options: _Options) -> bool: + if not is_document(loaded := json.loads(options.source.read_text(encoding="utf-8"))): + msg = f"{options.source} is not a CycloneDX document python-sbom can render" + raise SbomError(msg) + try: + rendered = to_spdx(loaded) + except (KeyError, TypeError) as error: + # the CLI renders without the schema check, which needs the validate extra + msg = f"{options.source} is not a CycloneDX document python-sbom can render: {error!r}" + raise SbomError(msg) from error + # bytes, so the line endings are the same on every operating system + options.target.write_bytes(rendered.encode()) + return True + + +__all__ = [ + "run", # simplint: ignore[SMP402] # the parser names it for pkgutil.resolve_name +] diff --git a/src/python_sbom/_cli/_validate.py b/src/python_sbom/_cli/_validate.py new file mode 100644 index 0000000..893a27a --- /dev/null +++ b/src/python_sbom/_cli/_validate.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, Protocol + +from python_sbom._validate._artifact import validate + +if TYPE_CHECKING: + from pathlib import Path + + +class _Options(Protocol): + paths: list[Path] + allowed_licenses: list[str] | None + max_components: int + + +def run(options: _Options) -> bool: + allowed = None if options.allowed_licenses is None else frozenset(options.allowed_licenses) + if not ( + artifacts := sorted( + artifact for path in options.paths for artifact in (path.glob("*.whl") if path.is_dir() else [path]) + ) + ): + sys.stderr.write(f"no wheel found in {', '.join(map(str, options.paths))}\n") + return False + valid = True + for artifact in artifacts: + for problem in (problems := validate(artifact, allowed, options.max_components)): + sys.stderr.write(f"SBOM invalid: {artifact.name}: {problem}\n") + if problems: + valid = False + else: + sys.stdout.write(f"SBOM in {artifact.name} is valid\n") + return valid + + +__all__ = [ + "run", # simplint: ignore[SMP402] # the parser names it for pkgutil.resolve_name +] diff --git a/tests/_cli/conftest.py b/tests/_cli/conftest.py new file mode 100644 index 0000000..6e33822 --- /dev/null +++ b/tests/_cli/conftest.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + from tests.conftest import WriteWheel + + +@pytest.fixture +def project(tmp_path: Path) -> Path: + (root := tmp_path / "project").mkdir() + (root / "pyproject.toml").write_text("[tool.python-sbom]\ndefault-branch = 'trunk'\n", encoding="utf-8") + return root + + +@pytest.fixture +def wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: + return write_wheel( + tmp_path / "dist" / "native-1.0-py3-none-any.whl", + {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"}, + ) diff --git a/tests/_cli/test_archive.py b/tests/_cli/test_archive.py new file mode 100644 index 0000000..74c1bac --- /dev/null +++ b/tests/_cli/test_archive.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import json +import stat +import struct +import zipfile +import zlib +from typing import TYPE_CHECKING + +import pytest + +from python_sbom._cli._main import main + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def zipapp(tmp_path: Path) -> Path: + with zipfile.ZipFile(target := tmp_path / "app.pyz", "w") as handle: + handle.writestr("__main__.py", "import app\n") + handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\nLicense-Expression: MIT\n") + handle.writestr("app/__init__.py", "") + return target + + +def test_archive_writes_sbom_next_to_archive(project: Path, zipapp: Path, capsys: pytest.CaptureFixture[str]) -> None: + main(["archive", str(zipapp), "--root", str(project), "--build-env", str(project)]) + assert ( + capsys.readouterr().out, + json.loads(zipapp.with_name("app.pyz.cdx.json").read_text())["metadata"]["component"]["name"], + ) == ( + f"wrote {zipapp.with_name('app.pyz.cdx.json')}\n", + "app.pyz", + ) + + +def test_archive_embeds_sbom(project: Path, zipapp: Path, tmp_path: Path) -> None: + (project / "build_app.py").write_text("", encoding="utf-8") + main([ + "archive", + str(zipapp), + "--root", + str(project), + "--build-env", + str(project), + "--build-script", + "build_app.py", + "--output", + str(tmp_path / "out.json"), + "--embed", + ]) + entry = zipfile.ZipFile(zipapp).getinfo("app.pyz.cdx.json") + assert (entry.create_system, entry.external_attr, entry.compress_type, zipfile.ZipFile(zipapp).read(entry)) == ( + 3, + (stat.S_IFREG | 0o644) << 16, + zipfile.ZIP_STORED, + (tmp_path / "out.json").read_bytes(), + ) + + +@pytest.mark.parametrize( + ("flags", "level"), + [ + pytest.param(0b000, zlib.Z_DEFAULT_COMPRESSION, id="normal"), + pytest.param(0b010, 9, id="maximum"), + pytest.param(0b100, 2, id="fast"), + pytest.param(0b110, 1, id="super-fast"), + ], +) +def test_archive_embeds_sbom_compressed_as_its_members(project: Path, tmp_path: Path, flags: int, level: int) -> None: + with zipfile.ZipFile(app := tmp_path / "app.pyz", "w", zipfile.ZIP_DEFLATED) as handle: + handle.writestr("__main__.py", "import app\n") + handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") + # zipfile clears the option bits it writes, so they go straight into the central directory, where readers take them + data = bytearray(app.read_bytes()) + for at in (index for index in range(len(data)) if data[index : index + 4] == b"PK\x01\x02"): + struct.pack_into(" None: + arguments = ["archive", str(zipapp), "--root", str(project), "--build-env", str(project), "--embed"] + main(arguments) + capsys.readouterr() + with pytest.raises(SystemExit): + main(arguments) + assert capsys.readouterr().err == "python-sbom: app.pyz already holds app.pyz.cdx.json\n" diff --git a/tests/_cli/test_commands.py b/tests/_cli/test_commands.py deleted file mode 100644 index a62f8d3..0000000 --- a/tests/_cli/test_commands.py +++ /dev/null @@ -1,278 +0,0 @@ -from __future__ import annotations - -import json -import runpy -import stat -import struct -import subprocess -import sys -import zipfile -import zlib -from pathlib import Path -from typing import TYPE_CHECKING, Final - -import pytest - -from python_sbom._cli._main import main -from python_sbom._formats._json import dumps -from python_sbom.version import __version__ - -if TYPE_CHECKING: - from conftest import Describe, Install, WriteWheel - - from python_sbom._model._types import Document - -_SBOM: Final[str] = "native-1.0.dist-info/sboms/native.cdx.json" - - -@pytest.fixture -def project(tmp_path: Path) -> Path: - (root := tmp_path / "project").mkdir() - (root / "pyproject.toml").write_text("[tool.python-sbom]\ndefault-branch = 'trunk'\n", encoding="utf-8") - return root - - -@pytest.fixture -def wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: - return write_wheel( - tmp_path / "dist" / "native-1.0-py3-none-any.whl", - {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"}, - ) - - -def test_embed( - project: Path, wheel: Path, tmp_path: Path, install: Install, capsys: pytest.CaptureFixture[str] -) -> None: - commit = "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" - install(site := tmp_path / "site", "Name: meson-python\nVersion: 0.18\n", {}) - main(["embed", str(wheel), "--root", str(project), "--commit", commit, "--build-env", str(site)]) - document = embedded(wheel) - assert capsys.readouterr().out == f"wrote {_SBOM} into {wheel.name}\n" - assert {"name": "native:vcs-commit", "value": commit} in document["metadata"]["component"]["properties"] - assert [tool["name"] for tool in document["metadata"]["tools"]["components"][1:]] == ["meson-python"] - - -def embedded(wheel: Path) -> Document: - return json.loads(zipfile.ZipFile(wheel).read(_SBOM)) - - -def test_embed_defaults_to_current_directory(project: Path, wheel: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(project) - monkeypatch.setenv("PYTHON_SBOM_COMMIT", "cafecafecafecafecafecafecafecafecafecafe") - main(["embed", str(wheel), "--build-env", str(project)]) - assert {"name": "native:vcs-commit", "value": "cafecafecafecafecafecafecafecafecafecafe"} in embedded(wheel)[ - "metadata" - ]["component"]["properties"] - - -@pytest.mark.parametrize( - ("option", "table"), - [ - pytest.param(["--content-provider", "MODULE:Native"], "", id="option"), - pytest.param([], "content-provider = 'MODULE:Native'\n", id="setting"), - ], -) -def test_embed_with_content_provider(tmp_path: Path, project: Path, wheel: Path, option: list[str], table: str) -> None: - # a module name unique to the test, so the import cache never hands over another test's module - provider = """\ -from python_sbom import ContentProvider, Contributions - - -class Native(ContentProvider): - def contributions(self, context): - return Contributions( - root_properties=[{"name": "extra:by", "value": "cli"}] - ) -""" - (project / f"{tmp_path.name}.py").write_text(provider, encoding="utf-8") - (project / "pyproject.toml").write_text(f"[tool.python-sbom]\n{table.replace('MODULE', tmp_path.name)}") - main([ - "embed", - str(wheel), - "--root", - str(project), - "--build-env", - str(project), - *(argument.replace("MODULE", tmp_path.name) for argument in option), - ]) - document = embedded(wheel) - assert {"name": "extra:by", "value": "cli"} in document["metadata"]["component"]["properties"] - assert document["metadata"]["tools"]["components"][0]["name"] == f"{tmp_path.name}.py" - - -def test_error_without_traceback(tmp_path: Path, wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: - with pytest.raises(SystemExit) as exit_info: - main(["embed", str(wheel), "--root", str(tmp_path)]) - assert exit_info.value.code == 1 - assert capsys.readouterr().err == f"python-sbom: {tmp_path} has no pyproject.toml to read [tool.python-sbom] from\n" - - -def test_spdx(tmp_path: Path, describe: Describe) -> None: - (source := tmp_path / "example.cdx.json").write_bytes(dumps(describe())) - main(["spdx", str(source), str(target := tmp_path / "example.spdx.json")]) - assert json.loads(target.read_text(encoding="utf-8"))["spdxVersion"] == "SPDX-2.3" - - -def test_validate_directory(project: Path, wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: - main(["embed", str(wheel), "--root", str(project), "--build-env", str(project)]) - capsys.readouterr() - main(["validate", str(wheel.parent), "--allow-license", "MIT"]) - assert capsys.readouterr().out == f"SBOM in {wheel.name} is valid\n" - - -def test_validate_reports_problems(wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: - with pytest.raises(SystemExit): - main(["validate", str(wheel)]) - assert capsys.readouterr().err == f"SBOM invalid: {wheel.name}: no SBOM found in {wheel.name}\n" - - -@pytest.fixture -def crowded_wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: - # 100 groups of 50 parts, 5,100 components in all, yet no array long enough to slow the schema check - groups = [ - { - "type": "library", - "name": f"g{group}", - "components": [{"type": "library", "name": f"p{part}"} for part in range(50)], - } - for group in range(100) - ] - sbom = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "components": groups}).encode() - return write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"", _SBOM: sbom}) - - -def test_validate_max_components_sets_the_cap(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: - with pytest.raises(SystemExit): - main(["validate", str(crowded_wheel), "--max-components", "5000"]) - assert capsys.readouterr().err == ( - f"SBOM invalid: {crowded_wheel.name}: {_SBOM}: lists more than 5000 components, nested ones included;" - " pass a higher --max-components to check it\n" - ) - - -def test_validate_default_cap_admits_thousands(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: - main(["validate", str(crowded_wheel)]) - assert capsys.readouterr().out == f"SBOM in {crowded_wheel.name} is valid\n" - - -def test_validate_without_wheels(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - with pytest.raises(SystemExit): - main(["validate", str(tmp_path)]) - assert capsys.readouterr().err == f"no wheel found in {tmp_path}\n" - - -def test_module_entry_point(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - monkeypatch.setattr(sys, "argv", ["python-sbom", "--help"]) - with pytest.raises(SystemExit): - runpy.run_module("python_sbom", run_name="__main__") - assert capsys.readouterr().out.startswith("usage: python-sbom ") - - -def test_console_script() -> None: - script = Path(sys.executable).parent / "python-sbom" - assert subprocess.check_output([script, "--help"], text=True).startswith("usage: python-sbom ") - - -@pytest.fixture -def zipapp(tmp_path: Path) -> Path: - with zipfile.ZipFile(target := tmp_path / "app.pyz", "w") as handle: - handle.writestr("__main__.py", "import app\n") - handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\nLicense-Expression: MIT\n") - handle.writestr("app/__init__.py", "") - return target - - -def test_archive_writes_sbom_next_to_archive(project: Path, zipapp: Path, capsys: pytest.CaptureFixture[str]) -> None: - main(["archive", str(zipapp), "--root", str(project), "--build-env", str(project)]) - assert ( - capsys.readouterr().out, - json.loads(zipapp.with_name("app.pyz.cdx.json").read_text())["metadata"]["component"]["name"], - ) == ( - f"wrote {zipapp.with_name('app.pyz.cdx.json')}\n", - "app.pyz", - ) - - -def test_archive_embeds_sbom(project: Path, zipapp: Path, tmp_path: Path) -> None: - (project / "build_app.py").write_text("", encoding="utf-8") - main([ - "archive", - str(zipapp), - "--root", - str(project), - "--build-env", - str(project), - "--build-script", - "build_app.py", - "--output", - str(tmp_path / "out.json"), - "--embed", - ]) - entry = zipfile.ZipFile(zipapp).getinfo("app.pyz.cdx.json") - assert (entry.create_system, entry.external_attr, entry.compress_type, zipfile.ZipFile(zipapp).read(entry)) == ( - 3, - (stat.S_IFREG | 0o644) << 16, - zipfile.ZIP_STORED, - (tmp_path / "out.json").read_bytes(), - ) - - -@pytest.mark.parametrize( - ("flags", "level"), - [ - pytest.param(0b000, zlib.Z_DEFAULT_COMPRESSION, id="normal"), - pytest.param(0b010, 9, id="maximum"), - pytest.param(0b100, 2, id="fast"), - pytest.param(0b110, 1, id="super-fast"), - ], -) -def test_archive_embeds_sbom_compressed_as_its_members(project: Path, tmp_path: Path, flags: int, level: int) -> None: - with zipfile.ZipFile(app := tmp_path / "app.pyz", "w", zipfile.ZIP_DEFLATED) as handle: - handle.writestr("__main__.py", "import app\n") - handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") - # zipfile clears the option bits it writes, so they go straight into the central directory, where readers take them - data = bytearray(app.read_bytes()) - for at in (index for index in range(len(data)) if data[index : index + 4] == b"PK\x01\x02"): - struct.pack_into(" None: - arguments = ["archive", str(zipapp), "--root", str(project), "--build-env", str(project), "--embed"] - main(arguments) - capsys.readouterr() - with pytest.raises(SystemExit): - main(arguments) - assert capsys.readouterr().err == "python-sbom: app.pyz already holds app.pyz.cdx.json\n" - - -def test_version(capsys: pytest.CaptureFixture[str]) -> None: - with pytest.raises(SystemExit): - main(["--version"]) - assert capsys.readouterr().out == f"python-sbom {__version__}\n" - - -def test_spdx_refuses_non_document(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - (source := tmp_path / "other.json").write_text("[]", encoding="utf-8") - with pytest.raises(SystemExit): - main(["spdx", str(source), str(tmp_path / "out.json")]) - assert capsys.readouterr().err == f"python-sbom: {source} is not a CycloneDX document python-sbom can render\n" - - -def test_spdx_refuses_malformed_document(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - document = {"metadata": {"component": {"bom-ref": "app", "name": "app"}}, "components": [{"name": "x"}]} - (source := tmp_path / "broken.json").write_text(json.dumps(document), encoding="utf-8") - with pytest.raises(SystemExit): - main(["spdx", str(source), str(tmp_path / "out.json")]) - assert capsys.readouterr().err.startswith( - f"python-sbom: {source} is not a CycloneDX document python-sbom can render: " - ) diff --git a/tests/_cli/test_embed.py b/tests/_cli/test_embed.py new file mode 100644 index 0000000..721ebcc --- /dev/null +++ b/tests/_cli/test_embed.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import zipfile +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom._cli._main import main + +if TYPE_CHECKING: + from pathlib import Path + + from python_sbom._model._types import Document + from tests.conftest import Install + +_SBOM: Final[str] = "native-1.0.dist-info/sboms/native.cdx.json" + + +def test_embed( + project: Path, wheel: Path, tmp_path: Path, install: Install, capsys: pytest.CaptureFixture[str] +) -> None: + commit = "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" + install(site := tmp_path / "site", "Name: meson-python\nVersion: 0.18\n", {}) + main(["embed", str(wheel), "--root", str(project), "--commit", commit, "--build-env", str(site)]) + document = embedded(wheel) + assert capsys.readouterr().out == f"wrote {_SBOM} into {wheel.name}\n" + assert {"name": "native:vcs-commit", "value": commit} in document["metadata"]["component"]["properties"] + assert [tool["name"] for tool in document["metadata"]["tools"]["components"][1:]] == ["meson-python"] + + +def embedded(wheel: Path) -> Document: + return json.loads(zipfile.ZipFile(wheel).read(_SBOM)) + + +def test_embed_defaults_to_current_directory(project: Path, wheel: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(project) + monkeypatch.setenv("PYTHON_SBOM_COMMIT", "cafecafecafecafecafecafecafecafecafecafe") + main(["embed", str(wheel), "--build-env", str(project)]) + assert {"name": "native:vcs-commit", "value": "cafecafecafecafecafecafecafecafecafecafe"} in embedded(wheel)[ + "metadata" + ]["component"]["properties"] + + +@pytest.mark.parametrize( + ("option", "table"), + [ + pytest.param(["--content-provider", "MODULE:Native"], "", id="option"), + pytest.param([], "content-provider = 'MODULE:Native'\n", id="setting"), + ], +) +def test_embed_with_content_provider(tmp_path: Path, project: Path, wheel: Path, option: list[str], table: str) -> None: + # a module name unique to the test, so the import cache never hands over another test's module + provider = """\ +from python_sbom import ContentProvider, Contributions + + +class Native(ContentProvider): + def contributions(self, context): + return Contributions( + root_properties=[{"name": "extra:by", "value": "cli"}] + ) +""" + (project / f"{tmp_path.name}.py").write_text(provider, encoding="utf-8") + (project / "pyproject.toml").write_text(f"[tool.python-sbom]\n{table.replace('MODULE', tmp_path.name)}") + main([ + "embed", + str(wheel), + "--root", + str(project), + "--build-env", + str(project), + *(argument.replace("MODULE", tmp_path.name) for argument in option), + ]) + document = embedded(wheel) + assert {"name": "extra:by", "value": "cli"} in document["metadata"]["component"]["properties"] + assert document["metadata"]["tools"]["components"][0]["name"] == f"{tmp_path.name}.py" diff --git a/tests/_cli/test_main.py b/tests/_cli/test_main.py new file mode 100644 index 0000000..a2677e5 --- /dev/null +++ b/tests/_cli/test_main.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + +from python_sbom._cli._main import main +from python_sbom.version import __version__ + + +def test_error_without_traceback(tmp_path: Path, wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as exit_info: + main(["embed", str(wheel), "--root", str(tmp_path)]) + assert exit_info.value.code == 1 + assert capsys.readouterr().err == f"python-sbom: {tmp_path} has no pyproject.toml to read [tool.python-sbom] from\n" + + +def test_module_entry_point(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sys, "argv", ["python-sbom", "--help"]) + with pytest.raises(SystemExit): + runpy.run_module("python_sbom", run_name="__main__") + assert capsys.readouterr().out.startswith("usage: python-sbom ") + + +def test_console_script() -> None: + script = Path(sys.executable).parent / "python-sbom" + assert subprocess.check_output([script, "--help"], text=True).startswith("usage: python-sbom ") + + +def test_version(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["--version"]) + assert capsys.readouterr().out == f"python-sbom {__version__}\n" + + +@pytest.mark.parametrize( + "arguments", [pytest.param("--version", id="version"), pytest.param("embed --help", id="help")] +) +def test_parsing_loads_no_command(arguments: str) -> None: + code = ( + "import sys\nfrom python_sbom._cli._main import main\ntry:\n" + f" main({arguments.split()!r})\nexcept SystemExit:\n" + " print(sorted(name for name in sys.modules if name.startswith('python_sbom._cli.')))\n" + ) + loaded = subprocess.check_output([sys.executable, "-c", code], text=True).splitlines()[-1] + assert loaded == "['python_sbom._cli._main', 'python_sbom._cli._parser']" diff --git a/tests/_cli/test_spdx.py b/tests/_cli/test_spdx.py new file mode 100644 index 0000000..7b00097 --- /dev/null +++ b/tests/_cli/test_spdx.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +from python_sbom._cli._main import main +from python_sbom._formats._json import dumps + +if TYPE_CHECKING: + from pathlib import Path + + from tests.conftest import Describe + + +def test_spdx(tmp_path: Path, describe: Describe) -> None: + (source := tmp_path / "example.cdx.json").write_bytes(dumps(describe())) + main(["spdx", str(source), str(target := tmp_path / "example.spdx.json")]) + assert json.loads(target.read_text(encoding="utf-8"))["spdxVersion"] == "SPDX-2.3" + + +def test_spdx_refuses_non_document(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + (source := tmp_path / "other.json").write_text("[]", encoding="utf-8") + with pytest.raises(SystemExit): + main(["spdx", str(source), str(tmp_path / "out.json")]) + assert capsys.readouterr().err == f"python-sbom: {source} is not a CycloneDX document python-sbom can render\n" + + +def test_spdx_refuses_malformed_document(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + document = {"metadata": {"component": {"bom-ref": "app", "name": "app"}}, "components": [{"name": "x"}]} + (source := tmp_path / "broken.json").write_text(json.dumps(document), encoding="utf-8") + with pytest.raises(SystemExit): + main(["spdx", str(source), str(tmp_path / "out.json")]) + assert capsys.readouterr().err.startswith( + f"python-sbom: {source} is not a CycloneDX document python-sbom can render: " + ) diff --git a/tests/_cli/test_validate.py b/tests/_cli/test_validate.py new file mode 100644 index 0000000..bea3bd7 --- /dev/null +++ b/tests/_cli/test_validate.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom._cli._main import main + +if TYPE_CHECKING: + from pathlib import Path + + from tests.conftest import WriteWheel + +_SBOM: Final[str] = "native-1.0.dist-info/sboms/native.cdx.json" + + +def test_validate_directory(project: Path, wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + main(["embed", str(wheel), "--root", str(project), "--build-env", str(project)]) + capsys.readouterr() + main(["validate", str(wheel.parent), "--allow-license", "MIT"]) + assert capsys.readouterr().out == f"SBOM in {wheel.name} is valid\n" + + +def test_validate_reports_problems(wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["validate", str(wheel)]) + assert capsys.readouterr().err == f"SBOM invalid: {wheel.name}: no SBOM found in {wheel.name}\n" + + +@pytest.fixture +def crowded_wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: + # 100 groups of 50 parts, 5,100 components in all, yet no array long enough to slow the schema check + groups = [ + { + "type": "library", + "name": f"g{group}", + "components": [{"type": "library", "name": f"p{part}"} for part in range(50)], + } + for group in range(100) + ] + sbom = json.dumps({"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "components": groups}).encode() + return write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"", _SBOM: sbom}) + + +def test_validate_max_components_sets_the_cap(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["validate", str(crowded_wheel), "--max-components", "5000"]) + assert capsys.readouterr().err == ( + f"SBOM invalid: {crowded_wheel.name}: {_SBOM}: lists more than 5000 components, nested ones included;" + " pass a higher --max-components to check it\n" + ) + + +def test_validate_default_cap_admits_thousands(crowded_wheel: Path, capsys: pytest.CaptureFixture[str]) -> None: + main(["validate", str(crowded_wheel)]) + assert capsys.readouterr().out == f"SBOM in {crowded_wheel.name} is valid\n" + + +def test_validate_without_wheels(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["validate", str(tmp_path)]) + assert capsys.readouterr().err == f"no wheel found in {tmp_path}\n" From 5ddcba290085975b96b28e595504805fd89a7283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:10:16 -0800 Subject: [PATCH 090/115] =?UTF-8?q?=F0=9F=93=9D=20docs(cli):=20describe=20?= =?UTF-8?q?the=20spdx=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/python_sbom/_cli/_parser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/python_sbom/_cli/_parser.py b/src/python_sbom/_cli/_parser.py index b79ec4d..9910a83 100644 --- a/src/python_sbom/_cli/_parser.py +++ b/src/python_sbom/_cli/_parser.py @@ -83,6 +83,9 @@ def _add_spdx(commands: _SubParsersAction[ArgumentParser]) -> None: parser = commands.add_parser( "spdx", help="render a CycloneDX SBOM as SPDX 2.3 JSON", + description="Render a CycloneDX JSON SBOM, such as one python-sbom wrote, as an SPDX 2.3 JSON document. It" + " checks only the fields the rendering reads, so it needs no extra; python-sbom validate checks the whole" + " document.", ) parser.set_defaults(handler="python_sbom._cli._spdx:run") parser.add_argument("source", type=Path, metavar="SOURCE", help="CycloneDX JSON file to read") From 0b53ee0d500663d126dd8e0457cdb96c06c88501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:10:47 -0800 Subject: [PATCH 091/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20refuse=20a=20--ma?= =?UTF-8?q?x-components=20below=20one?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cap of zero rejects every SBOM with components and a negative one rejects all of them, and the cap also sets the size limit. argparse now reports anything but a positive integer as a usage error. --- src/python_sbom/_cli/_parser.py | 12 ++++++++++-- tests/_cli/test_parser.py | 27 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 tests/_cli/test_parser.py diff --git a/src/python_sbom/_cli/_parser.py b/src/python_sbom/_cli/_parser.py index 9910a83..07c5b2b 100644 --- a/src/python_sbom/_cli/_parser.py +++ b/src/python_sbom/_cli/_parser.py @@ -1,6 +1,6 @@ from __future__ import annotations -from argparse import ArgumentParser, _SubParsersAction +from argparse import ArgumentParser, ArgumentTypeError, _SubParsersAction from pathlib import Path from typing import Protocol @@ -113,7 +113,7 @@ def _add_validate(commands: _SubParsersAction[ArgumentParser]) -> None: ) parser.add_argument( "--max-components", - type=int, + type=_positive_int, default=MAX_COMPONENTS, metavar="N", help="reject an SBOM listing more than N components, nested ones included, without checking it further, which" @@ -121,6 +121,14 @@ def _add_validate(commands: _SubParsersAction[ArgumentParser]) -> None: ) +def _positive_int(text: str) -> int: + # argparse reports the error through parser.error, with the usage, like any other bad argument + if text.isdecimal() and (number := int(text)) > 0: + return number + msg = f"must be a positive integer, got {text!r}" + raise ArgumentTypeError(msg) + + class BuildOptions(Protocol): root: Path | None commit: str | None diff --git a/tests/_cli/test_parser.py b/tests/_cli/test_parser.py new file mode 100644 index 0000000..fee73ac --- /dev/null +++ b/tests/_cli/test_parser.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from python_sbom._cli._parser import build_parser + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.mark.parametrize( + "value", + [pytest.param("0", id="zero"), pytest.param("-1", id="negative"), pytest.param("many", id="not-a-number")], +) +def test_max_components_must_be_positive(value: str, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as exit_info: + build_parser().parse_args(["validate", str(tmp_path), "--max-components", value]) + assert (exit_info.value.code, capsys.readouterr().err.splitlines()[-1]) == ( + 2, + f"python-sbom validate: error: argument --max-components: must be a positive integer, got {value!r}", + ) + + +def test_max_components_takes_positive(tmp_path: Path) -> None: + assert build_parser().parse_args(["validate", str(tmp_path), "--max-components", "1"]).max_components == 1 From 1aea9a3ecfb92868c0b9405bb6bf41b4a8558449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:10:59 -0800 Subject: [PATCH 092/115] =?UTF-8?q?=F0=9F=93=9D=20docs(cli):=20say=20the?= =?UTF-8?q?=20component=20cap=20sets=20the=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/python_sbom/_cli/_parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python_sbom/_cli/_parser.py b/src/python_sbom/_cli/_parser.py index 07c5b2b..756f691 100644 --- a/src/python_sbom/_cli/_parser.py +++ b/src/python_sbom/_cli/_parser.py @@ -116,8 +116,9 @@ def _add_validate(commands: _SubParsersAction[ArgumentParser]) -> None: type=_positive_int, default=MAX_COMPONENTS, metavar="N", - help="reject an SBOM listing more than N components, nested ones included, without checking it further, which" - f" bounds the work an untrusted SBOM can ask for (default: {MAX_COMPONENTS})", + help="reject an SBOM listing more than N components, nested ones included, or larger than 1 MiB plus 1 KiB per" + " component, without checking it further, which bounds the work an untrusted SBOM can ask for (default:" + f" {MAX_COMPONENTS})", ) From 13dcb697f7c0f96b6da4cfb5058b3fb6babc670c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:12:45 -0800 Subject: [PATCH 093/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20report=20unreadab?= =?UTF-8?q?le=20spdx=20input=20without=20a=20trace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit python-sbom spdx crashed with a traceback on a missing file, invalid JSON, input that is not UTF-8, deep nesting, or a timestamp that is no ISO date. It now reports each as an error, and refuses input over the size validate accepts at its default component cap, so the JSON parser never holds more than validate lets it. --- src/python_sbom/_cli/_spdx.py | 42 ++++++++++++++++++++++++---- tests/_cli/test_spdx.py | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/python_sbom/_cli/_spdx.py b/src/python_sbom/_cli/_spdx.py index f4dcec8..8ef93cb 100644 --- a/src/python_sbom/_cli/_spdx.py +++ b/src/python_sbom/_cli/_spdx.py @@ -5,11 +5,13 @@ from python_sbom._errors import SbomError from python_sbom._formats._spdx import to_spdx -from python_sbom._validate._document import is_document +from python_sbom._validate._document import MAX_COMPONENTS, byte_limit, is_document if TYPE_CHECKING: from pathlib import Path + from python_sbom._model._types import Document + class _Options(Protocol): source: Path @@ -17,12 +19,12 @@ class _Options(Protocol): def run(options: _Options) -> bool: - if not is_document(loaded := json.loads(options.source.read_text(encoding="utf-8"))): - msg = f"{options.source} is not a CycloneDX document python-sbom can render" - raise SbomError(msg) + document = _read(options.source) try: - rendered = to_spdx(loaded) - except (KeyError, TypeError) as error: + rendered = to_spdx(document) + # ValueError: a timestamp fromisoformat rejects; RecursionError: nesting json.dumps, a few frames deeper than + # json.loads, gives up on + except (KeyError, TypeError, ValueError, RecursionError) as error: # the CLI renders without the schema check, which needs the validate extra msg = f"{options.source} is not a CycloneDX document python-sbom can render: {error!r}" raise SbomError(msg) from error @@ -31,6 +33,34 @@ def run(options: _Options) -> bool: return True +def _read(source: Path) -> Document: + # the size validate accepts at its default component cap, so the parser never holds more than validate lets it + limit = byte_limit(MAX_COMPONENTS) + try: + with source.open("rb") as handle: + raw = handle.read(limit + 1) + except OSError as error: + raise SbomError(str(error)) from error + if len(raw) > limit: + msg = f"{source} is over {limit} bytes, the size limit of an SBOM python-sbom reads" + raise SbomError(msg) + try: + loaded = json.loads(raw.decode("utf-8")) + except UnicodeDecodeError as error: + msg = f"{source} is not UTF-8: {error.reason} at byte {error.start}" + raise SbomError(msg) from error + except json.JSONDecodeError as error: + msg = f"{source} is not JSON: {error}" + raise SbomError(msg) from error + except RecursionError as error: + msg = f"{source} nests too deep for the JSON parser to read" + raise SbomError(msg) from error + if not is_document(loaded): + msg = f"{source} is not a CycloneDX document python-sbom can render" + raise SbomError(msg) + return loaded + + __all__ = [ "run", # simplint: ignore[SMP402] # the parser names it for pkgutil.resolve_name ] diff --git a/tests/_cli/test_spdx.py b/tests/_cli/test_spdx.py index 7b00097..5d50746 100644 --- a/tests/_cli/test_spdx.py +++ b/tests/_cli/test_spdx.py @@ -7,6 +7,7 @@ from python_sbom._cli._main import main from python_sbom._formats._json import dumps +from python_sbom._validate._document import MAX_COMPONENTS, byte_limit if TYPE_CHECKING: from pathlib import Path @@ -35,3 +36,54 @@ def test_spdx_refuses_malformed_document(tmp_path: Path, capsys: pytest.CaptureF assert capsys.readouterr().err.startswith( f"python-sbom: {source} is not a CycloneDX document python-sbom can render: " ) + + +@pytest.mark.parametrize( + ("content", "problem"), + [ + pytest.param( + b"{", "is not JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)", id="json" + ), + pytest.param(b'{"name": "\xff"}', "is not UTF-8: invalid start byte at byte 10", id="utf-8"), + pytest.param(b"[" * 100_000, "nests too deep for the JSON parser to read", id="nesting"), + ], +) +def test_spdx_refuses_unreadable_source( + content: bytes, problem: str, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (source := tmp_path / "source.json").write_bytes(content) + with pytest.raises(SystemExit): + main(["spdx", str(source), str(tmp_path / "out.json")]) + assert capsys.readouterr().err == f"python-sbom: {source} {problem}\n" + + +def test_spdx_refuses_missing_source(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["spdx", str(source := tmp_path / "missing.json"), str(tmp_path / "out.json")]) + assert capsys.readouterr().err == f"python-sbom: [Errno 2] No such file or directory: {str(source)!r}\n" + + +def test_spdx_refuses_oversized_source(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + limit = byte_limit(MAX_COMPONENTS) + # sparse, so the test writes nothing; the size alone is refused, before any parsing + with (source := tmp_path / "large.json").open("wb") as handle: + handle.truncate(limit + 1) + with pytest.raises(SystemExit): + main(["spdx", str(source), str(tmp_path / "out.json")]) + assert capsys.readouterr().err == ( + f"python-sbom: {source} is over {limit} bytes, the size limit of an SBOM python-sbom reads\n" + ) + + +def test_spdx_refuses_unparsable_timestamp( + tmp_path: Path, describe: Describe, capsys: pytest.CaptureFixture[str] +) -> None: + document = describe() + document["metadata"]["timestamp"] = "yesterday" + (source := tmp_path / "example.cdx.json").write_bytes(dumps(document)) + with pytest.raises(SystemExit): + main(["spdx", str(source), str(tmp_path / "out.json")]) + assert capsys.readouterr().err == ( + f"python-sbom: {source} is not a CycloneDX document python-sbom can render:" + " ValueError(\"Invalid isoformat string: 'yesterday'\")\n" + ) From 0d4fd8e427d530af86aeef58cc613f5bff5d9559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:14:42 -0800 Subject: [PATCH 094/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20read=20the=20sour?= =?UTF-8?q?ce=20revision=20once=20per=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hook cached the revision for the whole process, so a build after a commit in the same process recorded the earlier commit and dirty state. Each build now reads it in initialize, before it writes the artifact, and finalize reuses that value. --- src/python_sbom/hatch/_hook.py | 23 +++++++++++------------ tests/hatch/test_hook.py | 8 ++++++++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index 5996c1c..b9afe03 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -1,7 +1,7 @@ from __future__ import annotations import tempfile -from functools import cache, cached_property +from functools import cached_property from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Final @@ -15,7 +15,7 @@ if TYPE_CHECKING: from python_sbom._model._content import ContentProvider - from python_sbom._model._inputs import Revision, Settings + from python_sbom._model._inputs import Settings # options of every hatchling hook table, which hatchling itself reads _HATCHLING_OPTIONS: Final[frozenset[str]] = frozenset({ @@ -44,18 +44,23 @@ class SbomBuildHook(BuildHookInterface): def initialize( self, - version: str, # ruff: ignore[unused-method-argument] # hatchling's signature + version: str, build_data: dict[str, Any], # simplint: ignore[SMP602] # hatchling's signature ) -> None: """ - Record the source revision in an sdist. + Read the source revision, and record it in an sdist. :param version: hatchling's build variant :param build_data: hatchling's build data, which receives the revision file """ + if self.target_name not in {"sdist", "wheel"} or version == "editable": + return + # once per build, before it writes the artifact into a checkout that may not ignore it; a value kept for the + # process would name an older commit in the next build after a commit + self._revision = revision = default_revision(Path(self.root)) # a dirty checkout has no commit that reproduces the sdist, so a wheel built from it names none - if self.target_name != "sdist" or (revision := _revision(Path(self.root))) is None or revision.dirty: + if self.target_name != "sdist" or revision is None or revision.dirty: return # outside the project, so the build leaves nothing behind in it; bytes, so the sdist holds the same LF line on # every OS @@ -95,7 +100,7 @@ def finalize( # the zip entries hatchling wrote facts = collect_facts( root, - _revision, + lambda _root: self._revision, default_epoch=get_reproducible_timestamp() if self.build_config.reproducible else None, ) wheel = Path(artifact_path) @@ -128,12 +133,6 @@ def _settings(self) -> Settings: return settings_from_table(self.metadata.config.get("tool", {}).get("python-sbom", {}), "[tool.python-sbom]") -@cache -def _revision(root: Path) -> Revision | None: - # read once per process: hatch build makes the sdist and then the wheel in one, of the same checkout - return default_revision(root) - - __all__ = [ "SbomBuildHook", ] diff --git a/tests/hatch/test_hook.py b/tests/hatch/test_hook.py index 4a32557..2fcd623 100644 --- a/tests/hatch/test_hook.py +++ b/tests/hatch/test_hook.py @@ -182,3 +182,11 @@ def sdist_files(sdist: Path) -> dict[str, bytes]: def test_sdist_of_dirty_checkout_records_no_revision(checkout: Path, build_wheel: BuildWheel) -> None: (checkout / "LICENSE").write_text("changed\n", encoding="utf-8") assert ".python-sbom-revision" not in sdist_files(build_wheel(checkout, target="sdist")) + + +def test_each_build_reads_the_revision(checkout: Path, git: Git, build_wheel: BuildWheel) -> None: + build_wheel(checkout, target="sdist") + (checkout / "LICENSE").write_text("changed\n", encoding="utf-8") + git("commit", "--quiet", "--all", "--message", "second") + files = sdist_files(build_wheel(checkout, target="sdist")) + assert files.get(".python-sbom-revision") == f"node: {git('rev-parse', 'HEAD')}\n".encode() From 18810cf333a395531bc6afd4bd3322817362c743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:17:24 -0800 Subject: [PATCH 095/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20remove=20the=20sd?= =?UTF-8?q?ist=20revision=20file=20after=20the=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit initialize wrote the revision file into a directory from tempfile.mkdtemp that nothing removed, so every sdist build left one behind. finalize now removes it once the sdist holds the file. hatchling skips finalize when the build fails, so the directory then goes with the hook, at the latest at exit. --- src/python_sbom/hatch/_hook.py | 20 ++++++++++++----- tests/_cli/conftest.py | 9 ++++++++ tests/_cli/test_archive.py | 9 ++++++++ tests/_cli/test_embed.py | 6 +++++ tests/hatch/test_hook.py | 41 ++++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 5 deletions(-) diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index b9afe03..658b2af 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -1,6 +1,8 @@ from __future__ import annotations +import shutil import tempfile +import weakref from functools import cached_property from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Final @@ -14,6 +16,8 @@ from python_sbom._errors import SettingsError if TYPE_CHECKING: + from collections.abc import Callable + from python_sbom._model._content import ContentProvider from python_sbom._model._inputs import Settings @@ -41,6 +45,7 @@ class SbomBuildHook(BuildHookInterface): """ PLUGIN_NAME = "python-sbom" + _remove_revision_file: Callable[[], None] | None = None def initialize( self, @@ -62,10 +67,13 @@ def initialize( # a dirty checkout has no commit that reproduces the sdist, so a wheel built from it names none if self.target_name != "sdist" or revision is None or revision.dirty: return - # outside the project, so the build leaves nothing behind in it; bytes, so the sdist holds the same LF line on - # every OS - written = Path(tempfile.mkdtemp()) / REVISION_FILE - written.write_bytes(f"node: {revision.commit}\n".encode()) + # outside the project, so the build leaves nothing behind in it + directory = tempfile.mkdtemp() + # hatchling reads the file only while it writes the sdist, and skips finalize when the build fails, so the + # directory then goes with the hook, at the latest when the interpreter exits + self._remove_revision_file = weakref.finalize(self, shutil.rmtree, directory) + # bytes, so the sdist holds the same LF line on every OS + (written := Path(directory) / REVISION_FILE).write_bytes(f"node: {revision.commit}\n".encode()) build_data["force_include"][str(written)] = REVISION_FILE def finalize( @@ -75,13 +83,15 @@ def finalize( artifact_path: str, ) -> None: """ - Write the SBOM into the wheel hatchling built. + Write the SBOM into the wheel hatchling built, and remove the revision file an sdist build wrote. :param version: hatchling's build variant; an editable wheel gets no SBOM, since it carries no code :param build_data: hatchling's build data :param artifact_path: the built wheel """ + if self._remove_revision_file is not None: + self._remove_revision_file() if self.target_name != "wheel" or version == "editable": return # what a wheel needs loads only when one is built, so an sdist build stays light diff --git a/tests/_cli/conftest.py b/tests/_cli/conftest.py index 6e33822..30836ed 100644 --- a/tests/_cli/conftest.py +++ b/tests/_cli/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import tempfile from typing import TYPE_CHECKING import pytest @@ -23,3 +24,11 @@ def wheel(tmp_path: Path, write_wheel: WriteWheel) -> Path: tmp_path / "dist" / "native-1.0-py3-none-any.whl", {"native/__init__.py": b"", "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\n"}, ) + + +@pytest.fixture +def temp_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + (directory := tmp_path / "temp").mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(directory)) + monkeypatch.setenv("TMPDIR", str(directory)) + return directory diff --git a/tests/_cli/test_archive.py b/tests/_cli/test_archive.py index 74c1bac..54ad459 100644 --- a/tests/_cli/test_archive.py +++ b/tests/_cli/test_archive.py @@ -95,3 +95,12 @@ def test_archive_refuses_second_embed(project: Path, zipapp: Path, capsys: pytes with pytest.raises(SystemExit): main(arguments) assert capsys.readouterr().err == "python-sbom: app.pyz already holds app.pyz.cdx.json\n" + + +def test_archive_embed_leaves_no_temporary_files(project: Path, zipapp: Path, temp_dir: Path) -> None: + main(["archive", str(zipapp), "--root", str(project), "--build-env", str(project), "--embed"]) + # the rewritten archive is staged next to the original + assert (list(temp_dir.iterdir()), sorted(path.name for path in zipapp.parent.iterdir())) == ( + [], + ["app.pyz", "app.pyz.cdx.json", "project", "temp"], + ) diff --git a/tests/_cli/test_embed.py b/tests/_cli/test_embed.py index 721ebcc..3c36cea 100644 --- a/tests/_cli/test_embed.py +++ b/tests/_cli/test_embed.py @@ -75,3 +75,9 @@ def contributions(self, context): document = embedded(wheel) assert {"name": "extra:by", "value": "cli"} in document["metadata"]["component"]["properties"] assert document["metadata"]["tools"]["components"][0]["name"] == f"{tmp_path.name}.py" + + +def test_embed_leaves_no_temporary_files(project: Path, wheel: Path, temp_dir: Path) -> None: + main(["embed", str(wheel), "--root", str(project), "--build-env", str(project)]) + # the rewritten wheel is staged next to the original + assert (list(temp_dir.iterdir()), list(wheel.parent.iterdir())) == ([], [wheel]) diff --git a/tests/hatch/test_hook.py b/tests/hatch/test_hook.py index 2fcd623..c5bd2cc 100644 --- a/tests/hatch/test_hook.py +++ b/tests/hatch/test_hook.py @@ -1,8 +1,10 @@ from __future__ import annotations +import gc import json import shutil import tarfile +import tempfile import time import zipfile from typing import TYPE_CHECKING, Final @@ -31,6 +33,15 @@ def initialize(self, version, build_data): build_data["dependencies"].append("late-dependency>=1") build_data["tag"] = "py3-none-manylinux_2_17_x86_64" """ +# a hook after the SBOM hook that fails the build, before hatchling writes anything +_BREAK: Final[str] = """\ +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + + +class Break(BuildHookInterface): + def initialize(self, version, build_data): + raise RuntimeError("broken") +""" def sbom_of(wheel: Path, name: str = _SBOM) -> Document: @@ -190,3 +201,33 @@ def test_each_build_reads_the_revision(checkout: Path, git: Git, build_wheel: Bu git("commit", "--quiet", "--all", "--message", "second") files = sdist_files(build_wheel(checkout, target="sdist")) assert files.get(".python-sbom-revision") == f"node: {git('rev-parse', 'HEAD')}\n".encode() + + +@pytest.fixture +def temp_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + (directory := tmp_path / "temp").mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(directory)) + monkeypatch.setenv("TMPDIR", str(directory)) + return directory + + +@pytest.mark.parametrize("target", [pytest.param("sdist", id="sdist"), pytest.param("wheel", id="wheel")]) +def test_build_leaves_no_temporary_files(checkout: Path, temp_dir: Path, build_wheel: BuildWheel, target: str) -> None: + build_wheel(checkout, target=target) + assert list(temp_dir.iterdir()) == [] + + +def test_failed_build_leaves_no_temporary_files( + checkout: Path, temp_dir: Path, git: Git, build_wheel: BuildWheel +) -> None: + (checkout / "hatch_build.py").write_text(_BREAK, encoding="utf-8") + # hatchling compiles the hook script, which must not make the checkout dirty, so the sdist records the revision + (checkout / ".gitignore").write_text("__pycache__/\n", encoding="utf-8") + configure(checkout, "[tool.hatch.build.hooks.custom]\n") + git("add", ".") + git("commit", "--quiet", "--message", "break the build") + with pytest.raises(RuntimeError, match="broken"): + build_wheel(checkout, target="sdist") + # the failed build's frames hold the hook in a reference cycle + gc.collect() + assert list(temp_dir.iterdir()) == [] From b9bb91a2822f0fcda7c119caf74f54da7f81f079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:20:18 -0800 Subject: [PATCH 096/115] =?UTF-8?q?=F0=9F=8E=A8=20style:=20fit=20the=20hoo?= =?UTF-8?q?k's=20suppressions=20in=20120=20columns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resolver of the content provider costs a third of a millisecond on top of hatchling, so it loads with the hook instead of twice behind a suppression. finalize's build_data now carries one suppression per line, each under 120 columns. --- src/python_sbom/hatch/_hook.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index 658b2af..ba36b4b 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -11,6 +11,7 @@ from hatchling.builders.utils import get_reproducible_timestamp from hatchling.utils.constants import DEFAULT_BUILD_SCRIPT +from python_sbom._collect._content import resolve_provider from python_sbom._collect._revision import REVISION_FILE, default_revision from python_sbom._collect._settings import settings_from_table from python_sbom._errors import SettingsError @@ -79,7 +80,9 @@ def initialize( def finalize( self, version: str, - build_data: dict[str, Any], # ruff: ignore[unused-method-argument] # simplint: ignore[SMP602] # hatchling's signature + build_data: ( # ruff: ignore[unused-method-argument] # hatchling's signature + dict[str, Any] # simplint: ignore[SMP602] # see above + ), artifact_path: str, ) -> None: """ @@ -95,7 +98,6 @@ def finalize( if self.target_name != "wheel" or version == "editable": return # what a wheel needs loads only when one is built, so an sdist build stays light - from python_sbom._collect._content import resolve_provider # ruff: ignore[import-outside-top-level] # see above from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # see above from python_sbom._collect._source import wheel_request # ruff: ignore[import-outside-top-level] # see above from python_sbom._generate._embed import embed_into # ruff: ignore[import-outside-top-level] # see above @@ -126,8 +128,6 @@ def content_provider(self) -> ContentProvider: :returns: the ``content-provider`` option's provider, or one that adds nothing """ - from python_sbom._collect._content import resolve_provider # ruff: ignore[import-outside-top-level] # see finalize - return resolve_provider(None, self._settings, Path(self.root))[0] @cached_property From 9395d00eb3d0ba1256cc6af46296e36921797be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:21:01 -0800 Subject: [PATCH 097/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20record=20the=20re?= =?UTF-8?q?vision=20in=20python-sbom's=20own=20sdist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The self-build hook delegated only finalize, so python-sbom's sdist carried no .python-sbom-revision and a wheel built from it named no commit. One helper now builds the SbomBuildHook for the whole build, and both initialize and finalize go to it. --- hatch_build.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index 354f4a2..1c36882 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -3,21 +3,29 @@ from __future__ import annotations import sys +from functools import cached_property from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from hatchling.builders.hooks.plugin.interface import BuildHookInterface +if TYPE_CHECKING: + from python_sbom.hatch import SbomBuildHook + class SelfSbomHook(BuildHookInterface): # simplint: ignore[SMP409] # hatchling loads it from this script - def finalize( + def initialize( self, version: str, build_data: dict[str, Any], # simplint: ignore[SMP602] # hatchling's signature - artifact_path: str, ) -> None: + self._hook.initialize(version, build_data) + + @cached_property + def _hook(self) -> SbomBuildHook: # python-sbom cannot be a build requirement of itself, so its hook comes from this source tree; by now the vcs - # hook has written the version module the package imports + # hook has written the version module the package imports. One hook serves the whole build, so finalize reuses + # the revision initialize read and removes the revision file it wrote sys.path.insert(0, str(Path(self.root) / "src")) from python_sbom.hatch import SbomBuildHook # ruff: ignore[import-outside-top-level] # see above @@ -26,4 +34,12 @@ def finalize( ) # as hatchling names a custom hook, so the SBOM lists this script among the build tools hook.PLUGIN_NAME = self.PLUGIN_NAME - hook.finalize(version, build_data, artifact_path) + return hook + + def finalize( + self, + version: str, + build_data: dict[str, Any], # simplint: ignore[SMP602] # hatchling's signature + artifact_path: str, + ) -> None: + self._hook.finalize(version, build_data, artifact_path) From a935e88ad7d5aae8669f50284dd6c5e6d10cbfde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:23:18 -0800 Subject: [PATCH 098/115] =?UTF-8?q?=E2=9C=85=20test:=20check=20one=20behav?= =?UTF-8?q?iour=20per=20CLI=20and=20hook=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The embed test and the retagged wheel test each asserted three behaviours; fixtures now run the command or build once per test, and each test checks one. The archive compression test takes its flagged archive from a fixture instead of patching bytes in its body. --- tests/_cli/test_archive.py | 25 +++++++++++++++---------- tests/_cli/test_embed.py | 30 ++++++++++++++++++++++-------- tests/hatch/test_hook.py | 29 +++++++++++++++++++++-------- 3 files changed, 58 insertions(+), 26 deletions(-) diff --git a/tests/_cli/test_archive.py b/tests/_cli/test_archive.py index 54ad459..114ab9f 100644 --- a/tests/_cli/test_archive.py +++ b/tests/_cli/test_archive.py @@ -68,7 +68,20 @@ def test_archive_embeds_sbom(project: Path, zipapp: Path, tmp_path: Path) -> Non pytest.param(0b110, 1, id="super-fast"), ], ) -def test_archive_embeds_sbom_compressed_as_its_members(project: Path, tmp_path: Path, flags: int, level: int) -> None: +def test_archive_embeds_sbom_compressed_as_its_members(project: Path, flagged_app: Path, level: int) -> None: + main(["archive", str(flagged_app), "--root", str(project), "--build-env", str(project), "--embed"]) + with zipfile.ZipFile(flagged_app) as handle: + entry = handle.getinfo("app.pyz.cdx.json") + content = handle.read(entry) + data = flagged_app.read_bytes() + name_length, extra_length = struct.unpack(" Path: with zipfile.ZipFile(app := tmp_path / "app.pyz", "w", zipfile.ZIP_DEFLATED) as handle: handle.writestr("__main__.py", "import app\n") handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") @@ -77,15 +90,7 @@ def test_archive_embeds_sbom_compressed_as_its_members(project: Path, tmp_path: for at in (index for index in range(len(data)) if data[index : index + 4] == b"PK\x01\x02"): struct.pack_into(" None: diff --git a/tests/_cli/test_embed.py b/tests/_cli/test_embed.py index 3c36cea..f713682 100644 --- a/tests/_cli/test_embed.py +++ b/tests/_cli/test_embed.py @@ -17,22 +17,36 @@ _SBOM: Final[str] = "native-1.0.dist-info/sboms/native.cdx.json" -def test_embed( +_COMMIT: Final[str] = "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" + + +@pytest.fixture +def embed_output( project: Path, wheel: Path, tmp_path: Path, install: Install, capsys: pytest.CaptureFixture[str] -) -> None: - commit = "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" +) -> str: install(site := tmp_path / "site", "Name: meson-python\nVersion: 0.18\n", {}) - main(["embed", str(wheel), "--root", str(project), "--commit", commit, "--build-env", str(site)]) - document = embedded(wheel) - assert capsys.readouterr().out == f"wrote {_SBOM} into {wheel.name}\n" - assert {"name": "native:vcs-commit", "value": commit} in document["metadata"]["component"]["properties"] - assert [tool["name"] for tool in document["metadata"]["tools"]["components"][1:]] == ["meson-python"] + main(["embed", str(wheel), "--root", str(project), "--commit", _COMMIT, "--build-env", str(site)]) + return capsys.readouterr().out + + +def test_embed_reports_the_sbom(embed_output: str, wheel: Path) -> None: + assert embed_output == f"wrote {_SBOM} into {wheel.name}\n" + + +@pytest.mark.usefixtures("embed_output") +def test_embed_records_the_commit(wheel: Path) -> None: + assert {"name": "native:vcs-commit", "value": _COMMIT} in embedded(wheel)["metadata"]["component"]["properties"] def embedded(wheel: Path) -> Document: return json.loads(zipfile.ZipFile(wheel).read(_SBOM)) +@pytest.mark.usefixtures("embed_output") +def test_embed_lists_the_build_environment(wheel: Path) -> None: + assert [tool["name"] for tool in embedded(wheel)["metadata"]["tools"]["components"][1:]] == ["meson-python"] + + def test_embed_defaults_to_current_directory(project: Path, wheel: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(project) monkeypatch.setenv("PYTHON_SBOM_COMMIT", "cafecafecafecafecafecafecafecafecafecafe") diff --git a/tests/hatch/test_hook.py b/tests/hatch/test_hook.py index c5bd2cc..90043d9 100644 --- a/tests/hatch/test_hook.py +++ b/tests/hatch/test_hook.py @@ -52,16 +52,11 @@ def test_wheel_carries_valid_sbom(project: Path, build_wheel: BuildWheel) -> Non assert validate(build_wheel(project), frozenset({"MIT"})) == [] -def test_sbom_describes_the_built_wheel(project: Path, build_wheel: BuildWheel) -> None: +@pytest.fixture +def retagged_wheel(project: Path, build_wheel: BuildWheel) -> Path: (project / "hatch_build.py").write_text(_RETAG, encoding="utf-8") configure(project, "[tool.hatch.build.hooks.custom]\n") - document = sbom_of(wheel := build_wheel(project)) - references = { - reference["type"]: reference["url"] for reference in document["metadata"]["component"]["externalReferences"] - } - assert [component["name"] for component in document["components"]] == ["packaging", "late-dependency"] - assert references["attestation"] == f"https://pypi.org/integrity/Example/1.0/{wheel.name}/provenance" - assert wheel.name == "example-1.0-py3-none-manylinux_2_17_x86_64.whl" + return build_wheel(project) def configure(root: Path, tables: str) -> None: @@ -69,6 +64,24 @@ def configure(root: Path, tables: str) -> None: pyproject.write_text(f"{pyproject.read_text(encoding='utf-8')}{tables}", encoding="utf-8") +def test_sbom_lists_dependencies_a_later_hook_adds(retagged_wheel: Path) -> None: + assert [component["name"] for component in sbom_of(retagged_wheel)["components"]] == [ + "packaging", + "late-dependency", + ] + + +def test_sbom_names_the_retagged_wheel(retagged_wheel: Path) -> None: + references = sbom_of(retagged_wheel)["metadata"]["component"]["externalReferences"] + assert {reference["type"]: reference["url"] for reference in references}["attestation"] == ( + f"https://pypi.org/integrity/Example/1.0/{retagged_wheel.name}/provenance" + ) + + +def test_later_hook_retags_the_wheel(retagged_wheel: Path) -> None: + assert retagged_wheel.name == "example-1.0-py3-none-manylinux_2_17_x86_64.whl" + + def test_project_table_options(project: Path, build_wheel: BuildWheel) -> None: configure( project, From 09c32544ae2a35b56dc35bad4c84b9726a805033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:25:28 -0800 Subject: [PATCH 099/115] =?UTF-8?q?=E2=9A=A1=20perf:=20keep=20validation?= =?UTF-8?q?=20out=20of=20--help=20and=20--version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser imported _validate._document, and with it _formats._spdx, only to show the --max-components default. The cap and the size limit it sets now live in _validate._limits, which imports nothing. --- src/python_sbom/_cli/_parser.py | 3 ++- src/python_sbom/_cli/_spdx.py | 3 ++- src/python_sbom/_validate/_artifact.py | 3 ++- src/python_sbom/_validate/_document.py | 14 ++------------ src/python_sbom/_validate/_limits.py | 20 ++++++++++++++++++++ tests/_cli/test_spdx.py | 2 +- 6 files changed, 29 insertions(+), 16 deletions(-) create mode 100644 src/python_sbom/_validate/_limits.py diff --git a/src/python_sbom/_cli/_parser.py b/src/python_sbom/_cli/_parser.py index 756f691..88b09f5 100644 --- a/src/python_sbom/_cli/_parser.py +++ b/src/python_sbom/_cli/_parser.py @@ -4,7 +4,8 @@ from pathlib import Path from typing import Protocol -from python_sbom._validate._document import MAX_COMPONENTS +# the limits module imports nothing, so --help and --version load none of validation +from python_sbom._validate._limits import MAX_COMPONENTS from python_sbom.version import __version__ diff --git a/src/python_sbom/_cli/_spdx.py b/src/python_sbom/_cli/_spdx.py index 8ef93cb..c2320e8 100644 --- a/src/python_sbom/_cli/_spdx.py +++ b/src/python_sbom/_cli/_spdx.py @@ -5,7 +5,8 @@ from python_sbom._errors import SbomError from python_sbom._formats._spdx import to_spdx -from python_sbom._validate._document import MAX_COMPONENTS, byte_limit, is_document +from python_sbom._validate._document import is_document +from python_sbom._validate._limits import MAX_COMPONENTS, byte_limit if TYPE_CHECKING: from pathlib import Path diff --git a/src/python_sbom/_validate/_artifact.py b/src/python_sbom/_validate/_artifact.py index a43038d..99b54eb 100644 --- a/src/python_sbom/_validate/_artifact.py +++ b/src/python_sbom/_validate/_artifact.py @@ -5,7 +5,8 @@ from python_sbom._wheel._archive import open_archive -from ._document import MAX_COMPONENTS, byte_limit, validate_document +from ._document import validate_document +from ._limits import MAX_COMPONENTS, byte_limit if TYPE_CHECKING: from pathlib import Path diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index c9d258f..237d79e 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -12,6 +12,8 @@ from python_sbom._model._document import walk_components from python_sbom._model._names import SBOM_FILE +from ._limits import MAX_COMPONENTS, byte_limit + if TYPE_CHECKING: from typing import TypeIs # new in 3.13; annotations here are never evaluated @@ -29,9 +31,6 @@ ("dependencies",), ) _SERIAL: Final[re.Pattern[str]] = re.compile(r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") -# every check takes time in proportion to the count, so the cap bounds the work an untrusted document can ask for, far -# above the few thousand components a wheel with its build tools' files lists -MAX_COMPONENTS: Final[int] = 100_000 # the modules of the validate extra _EXTRA: Final[tuple[str, ...]] = ("jsonschema_rs", "license_expression", "spdx_tools") # an exception's message, or a schema error's, can quote the hostile value that caused it, which runs to megabytes @@ -41,13 +40,6 @@ _MAX_DEPTH: Final[int] = 128 -def byte_limit(max_components: int) -> int: - # python-sbom writes about 540 bytes per component, since every file of the wheel and of its build tools is one; a - # KiB each leaves room for documents with fuller components, plus a MiB for the metadata, dependencies and - # formulation beside them, while tying what the JSON parser may be asked to hold to the component cap - return 1024 * 1024 + max_components * 1024 - - def validate_document( raw: bytes, allowed_licenses: frozenset[str] | None, max_components: int = MAX_COMPONENTS ) -> list[str]: @@ -191,10 +183,8 @@ def _reference_problems(document: Document) -> list[str]: __all__ = [ - "MAX_COMPONENTS", "MAX_TEXT", "Json", - "byte_limit", "is_document", "validate_document", ] diff --git a/src/python_sbom/_validate/_limits.py b/src/python_sbom/_validate/_limits.py new file mode 100644 index 0000000..602e551 --- /dev/null +++ b/src/python_sbom/_validate/_limits.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from typing import Final + +# every check takes time in proportion to the count, so the cap bounds the work an untrusted document can ask for, far +# above the few thousand components a wheel with its build tools' files lists +MAX_COMPONENTS: Final[int] = 100_000 + + +def byte_limit(max_components: int) -> int: + # python-sbom writes about 540 bytes per component, since every file of the wheel and of its build tools is one; a + # KiB each leaves room for documents with fuller components, plus a MiB for the metadata, dependencies and + # formulation beside them, while tying what the JSON parser may be asked to hold to the component cap + return 1024 * 1024 + max_components * 1024 + + +__all__ = [ + "MAX_COMPONENTS", + "byte_limit", +] diff --git a/tests/_cli/test_spdx.py b/tests/_cli/test_spdx.py index 5d50746..8207a85 100644 --- a/tests/_cli/test_spdx.py +++ b/tests/_cli/test_spdx.py @@ -7,7 +7,7 @@ from python_sbom._cli._main import main from python_sbom._formats._json import dumps -from python_sbom._validate._document import MAX_COMPONENTS, byte_limit +from python_sbom._validate._limits import MAX_COMPONENTS, byte_limit if TYPE_CHECKING: from pathlib import Path From b3728af23fb13399da715bf9162fc196787736a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:26:05 -0800 Subject: [PATCH 100/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20raise?= =?UTF-8?q?=20one=20error=20class?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SettingsError and WheelError, and the ValueError and ImportError bases, had no caller that told them apart from SbomError. Every error is now an SbomError; MissingExtraError stays, as validation lets it through where it reports every other error per document. --- src/python_sbom/_collect/_archive.py | 4 ++-- src/python_sbom/_collect/_content.py | 8 ++++---- src/python_sbom/_collect/_facts.py | 4 ++-- src/python_sbom/_collect/_revision.py | 4 ++-- src/python_sbom/_collect/_settings.py | 18 +++++++++--------- src/python_sbom/_collect/_source.py | 20 ++++++++++---------- src/python_sbom/_errors.py | 13 ++----------- src/python_sbom/_wheel/_archive.py | 16 ++++++++-------- src/python_sbom/_wheel/_record.py | 6 +++--- src/python_sbom/hatch/_hook.py | 4 ++-- tests/_collect/test_archive.py | 4 ++-- tests/_collect/test_content.py | 4 ++-- tests/_collect/test_facts.py | 4 ++-- tests/_collect/test_revision.py | 8 ++++---- tests/_collect/test_settings.py | 10 +++++----- tests/_collect/test_source.py | 24 ++++++++++++------------ tests/_generate/test_archive.py | 4 ++-- tests/_validate/test_artifact.py | 6 +++--- tests/_wheel/test_archive.py | 22 +++++++++++----------- tests/_wheel/test_record.py | 4 ++-- tests/hatch/test_hook.py | 4 ++-- 21 files changed, 91 insertions(+), 100 deletions(-) diff --git a/src/python_sbom/_collect/_archive.py b/src/python_sbom/_collect/_archive.py index 044d91d..01d4683 100644 --- a/src/python_sbom/_collect/_archive.py +++ b/src/python_sbom/_collect/_archive.py @@ -3,7 +3,7 @@ from io import BytesIO from typing import TYPE_CHECKING, Final -from python_sbom._errors import WheelError +from python_sbom._errors import SbomError from python_sbom._model._inputs import ArchiveTree, RecordRow from python_sbom._model._metadata import parse_metadata from python_sbom._wheel._archive import member_digest, open_archive, read_member @@ -35,7 +35,7 @@ def read_archive(archive: Path) -> ArchiveTree: } if (top := sum(name.count("/") == 1 and name.endswith(".dist-info/METADATA") for name in members)) != 1: msg = f"{where} holds {top} top-level .dist-info/METADATA, not the one naming the project" - raise WheelError(msg) + raise SbomError(msg) files: Final[dict[str, RecordRow]] = { name: RecordRow( path=name, diff --git a/src/python_sbom/_collect/_content.py b/src/python_sbom/_collect/_content.py index 3558e5a..04dc728 100644 --- a/src/python_sbom/_collect/_content.py +++ b/src/python_sbom/_collect/_content.py @@ -6,7 +6,7 @@ from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Final -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError from python_sbom._model._content import ContentProvider if TYPE_CHECKING: @@ -32,21 +32,21 @@ def _load(spec: str, root: Path) -> ContentProvider: module_name, _, attribute = spec.partition(":") if not (module_name and attribute): msg = f"content provider {spec!r} must be `module:attribute`" - raise SettingsError(msg) + raise SbomError(msg) sys.path.insert(0, str(root)) try: provider = pkgutil.resolve_name(spec) # ValueError: a name resolve_name cannot parse, such as a second colon except (ImportError, AttributeError, ValueError) as error: msg = f"content provider {spec!r} does not load from {root}: {error}" - raise SettingsError(msg) from error + raise SbomError(msg) from error finally: sys.path.remove(str(root)) if isinstance(provider, type) and issubclass(provider, ContentProvider): provider = provider() if not isinstance(provider, ContentProvider): msg = f"content provider {spec!r} is not a ContentProvider subclass or instance" - raise SettingsError(msg) + raise SbomError(msg) return provider diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index b8856fa..9759294 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -15,7 +15,7 @@ from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError from python_sbom._model._inputs import BuildFacts, InstalledDistribution, Interpreter, RecordRow from python_sbom._model._metadata import parse_metadata from python_sbom._wheel._record import record_rows @@ -38,7 +38,7 @@ def collect_facts( epoch = int(raw) except ValueError as error: msg = f"SOURCE_DATE_EPOCH must be a Unix timestamp, got {raw!r}" - raise SettingsError(msg) from error + raise SbomError(msg) from error else: epoch = int(time.time()) if default_epoch is None else default_epoch return BuildFacts( diff --git a/src/python_sbom/_collect/_revision.py b/src/python_sbom/_collect/_revision.py index 3efdf70..142fb88 100644 --- a/src/python_sbom/_collect/_revision.py +++ b/src/python_sbom/_collect/_revision.py @@ -7,7 +7,7 @@ import warnings from typing import TYPE_CHECKING, Final, Protocol -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError from python_sbom._model._inputs import Revision if TYPE_CHECKING: @@ -81,7 +81,7 @@ def _commit_revision(commit: str, origin: str) -> Revision: # the commit lands in the source link, so anything but a full hash could point the link elsewhere if not _COMMIT.fullmatch(commit): msg = f"{origin} must be a full lowercase hex commit hash of 40 or 64 characters, got {commit!r}" - raise SettingsError(msg) + raise SbomError(msg) return Revision(commit=commit, dirty=False) diff --git a/src/python_sbom/_collect/_settings.py b/src/python_sbom/_collect/_settings.py index a0df191..751d47f 100644 --- a/src/python_sbom/_collect/_settings.py +++ b/src/python_sbom/_collect/_settings.py @@ -3,7 +3,7 @@ import tomllib from typing import TYPE_CHECKING, TypeAlias -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError from python_sbom._model._inputs import Settings if TYPE_CHECKING: @@ -24,10 +24,10 @@ def settings_from_pyproject(root: Path) -> Settings: loaded = tomllib.loads(pyproject.read_text(encoding="utf-8")) except FileNotFoundError as error: msg = f"{root} has no pyproject.toml to read [tool.python-sbom] from" - raise SettingsError(msg) from error + raise SbomError(msg) from error except tomllib.TOMLDecodeError as error: msg = f"{pyproject} is not valid TOML: {error}" - raise SettingsError(msg) from error + raise SbomError(msg) from error return settings_from_table(loaded.get("tool", {}).get("python-sbom", {}), f"[tool.python-sbom] of {pyproject}") @@ -35,19 +35,19 @@ def settings_from_table(table: Mapping[str, Toml], where: str) -> Settings: known = {"bundled-wheels", "content-provider", "default-branch", "repository", "supplier"} if unknown := sorted(table.keys() - known): msg = f"{where} has unknown option(s) {', '.join(unknown)}; known: {', '.join(sorted(known))}" - raise SettingsError(msg) + raise SbomError(msg) if not isinstance(repository := table.get("repository"), str | None): msg = f"`repository` in {where} must be a string" - raise SettingsError(msg) + raise SbomError(msg) if not isinstance(default_branch := table.get("default-branch", "main"), str): msg = f"`default-branch` in {where} must be a string" - raise SettingsError(msg) + raise SbomError(msg) if not _strings(patterns := table.get("bundled-wheels", [])): msg = f"`bundled-wheels` in {where} must be an array of strings" - raise SettingsError(msg) + raise SbomError(msg) if not isinstance(content_provider := table.get("content-provider"), str | None): msg = f"`content-provider` in {where} must be a string, `module:attribute`" - raise SettingsError(msg) + raise SbomError(msg) return Settings( supplier=_supplier(table.get("supplier"), where), repository=None if repository is None else repository.rstrip("/"), @@ -72,7 +72,7 @@ def _supplier(value: Toml | None, where: str) -> OrganizationalEntity | None: ): return {"name": name, "url": urls} msg = f"`supplier` in {where} must be a table with a string `name` and an optional `url` string array" - raise SettingsError(msg) + raise SbomError(msg) __all__ = [ diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index 8c7d6a6..c6c89fd 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -10,7 +10,7 @@ from installer.sources import WheelFile from packaging.requirements import InvalidRequirement, Requirement -from python_sbom._errors import SettingsError, WheelError +from python_sbom._errors import SbomError from python_sbom._model._inputs import BuildScript, BundledWheel, RecordRow, SbomRequest, SourceTree from python_sbom._model._metadata import CoreMetadata, parse_metadata from python_sbom._wheel._archive import open_archive, read_member, read_wheel_metadata @@ -37,7 +37,7 @@ def read_source_tree(request: SbomRequest) -> SourceTree: # a wheel outside the project is none the project ships, and pathlib refuses an absolute pattern outright if (path := PurePath(pattern)).anchor or ".." in path.parts: msg = f"`bundled-wheels` pattern {pattern!r} must be relative to the project, without `..`" - raise SettingsError(msg) + raise SbomError(msg) wheels.update(wheel.relative_to(root).as_posix() for wheel in root.glob(pattern)) return SourceTree( copyright=copyright_notice(root, request.metadata.values("license_files")), @@ -106,7 +106,7 @@ def _read_bundled_wheel(stream: IO[bytes], path: str) -> BundledWheel: def _check_named(metadata: CoreMetadata, where: str, name: str) -> None: if not (metadata.name and metadata.version): msg = f"{where}: {name} lacks Name or Version" - raise WheelError(msg) + raise SbomError(msg) def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> list[RecordRow]: @@ -121,12 +121,12 @@ def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> found: Final[list[RecordRow]] = record_rows(rows, where) except InvalidRecordEntry as error: msg = f"{where}: RECORD does not parse: {error}" - raise WheelError(msg) from error + raise SbomError(msg) from error # installer checks the last row of a path only, skips rows naming no member, and lets a RECORD.jws or RECORD.p7s # anywhere under the .dist-info, "../" included, go unlisted, so each of these checks is ours if repeated := sorted(path for path, count in Counter(path for path, _, _ in rows).items() if count > 1): msg = f"{where}: RECORD lists {', '.join(repeated)} more than once" - raise WheelError(msg) + raise SbomError(msg) members: Final[set[str]] = {info.filename for info in archive.infolist() if not info.is_dir()} recorded: Final[set[str]] = {path for path, _, _ in rows} # installer checks the row of RECORD itself, the one row without a digest @@ -134,10 +134,10 @@ def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> members - recorded - {f"{dist_info}/{name}" for name in ("RECORD", "RECORD.jws", "RECORD.p7s")} ): msg = f"{where}: RECORD lacks a digest for {', '.join(missing)}" - raise WheelError(msg) + raise SbomError(msg) if extra := sorted(recorded - members): msg = f"{where}: RECORD lists {', '.join(extra)}, which the wheel does not hold" - raise WheelError(msg) + raise SbomError(msg) try: wheel.validate_record() except WheelFile.validation_error as error: @@ -145,7 +145,7 @@ def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> issues: Final[list[str]] = getattr(error, "issues", [str(error)]) prefix: Final[str] = f"In {archive.filename}, " msg = f"{where}: RECORD disagrees with the wheel: {'; '.join(issue.removeprefix(prefix) for issue in issues)}" - raise WheelError(msg) from error + raise SbomError(msg) from error return found @@ -160,10 +160,10 @@ def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) - pins = list((requirement := Requirement(text)).specifier) except InvalidRequirement as error: msg = f"{where}: vendored requirement in {name} does not parse: {text}" - raise WheelError(msg) from error + raise SbomError(msg) from error if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: msg = f"{where}: vendored requirement in {name} pins no one exact version: {text}" - raise WheelError(msg) + raise SbomError(msg) found.append((name, CoreMetadata(raw={"name": requirement.name, "version": pins[0].version}, unparsed={}))) return found diff --git a/src/python_sbom/_errors.py b/src/python_sbom/_errors.py index 43400cb..b6e6ba1 100644 --- a/src/python_sbom/_errors.py +++ b/src/python_sbom/_errors.py @@ -5,20 +5,11 @@ class SbomError(Exception): """Base of the errors this package raises; the command line reports them without a traceback.""" -# callers catch these as SbomError, or as the built-in some of them also are; SettingsError covers every option, file or -# variable the project sets wrongly -class SettingsError(SbomError): ... # simplint: ignore[SMP704] # see above - - -class WheelError(SbomError, ValueError): ... # simplint: ignore[SMP704] # see above - - -class MissingExtraError(SbomError, ImportError): ... # simplint: ignore[SMP704] # see above +# validation reports every other error per document, and lets this one through, since it fails every document alike +class MissingExtraError(SbomError): ... # simplint: ignore[SMP704] # see above __all__ = [ "MissingExtraError", "SbomError", - "SettingsError", - "WheelError", ] diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index 4c62acc..c298b50 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -20,7 +20,7 @@ from installer.sources import WheelFile from installer.utils import copyfileobj_with_hashing -from python_sbom._errors import WheelError +from python_sbom._errors import SbomError from python_sbom._model._metadata import CoreMetadata, parse_metadata if TYPE_CHECKING: @@ -42,7 +42,7 @@ def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: record: Final[str] = f"{dist_info}/RECORD" if missing := sorted({f"{dist_info}/METADATA", record} - set(archive.namelist())): msg = f"{wheel} has no {', '.join(missing)}" - raise WheelError(msg) + raise SbomError(msg) record_info: Final[zipfile.ZipInfo] = archive.getinfo(record) record_data: Final[bytes] = archive.read(record_info) sbom: Final[str] = f"{dist_info}/sboms/{name}" @@ -82,12 +82,12 @@ def open_archive(file: Path | IO[bytes], where: str) -> Generator[zipfile.ZipFil name for name, count in Counter(name.casefold() for name in archive.namelist()).items() if count > 1 ): msg = f"{where} holds more than one member named {', '.join(clashes)}, ignoring case" - raise WheelError(msg) + raise SbomError(msg) yield archive # zipfile raises these as late as a member's last read, inside installer's checks too, which let them through except (zipfile.BadZipFile, zlib.error) as error: msg = f"{where} is not a valid zip archive: {error}" - raise WheelError(msg) from error + raise SbomError(msg) from error def _wheel_source(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, str]: @@ -98,7 +98,7 @@ def _wheel_source(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, str] dist_info = wheel.dist_info_dir except ValueError as error: msg = f"{where} is not a wheel: {error}" - raise WheelError(msg) from error + raise SbomError(msg) from error return wheel, dist_info @@ -139,7 +139,7 @@ def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _M info = archive.getinfo(name) except KeyError as error: msg = f"{where} has no {name}" - raise WheelError(msg) from error + raise SbomError(msg) from error # counted as it decompresses, since the size the archive states may lie and one read decompresses it all at once content: Final[bytearray] = bytearray() with archive.open(info) as member: @@ -147,7 +147,7 @@ def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _M content.extend(chunk) if len(content) > limit: msg = f"{where}: {name} is over the {limit} byte limit" - raise WheelError(msg) + raise SbomError(msg) return bytes(content) @@ -163,7 +163,7 @@ def append_member(archive: Path, name: str, content: bytes, epoch: int) -> None: with zipfile.ZipFile(archive, "a") as handle: if name in handle.namelist(): msg = f"{archive.name} already holds {name}" - raise WheelError(msg) + raise SbomError(msg) # compressed by the method most of the archive's files use, at the level their deflate option bits name, since # the zip format records no level itself and Python's zipfile leaves the bits at normal, zlib's default; # directories and empty files count for none, as zip tools store them whatever they compress diff --git a/src/python_sbom/_wheel/_record.py b/src/python_sbom/_wheel/_record.py index cfbdf18..db6e31d 100644 --- a/src/python_sbom/_wheel/_record.py +++ b/src/python_sbom/_wheel/_record.py @@ -7,7 +7,7 @@ from installer.records import RecordEntry -from python_sbom._errors import WheelError +from python_sbom._errors import SbomError from python_sbom._model._inputs import RecordRow if TYPE_CHECKING: @@ -41,12 +41,12 @@ def decode_digest(digest: str, where: str) -> tuple[str, str]: name, _, encoded = digest.partition("=") if (algorithm := _ALGORITHMS.get(name)) is None: msg = f"{where}: unknown hash algorithm {name!r}" - raise WheelError(msg) + raise SbomError(msg) try: raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) except binascii.Error as error: msg = f"{where}: digest {encoded!r} is not urlsafe base64" - raise WheelError(msg) from error + raise SbomError(msg) from error return algorithm, raw.hex() diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index ba36b4b..1655c05 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -14,7 +14,7 @@ from python_sbom._collect._content import resolve_provider from python_sbom._collect._revision import REVISION_FILE, default_revision from python_sbom._collect._settings import settings_from_table -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError if TYPE_CHECKING: from collections.abc import Callable @@ -139,7 +139,7 @@ def _settings(self) -> Settings: f"the {self.PLUGIN_NAME} hook's table holds {', '.join(unknown)}; move python-sbom's options to" " [tool.python-sbom], where python-sbom embed and python-sbom archive read them too" ) - raise SettingsError(msg) + raise SbomError(msg) return settings_from_table(self.metadata.config.get("tool", {}).get("python-sbom", {}), "[tool.python-sbom]") diff --git a/tests/_collect/test_archive.py b/tests/_collect/test_archive.py index 3d29cd9..24ef924 100644 --- a/tests/_collect/test_archive.py +++ b/tests/_collect/test_archive.py @@ -7,7 +7,7 @@ import pytest from python_sbom._collect._archive import member_reader, read_archive -from python_sbom._errors import WheelError +from python_sbom._errors import SbomError def test_member_reader_reads_a_member(tmp_path: Path) -> None: @@ -28,7 +28,7 @@ def test_member_reader_checks_the_archive(tmp_path: Path) -> None: with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: archive.writestr("table.py", "") archive.writestr("TABLE.py", "") - with pytest.raises(WheelError, match=r"holds more than one member named table\.py, ignoring case"): + with pytest.raises(SbomError, match=r"holds more than one member named table\.py, ignoring case"): member_reader(wheel)(PurePosixPath("table.py")) diff --git a/tests/_collect/test_content.py b/tests/_collect/test_content.py index ea7b472..8180605 100644 --- a/tests/_collect/test_content.py +++ b/tests/_collect/test_content.py @@ -9,7 +9,7 @@ import python_sbom from python_sbom import ContentProvider from python_sbom._collect._content import resolve_provider -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError if TYPE_CHECKING: from python_sbom import Settings @@ -110,7 +110,7 @@ class Local(ContentProvider): ], ) def test_load_refuses(tmp_path: Path, module: str, spec: str, message: str, settings: Settings) -> None: - with pytest.raises(SettingsError, match=message): + with pytest.raises(SbomError, match=message): resolve_provider(spec.replace("MODULE", module), settings, tmp_path) diff --git a/tests/_collect/test_facts.py b/tests/_collect/test_facts.py index 80d01d5..149fe7a 100644 --- a/tests/_collect/test_facts.py +++ b/tests/_collect/test_facts.py @@ -10,7 +10,7 @@ import pytest from python_sbom._collect._facts import collect_facts -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError from python_sbom._model._inputs import Interpreter, RecordRow, Revision if TYPE_CHECKING: @@ -50,7 +50,7 @@ def test_epoch_without_default_is_now(tmp_path: Path) -> None: def test_invalid_epoch(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("SOURCE_DATE_EPOCH", "yesterday") - with pytest.raises(SettingsError, match="SOURCE_DATE_EPOCH must be a Unix timestamp, got 'yesterday'"): + with pytest.raises(SbomError, match="SOURCE_DATE_EPOCH must be a Unix timestamp, got 'yesterday'"): collect_facts(tmp_path, no_revision, path=[]) diff --git a/tests/_collect/test_revision.py b/tests/_collect/test_revision.py index ea420d5..e6f23ed 100644 --- a/tests/_collect/test_revision.py +++ b/tests/_collect/test_revision.py @@ -5,7 +5,7 @@ import pytest from python_sbom._collect._revision import REVISION_FILE, default_revision, revision_reader -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError from python_sbom._model._inputs import Revision if TYPE_CHECKING: @@ -33,7 +33,7 @@ def test_commit_from_environment_must_be_full_hash( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, commit: str ) -> None: monkeypatch.setenv("PYTHON_SBOM_COMMIT", commit) - with pytest.raises(SettingsError, match="PYTHON_SBOM_COMMIT must be a full lowercase hex commit hash"): + with pytest.raises(SbomError, match="PYTHON_SBOM_COMMIT must be a full lowercase hex commit hash"): default_revision(tmp_path) @@ -58,7 +58,7 @@ def test_revision_file(tmp_path: Path) -> None: def test_invalid_revision_file(tmp_path: Path) -> None: (tmp_path / REVISION_FILE).write_text("node: d\n", encoding="utf-8") - with pytest.raises(SettingsError, match=r"\.python-sbom-revision must be a full lowercase hex commit hash"): + with pytest.raises(SbomError, match=r"\.python-sbom-revision must be a full lowercase hex commit hash"): default_revision(tmp_path) @@ -189,7 +189,7 @@ def test_given_commit_wins_over_checkout(checkout: Path) -> None: def test_given_commit_must_be_full_hash() -> None: - with pytest.raises(SettingsError, match="--commit must be a full lowercase hex commit hash"): + with pytest.raises(SbomError, match="--commit must be a full lowercase hex commit hash"): revision_reader("c0ffee") diff --git a/tests/_collect/test_settings.py b/tests/_collect/test_settings.py index 37a1a72..a069ab3 100644 --- a/tests/_collect/test_settings.py +++ b/tests/_collect/test_settings.py @@ -5,7 +5,7 @@ import pytest from python_sbom._collect._settings import settings_from_pyproject, settings_from_table -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError from python_sbom._model._inputs import Settings if TYPE_CHECKING: @@ -15,13 +15,13 @@ def test_missing_pyproject(tmp_path: Path) -> None: - with pytest.raises(SettingsError, match=r"has no pyproject\.toml to read"): + with pytest.raises(SbomError, match=r"has no pyproject\.toml to read"): settings_from_pyproject(tmp_path) def test_invalid_pyproject(tmp_path: Path) -> None: (tmp_path / "pyproject.toml").write_text("[tool\n", encoding="utf-8") - with pytest.raises(SettingsError, match=r"pyproject\.toml is not valid TOML"): + with pytest.raises(SbomError, match=r"pyproject\.toml is not valid TOML"): settings_from_pyproject(tmp_path) @@ -50,7 +50,7 @@ def test_table(tmp_path: Path) -> None: def test_unknown_option() -> None: - with pytest.raises(SettingsError, match=r"here has unknown option\(s\) bundled, suplier; known: bundled-wheels"): + with pytest.raises(SbomError, match=r"here has unknown option\(s\) bundled, suplier; known: bundled-wheels"): settings_from_table({"suplier": {}, "bundled": []}, "here") @@ -71,7 +71,7 @@ def test_unknown_option() -> None: ], ) def test_wrong_type(table: dict[str, Toml], message: str) -> None: - with pytest.raises(SettingsError, match=message): + with pytest.raises(SbomError, match=message): settings_from_table(table, "here") diff --git a/tests/_collect/test_source.py b/tests/_collect/test_source.py index a5e2128..97dd385 100644 --- a/tests/_collect/test_source.py +++ b/tests/_collect/test_source.py @@ -13,7 +13,7 @@ import pytest from python_sbom._collect._source import read_bundled_wheels, read_source_tree, wheel_request -from python_sbom._errors import SettingsError, WheelError +from python_sbom._errors import SbomError from python_sbom._model._inputs import BuildScript, BundledWheel, RecordRow, SbomRequest from python_sbom._model._metadata import parse_metadata @@ -91,7 +91,7 @@ def test_bundled_wheel_pattern_outside_project( (project := tmp_path / "project" / "src").mkdir(parents=True) pattern = pattern.replace("ROOT", tmp_path.as_posix()) settings = dataclasses.replace(settings, bundled_wheels=(pattern,)) - with pytest.raises(SettingsError, match=f"`bundled-wheels` pattern {re.escape(repr(pattern))} must be relative"): + with pytest.raises(SbomError, match=f"`bundled-wheels` pattern {re.escape(repr(pattern))} must be relative"): read_source_tree(request(project.parent, settings)) @@ -169,7 +169,7 @@ def test_malformed_bundled_wheel( tmp_path: Path, write_wheel: WriteWheel, files: Mapping[str, bytes], message: str ) -> None: wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - with pytest.raises(WheelError, match=message): + with pytest.raises(SbomError, match=message): _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") @@ -186,12 +186,12 @@ def test_bundled_wheel_file_name_names_the_project( tmp_path: Path, write_wheel: WriteWheel, name: str, message: str ) -> None: wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", _PIP) - with pytest.raises(WheelError, match=f"bundled wheel {name} is not a wheel: {message}"): + with pytest.raises(SbomError, match=f"bundled wheel {name} is not a wheel: {message}"): _bundled(wheel.read_bytes(), name) def test_corrupt_bundled_wheel() -> None: - with pytest.raises(WheelError, match=r"bundled wheel pip\.whl is not a valid zip archive"): + with pytest.raises(SbomError, match=r"bundled wheel pip\.whl is not a valid zip archive"): _bundled(b"not a zip", "pip.whl") @@ -203,7 +203,7 @@ def test_bundled_wheel_with_duplicate_member(tmp_path: Path) -> None: archive.writestr(_INIT, b"") archive.writestr(_METADATA, _PIP[_METADATA]) archive.writestr("pip-1.0.dist-info/RECORD", f"{row(_INIT, b'')}{_METADATA_ROW}") - with pytest.raises(WheelError, match=f"holds more than one member named {_INIT}, ignoring case"): + with pytest.raises(SbomError, match=f"holds more than one member named {_INIT}, ignoring case"): _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") @@ -288,7 +288,7 @@ def test_bundled_wheel_record_algorithm(tmp_path: Path, write_wheel: WriteWheel, def test_record_disagrees_with_wheel(tmp_path: Path, write_wheel: WriteWheel, record: str, message: str) -> None: files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - with pytest.raises(WheelError, match=message): + with pytest.raises(SbomError, match=message): _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") @@ -311,14 +311,14 @@ def test_signature_away_from_record_needs_a_row(tmp_path: Path, write_wheel: Wri record = "".join(starmap(row, _PIP.items())) + _RECORD_ROW files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), signature: b"import os\n"} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - with pytest.raises(WheelError, match=f"RECORD lacks a digest for {re.escape(signature)}"): + with pytest.raises(SbomError, match=f"RECORD lacks a digest for {re.escape(signature)}"): _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") def test_vendored_metadata_needs_name_and_version(tmp_path: Path, write_wheel: WriteWheel) -> None: files = {**_PIP, "pip/_vendor/rich-13.0.dist-info/METADATA": b"Metadata-Version: 2.1\n"} wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files) - with pytest.raises(WheelError, match=r"pip/_vendor/rich-13\.0\.dist-info/METADATA lacks Name or Version"): + with pytest.raises(SbomError, match=r"pip/_vendor/rich-13\.0\.dist-info/METADATA lacks Name or Version"): _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") @@ -335,7 +335,7 @@ def test_vendor_manifest_needs_exact_pins( tmp_path: Path, write_wheel: WriteWheel, manifest: bytes, message: str ) -> None: wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", {**_PIP, "pip/_vendor/vendor.txt": manifest}) - with pytest.raises(WheelError, match=message): + with pytest.raises(SbomError, match=message): _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") @@ -366,7 +366,7 @@ def test_wheel_request_needs_a_wheel( tmp_path: Path, settings: Settings, write_wheel: WriteWheel, name: str, message: str ) -> None: archive = write_wheel(tmp_path / name, {"native/__init__.py": b""}, record=False) - with pytest.raises(WheelError, match=message): + with pytest.raises(SbomError, match=message): wheel_request(archive, Path(), settings, ()) @@ -383,5 +383,5 @@ def test_bundled_wheels_come_in_the_order_given(tmp_path: Path, write_wheel: Wri paths[0], paths[0], ] - with pytest.raises(WheelError, match=r"^bundled wheel b\.whl is not a valid zip archive"): + with pytest.raises(SbomError, match=r"^bundled wheel b\.whl is not a valid zip archive"): read_bundled_wheels(paths, lambda name: BytesIO(wheels[name])) diff --git a/tests/_generate/test_archive.py b/tests/_generate/test_archive.py index ebfb396..96fc8e2 100644 --- a/tests/_generate/test_archive.py +++ b/tests/_generate/test_archive.py @@ -12,7 +12,7 @@ from python_sbom import BundledWheelInfo, ContentProvider, Contributions, ProviderContext from python_sbom._collect._facts import collect_facts from python_sbom._collect._revision import revision_reader -from python_sbom._errors import WheelError +from python_sbom._errors import SbomError from python_sbom._generate._archive import describe_archive from python_sbom._model._inputs import Revision from python_sbom._validate._artifact import validate @@ -227,7 +227,7 @@ def test_archive_without_project_distribution(tmp_path: Path) -> None: handle.writestr("__main__.py", "") (archive := tmp_path / "bare.pyz").write_bytes(buffer.getvalue()) with pytest.raises( - WheelError, match=r"bare\.pyz holds 0 top-level \.dist-info/METADATA, not the one naming the project" + SbomError, match=r"bare\.pyz holds 0 top-level \.dist-info/METADATA, not the one naming the project" ): describe_archive( tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None)), provider=None, build_scripts=() diff --git a/tests/_validate/test_artifact.py b/tests/_validate/test_artifact.py index a0ce0cb..55c7948 100644 --- a/tests/_validate/test_artifact.py +++ b/tests/_validate/test_artifact.py @@ -7,7 +7,7 @@ import pytest -from python_sbom._errors import WheelError +from python_sbom._errors import SbomError from python_sbom._formats._json import dumps from python_sbom._validate._artifact import validate @@ -129,7 +129,7 @@ def test_archive_with_duplicate_member(tmp_path: Path) -> None: archive.writestr("example.cdx.json", b"{}") with pytest.warns(UserWarning, match="Duplicate name"): archive.writestr("example.cdx.json", b"{}") - with pytest.raises(WheelError, match=r"holds more than one member named example\.cdx\.json"): + with pytest.raises(SbomError, match=r"holds more than one member named example\.cdx\.json"): validate(wheel, None) @@ -139,7 +139,7 @@ def test_corrupt_archive(tmp_path: Path) -> None: data = bytearray(wheel.read_bytes()) data[40:50] = b"\xff" * 10 wheel.write_bytes(bytes(data)) - with pytest.raises(WheelError, match=r"example-1\.0-py3-none-any\.whl is not a valid zip archive"): + with pytest.raises(SbomError, match=r"example-1\.0-py3-none-any\.whl is not a valid zip archive"): validate(wheel, None) diff --git a/tests/_wheel/test_archive.py b/tests/_wheel/test_archive.py index fef80c8..c98d371 100644 --- a/tests/_wheel/test_archive.py +++ b/tests/_wheel/test_archive.py @@ -12,7 +12,7 @@ import pytest -from python_sbom._errors import WheelError +from python_sbom._errors import SbomError from python_sbom._wheel._archive import append_member, open_archive, read_member, write_sbom if TYPE_CHECKING: @@ -162,7 +162,7 @@ def test_embed_refuses_non_wheel(tmp_path: Path, files: dict[str, bytes], messag with zipfile.ZipFile(archive := tmp_path / "native-1.0-py3-none-any.whl", "w") as handle: for name, content in files.items(): handle.writestr(name, content) - with pytest.raises(WheelError, match=message): + with pytest.raises(SbomError, match=message): write_sbom(archive, "native.cdx.json", b"{}\n", 1_700_000_000) @@ -181,13 +181,13 @@ def test_embed_refuses_duplicate_member(tmp_path: Path) -> None: handle.writestr("native-1.0.dist-info/RECORD", b"") with pytest.warns(UserWarning, match="Duplicate name"): handle.writestr("native-1.0.dist-info/RECORD", b"") - with pytest.raises(WheelError, match=r"holds more than one member named native-1\.0\.dist-info/record"): + with pytest.raises(SbomError, match=r"holds more than one member named native-1\.0\.dist-info/record"): write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) def test_embed_refuses_corrupt_archive(tmp_path: Path) -> None: (wheel := tmp_path / "native-1.0-py3-none-any.whl").write_bytes(b"not a zip") - with pytest.raises(WheelError, match="is not a valid zip archive: File is not a zip file"): + with pytest.raises(SbomError, match="is not a valid zip archive: File is not a zip file"): write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) @@ -200,7 +200,7 @@ def test_embed_refuses_corrupt_archive(tmp_path: Path) -> None: ) def test_open_archive_refuses_names_equal_ignoring_case(names: tuple[str, str]) -> None: message = f"w holds more than one member named {names[0].lower()}, ignoring case" - with pytest.raises(WheelError, match=message), ExitStack() as stack: + with pytest.raises(SbomError, match=message), ExitStack() as stack: stack.enter_context(open_archive(zip_of(*((name, b"") for name in names)), "w")) @@ -215,7 +215,7 @@ def zip_of(*members: tuple[str, bytes]) -> BytesIO: def test_open_archive_refuses_corrupt_member() -> None: data = bytearray(zip_of(("a", b"hello" * 1000)).getvalue()) data[40:50] = b"\xff" * 10 - with pytest.raises(WheelError, match="w is not a valid zip"), open_archive(BytesIO(bytes(data)), "w") as archive: + with pytest.raises(SbomError, match="w is not a valid zip"), open_archive(BytesIO(bytes(data)), "w") as archive: archive.read("a") @@ -225,12 +225,12 @@ def test_read_member() -> None: def test_read_member_over_limit() -> None: - with open_archive(zip_of(("a", b"12345")), "w") as archive, pytest.raises(WheelError, match="w: a is over the 4"): + with open_archive(zip_of(("a", b"12345")), "w") as archive, pytest.raises(SbomError, match="w: a is over the 4"): read_member(archive, "a", "w", limit=4) def test_read_member_missing() -> None: - with open_archive(zip_of(("a", b"")), "w") as archive, pytest.raises(WheelError, match="w has no b"): + with open_archive(zip_of(("a", b"")), "w") as archive, pytest.raises(SbomError, match="w has no b"): read_member(archive, "b", "w") @@ -240,7 +240,7 @@ def test_read_member_refuses_understated_size() -> None: data = bytearray(zip_of(("a", b"A" * 32 * 1024 * 1024)).getvalue()) data[22:26] = data[(central := data.rfind(b"PK\x01\x02")) + 24 : central + 28] = struct.pack(" None: with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w") as handle: handle.writestr("app.pyz.cdx.json", b"{}") - with pytest.raises(WheelError, match=r"app\.pyz already holds app\.pyz\.cdx\.json"): + with pytest.raises(SbomError, match=r"app\.pyz already holds app\.pyz\.cdx\.json"): append_member(archive, "app.pyz.cdx.json", b"{}\n", 1_700_000_000) diff --git a/tests/_wheel/test_record.py b/tests/_wheel/test_record.py index 3bb24a7..0e32df9 100644 --- a/tests/_wheel/test_record.py +++ b/tests/_wheel/test_record.py @@ -4,7 +4,7 @@ import pytest -from python_sbom._errors import WheelError +from python_sbom._errors import SbomError from python_sbom._wheel._record import decode_digest @@ -33,5 +33,5 @@ def test_decode_digest(digest: str, expected: tuple[str, str]) -> None: ], ) def test_decode_digest_rejects(digest: str, message: str) -> None: - with pytest.raises(WheelError, match=message): + with pytest.raises(SbomError, match=message): decode_digest(digest, "w") diff --git a/tests/hatch/test_hook.py b/tests/hatch/test_hook.py index 90043d9..d14ef87 100644 --- a/tests/hatch/test_hook.py +++ b/tests/hatch/test_hook.py @@ -11,7 +11,7 @@ import pytest -from python_sbom._errors import SettingsError +from python_sbom._errors import SbomError from python_sbom._validate._artifact import validate if TYPE_CHECKING: @@ -97,7 +97,7 @@ def test_project_table_options(project: Path, build_wheel: BuildWheel) -> None: def test_hook_table_refuses_python_sbom_options(project: Path, build_wheel: BuildWheel) -> None: configure(project, "default-branch = 'trunk'\n") - with pytest.raises(SettingsError, match=r"hook's table holds default-branch; move python-sbom's options"): + with pytest.raises(SbomError, match=r"hook's table holds default-branch; move python-sbom's options"): build_wheel(project) From 952da4c396d31e896cf5b2294970005a0d132cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:27:20 -0800 Subject: [PATCH 101/115] =?UTF-8?q?=F0=9F=93=9D=20docs:=20redraw=20the=20a?= =?UTF-8?q?rchitecture=20from=20the=20imports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diagram missed _errors and version, drew the hook's lazy import of _generate as a load-time one, and lacked the CLI's imports of _wheel and, for types, _model. The text still described one pipeline, the generate package docstring named backends, the contributing guide pointed at the removed _cli/_commands.py, and the threat model claimed spdx caps nesting. --- .github/THREAT_MODEL.md | 10 +++---- docs/contributing.rst | 5 ++-- docs/explanation.rst | 40 +++++++++++++++++---------- src/python_sbom/_generate/__init__.py | 2 +- 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/.github/THREAT_MODEL.md b/.github/THREAT_MODEL.md index a50cc46..474bf1d 100644 --- a/.github/THREAT_MODEL.md +++ b/.github/THREAT_MODEL.md @@ -40,11 +40,11 @@ An attacker who controls any of these already has what python-sbom could give th ## Untrusted inputs -| Input | Comes from | What python-sbom guarantees | -| --------------------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Metadata and `RECORD` of bundled wheels | Upstream projects | Read as text with capped sizes, copied as JSON strings; every member is checked against its `RECORD` row, duplicate members are refused, nothing is executed. | -| `_vendor/vendor.txt` of bundled wheels | Upstream projects | Each line must pin one exact version, or the build fails. | -| SBOM files given to `spdx`, `validate` | Whoever produced the file | Parsed as JSON only, with capped sizes and nesting; problems are reported, not raised; `spdx` writes the one file named on the command line. | +| Input | Comes from | What python-sbom guarantees | +| --------------------------------------- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Metadata and `RECORD` of bundled wheels | Upstream projects | Read as text with capped sizes, copied as JSON strings; every member is checked against its `RECORD` row, duplicate members are refused, nothing is executed. | +| `_vendor/vendor.txt` of bundled wheels | Upstream projects | Each line must pin one exact version, or the build fails. | +| SBOM files given to `spdx`, `validate` | Whoever produced the file | Parsed as JSON only, with sizes capped by the component cap; `validate` refuses nesting past 128 levels and reports problems instead of raising them; `spdx` reports input too deep to parse as an error and writes the one file named on the command line. | ## Threats and controls diff --git a/docs/contributing.rst b/docs/contributing.rst index 24a0fdb..6309c52 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -11,8 +11,9 @@ It hands them, with a content provider, to the pipeline, which builds the docume :doc:`explanation` draws the packages and their imports. To record a new field, read it at the edge into ``_model/_inputs.py`` and write it in ``_model``; to add a subcommand, -add a registration function and a handler next to the others in ``_cli/_commands.py`` and list the registration in -``_cli/_parser.py``. Tests mirror the source: ``tests//test_.py``. +write a ``run`` function in a new ``_cli/_.py`` and register its arguments and the +``"python_sbom._cli._:run"`` handler in ``_cli/_parser.py``, which loads the module only when the command runs. +Tests mirror the source: ``tests//test_.py``. Updating the schemas -------------------- diff --git a/docs/explanation.rst b/docs/explanation.rst index 8870e42..0bf9a51 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -6,12 +6,15 @@ Architecture Each concern has its own subpackage, and only the adapters know a build system or the command line. The hatchling hook and the ``embed`` and ``archive`` commands gather what the edge knows: the request (core metadata, file name, options, -build scripts), the source tree, and the build facts (time, revision, toolchain). They hand those, with a -:class:`~python_sbom.ContentProvider` and its :class:`~python_sbom.ProviderContext`, to one pipeline, and the model -turns them into the document with pure functions. +build scripts), the source tree or the archive's members, and the build facts (time, revision, toolchain). They hand +those, with a :class:`~python_sbom.ContentProvider` and its :class:`~python_sbom.ProviderContext`, to the pipeline for +their artifact. Each pipeline maps what it read to components, and one function of the model adds what every artifact +of the project shares: supplier, source links, revision, tools, dependencies and formulation. -The diagram follows the imports between the subpackages: an arrow points at what a package imports, and a dashed arrow -at what it imports only for type annotations. +The diagram follows the imports between the subpackages: a solid arrow points at what a package imports when it loads, a +dotted arrow labeled *lazy* at what it imports only once a function needs it, and a dotted arrow labeled *types* at +what it imports only for type annotations. The command line loads each command's module, and what it imports, only when +that command runs. .. mermaid:: @@ -20,20 +23,25 @@ at what it imports only for type annotations. hook[hatch
hatchling build hook] cli[_cli
embed, archive, spdx, validate] end - api[python_sbom
provider API] + api[python_sbom
provider API, __main__] generate[_generate
wheel and archive pipelines] - collect[_collect
settings, facts, source tree, provider loading] - validate[_validate
schema, references, licenses] + collect[_collect
settings, facts, revision, source tree, archives, providers] + validate[_validate
limits, schema, references, licenses] formats[_formats
CycloneDX JSON, SPDX 2.3] - wheel[_wheel
RECORD rows, archive reads and rewrite] + wheel[_wheel
RECORD rows, member reads, rewrite] model[_model
inputs, types, identity, document building] - hook --> generate + errors[_errors
SbomError] + version[version] hook --> collect - hook -.-> model + hook -. lazy .-> generate + hook -. types .-> model cli --> generate cli --> collect cli --> validate cli --> formats + cli --> wheel + cli -. types .-> model + api --> cli api --> model generate --> collect generate --> formats @@ -45,15 +53,17 @@ at what it imports only for type annotations. validate --> wheel validate --> model formats --> model + formats --> version wheel --> model + model --> errors classDef edge fill:#cfe4ff,stroke:#1f6feb,color:#0b1f3a; classDef core fill:#d8f3dc,stroke:#2d6a4f,color:#081c15; class hook,cli,collect,api edge; - class generate,validate,formats,wheel,model core; + class generate,validate,formats,wheel,model,errors,version core; -The model imports nothing else of the package but its errors, and only the hook imports hatchling, so the provider API -and the command line work without it. Every package raises subclasses of :class:`~python_sbom.SbomError`, which the -command line reports without a traceback. +Every package raises :class:`~python_sbom.SbomError`, which the command line reports without a traceback; the diagram +draws only the model's import of it, since the model imports nothing else of the package. Only the hook imports +hatchling, so the provider API and the command line work without it. The one extension point is the :class:`~python_sbom.ContentProvider`, which every entry point loads from the ``content-provider`` option. A hatchling custom hook can override diff --git a/src/python_sbom/_generate/__init__.py b/src/python_sbom/_generate/__init__.py index 8f1ddab..49187f2 100644 --- a/src/python_sbom/_generate/__init__.py +++ b/src/python_sbom/_generate/__init__.py @@ -1 +1 @@ -"""The build pipeline: backends that adapt a build system, the content seam, and assembling the SBOM.""" +"""The pipelines that build a wheel's and an archive's SBOM from what _collect gathers, and write it.""" From e1919d4c692dc3bd274717ac698a8d6d786f308b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:29:29 -0800 Subject: [PATCH 102/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20pass=20the=20docs?= =?UTF-8?q?=20and=20coverage=20checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit proselint read the diagram's model --> errors after wheel --> model as a repeated word, and covdefaults excludes only the bare TYPE_CHECKING guard, so the API test's typing.TYPE_CHECKING import counted as uncovered. --- docs/explanation.rst | 2 +- tests/test_public_api.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/explanation.rst b/docs/explanation.rst index 0bf9a51..36b1be4 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -54,8 +54,8 @@ that command runs. validate --> model formats --> model formats --> version - wheel --> model model --> errors + wheel --> model classDef edge fill:#cfe4ff,stroke:#1f6feb,color:#0b1f3a; classDef core fill:#d8f3dc,stroke:#2d6a4f,color:#081c15; class hook,cli,collect,api edge; diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 02281c7..8286e1c 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -7,7 +7,7 @@ import pathlib import pkgutil import typing -from typing import Final +from typing import TYPE_CHECKING, Final import pytest @@ -17,7 +17,7 @@ from python_sbom._model._inputs import RecordRow from python_sbom._model._metadata import CoreMetadata -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from collections.abc import Callable, Iterator _Hint = object # simplint: ignore[SMP602] # hints are arbitrary typing objects From 8112d47004f112d38eb24bd493d98ed2519f461f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:32:54 -0800 Subject: [PATCH 103/115] =?UTF-8?q?=F0=9F=90=9B=20fix:=20normalize=20every?= =?UTF-8?q?=20pypi=20purl=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packageurl only lowercases a pypi name and turns underscores into dashes, so jaraco.context and jaraco_context got two purls, and a build tool's bom-ref, built from its normalized name, disagreed with its own purl (tool:pkg:pypi/boolean-py vs pkg:pypi/boolean.py). Names now go through canonicalize_name first, as before the switch to packageurl. The pinned SBOM changes only in the project's purl and the serial derived from it. --- src/python_sbom/_model/_names.py | 4 +++- tests/_generate/pinned/embed-members.json | 4 ++-- tests/_generate/pinned/embed.cdx.json | 12 ++++++------ tests/_model/test_components.py | 24 ++++++++++++++++++++--- tests/_model/test_tools.py | 9 +++++++++ 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/python_sbom/_model/_names.py b/src/python_sbom/_model/_names.py index 81c4374..fdb33bc 100644 --- a/src/python_sbom/_model/_names.py +++ b/src/python_sbom/_model/_names.py @@ -53,7 +53,9 @@ def property_prefix(name: str) -> str: def purl(name: str, version: str | None = None) -> str: - return PackageURL(type="pypi", name=name, version=version).to_string() + # normalized as PEP 503 does, so every spelling of a project, jaraco.context or jaraco_context, gets one purl and a + # component's bom-ref agrees with its purl; packageurl only lowercases and turns underscores into dashes + return PackageURL(type="pypi", name=canonicalize_name(name), version=version).to_string() def pypi_url(name: str, version: str | None = None) -> str: diff --git a/tests/_generate/pinned/embed-members.json b/tests/_generate/pinned/embed-members.json index 1ebd6d2..20926ad 100644 --- a/tests/_generate/pinned/embed-members.json +++ b/tests/_generate/pinned/embed-members.json @@ -81,7 +81,7 @@ ], 8, 2175008768, - "be78512aff2db98c3570bf08ada840be7ad95ab0035b962c4595b31059383a55" + "f4a42ed807c0f5cca7212382f6c68426e945baff4ca5f6ee4c15e0cba3081802" ], [ "pkg_name-1.0.dist-info/RECORD", @@ -95,6 +95,6 @@ ], 0, 25165824, - "ab2b152f6aa0db7eda9723b860ff7b3ff3fa06598fa0bf4304d636e66f6219dc" + "5ff2e7e7932db80e36379c90b7f157c7d17b4c912d9d98ce7876cc151fa8b58c" ] ] diff --git a/tests/_generate/pinned/embed.cdx.json b/tests/_generate/pinned/embed.cdx.json index e4ea1fc..b4d5fd1 100644 --- a/tests/_generate/pinned/embed.cdx.json +++ b/tests/_generate/pinned/embed.cdx.json @@ -2,7 +2,7 @@ "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", "bomFormat": "CycloneDX", "specVersion": "1.6", - "serialNumber": "urn:uuid:0c382715-4ac3-55f8-b2e5-bb3920847bd3", + "serialNumber": "urn:uuid:cc128def-101f-5a39-891f-b23b5e129f55", "version": 1, "metadata": { "timestamp": "2023-11-14T22:13:20+00:00", @@ -115,10 +115,10 @@ }, "component": { "type": "application", - "bom-ref": "pkg:pypi/pkg.name@1.0", + "bom-ref": "pkg:pypi/pkg-name@1.0", "name": "Pkg.Name", "version": "1.0", - "purl": "pkg:pypi/pkg.name@1.0", + "purl": "pkg:pypi/pkg-name@1.0", "description": "a pinned example", "authors": [ { @@ -495,7 +495,7 @@ ], "dependencies": [ { - "ref": "pkg:pypi/pkg.name@1.0", + "ref": "pkg:pypi/pkg-name@1.0", "dependsOn": [ "bundled:pkg_name/wheels/pip-1.0-py3-none-any.whl", "requires-dist:alpha>=1", @@ -529,7 +529,7 @@ { "aggregate": "incomplete", "dependencies": [ - "pkg:pypi/pkg.name@1.0" + "pkg:pypi/pkg-name@1.0" ] } ], @@ -562,7 +562,7 @@ { "type": "artifact", "resource": { - "ref": "pkg:pypi/pkg.name@1.0" + "ref": "pkg:pypi/pkg-name@1.0" } } ], diff --git a/tests/_model/test_components.py b/tests/_model/test_components.py index df86209..16b8e88 100644 --- a/tests/_model/test_components.py +++ b/tests/_model/test_components.py @@ -13,6 +13,7 @@ ) from python_sbom._model._inputs import BundledWheel, RecordRow from python_sbom._model._metadata import parse_metadata +from python_sbom._model._names import purl if TYPE_CHECKING: from python_sbom import Component, Property @@ -343,9 +344,26 @@ def test_vendored_spellings_of_one_package_merge() -> None: assert [child["name"] for child in component.get("components", [])] == ["jaraco_text"] -def test_pypi_purl_keeps_dots() -> None: - component = component_from_metadata(parse_metadata("Name: zope.interface\nVersion: 7.0\n")) - assert component["purl"] == "pkg:pypi/zope.interface@7.0" +@pytest.mark.parametrize( + "name", + [ + pytest.param("jaraco.context", id="dots"), + pytest.param("jaraco_context", id="underscore"), + pytest.param("Jaraco-Context", id="case"), + pytest.param("jaraco..context", id="run-of-separators"), + ], +) +def test_spellings_of_one_project_share_a_purl(name: str) -> None: + component = component_from_metadata(parse_metadata(f"Name: {name}\nVersion: 6.1.0\n")) + assert (component["purl"], component["bom-ref"]) == ( + "pkg:pypi/jaraco-context@6.1.0", + "pkg:pypi/jaraco-context@6.1.0", + ) + + +def test_purl_escapes_a_local_version() -> None: + # the purl spec percent-encodes every version character outside its unreserved set, + among them + assert purl("pkg", "1.0+local.1") == "pkg:pypi/pkg@1.0%2Blocal.1" def test_vendored_dependencies_at_any_depth_in_document_order() -> None: diff --git a/tests/_model/test_tools.py b/tests/_model/test_tools.py index a7c671e..cc3e859 100644 --- a/tests/_model/test_tools.py +++ b/tests/_model/test_tools.py @@ -55,3 +55,12 @@ def test_tools_list_interpreter_then_distributions_by_name(facts: BuildFacts) -> {"ref": "tool:pkg:pypi/alpha@1", "dependsOn": ["tool:pkg:pypi/zeta@2"]}, {"ref": "tool:pkg:pypi/zeta@2", "dependsOn": []}, ] + + +def test_tool_bom_ref_and_purl_agree(facts: BuildFacts) -> None: + # the distribution is keyed by its normalized name, while its metadata spells it with a dot + tool = InstalledDistribution( + name="boolean-py", metadata=parse_metadata("Name: boolean.py\nVersion: 5.0\n"), files=[], depends_on=[] + ) + (_, component), _ = tool_components(dataclasses.replace(facts, distributions=[tool])) + assert (component["bom-ref"], component.get("purl")) == ("tool:pkg:pypi/boolean-py@5.0", "pkg:pypi/boolean-py@5.0") From 9ea289b7561f88b2e35e86bd7381943a0e0eb29c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:35:34 -0800 Subject: [PATCH 104/115] =?UTF-8?q?=E2=9A=A1=20perf:=20load=20the=20licens?= =?UTF-8?q?e=20parser=20only=20for=20a=20policy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit license-expression and its SPDX index took 9-10 ms of every cold validate, though only --allow-license reads them. --- src/python_sbom/_validate/_document.py | 6 +++++- tests/_validate/test_schemas.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 237d79e..42e4deb 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -78,7 +78,6 @@ def _problems(raw: bytes, allowed_licenses: frozenset[str] | None, max_component return census # find_spec sees only the top-level modules, not a dependency of theirs that is missing or broken try: - from ._licenses import license_problems # ruff: ignore[import-outside-top-level] # the optional extra, see above from ._schemas import schema_error, spdx_problems # ruff: ignore[import-outside-top-level] # the optional extra, see above except ImportError as error: raise MissingExtraError(missing_extra) from error @@ -102,6 +101,11 @@ def _problems(raw: bytes, allowed_licenses: frozenset[str] | None, max_component # the SPDX rendering of a document whose references are broken only repeats those problems problems: Final[list[str]] = _reference_problems(loaded) or spdx_problems(render_spdx(loaded)) if allowed_licenses is not None: + # the license parser loads only for a policy to judge against, as most checks have none + try: + from ._licenses import license_problems # ruff: ignore[import-outside-top-level] # see above + except ImportError as error: + raise MissingExtraError(missing_extra) from error problems.extend(license_problems(loaded["components"], allowed_licenses)) return problems diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index 8a026ff..25b9919 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -10,6 +10,7 @@ from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document from python_sbom._errors import MissingExtraError +from python_sbom._formats._json import dumps from python_sbom._formats._spdx import render_spdx from python_sbom._validate._document import validate_document from python_sbom._validate._schemas import schema_error, spdx_problems @@ -81,6 +82,15 @@ def test_validate_with_broken_extra(monkeypatch: pytest.MonkeyPatch) -> None: assert isinstance(raised.value.__cause__, ImportError) +def test_license_policy_with_broken_extra(describe: Describe, monkeypatch: pytest.MonkeyPatch) -> None: + # the license parser loads only for a policy, and a broken one reports the extra as missing there too + document = dumps(describe()) + monkeypatch.setitem(sys.modules, "python_sbom._validate._licenses", None) + assert validate_document(document, None) == [] + with pytest.raises(MissingExtraError, match=r"pip install 'python-sbom\[validate\]'"): + validate_document(document, frozenset({"MIT"})) + + def test_schema_error_names_the_path() -> None: document: Json = {"bomFormat": "CycloneDX", "specVersion": "1.6", "version": "one"} assert schema_error(document) == '"one" is not of type "integer" at $.version' From 84412c7454603bd896739e2ff00e16bfa13840f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:36:08 -0800 Subject: [PATCH 105/115] =?UTF-8?q?=E2=9A=A1=20perf:=20stack=20only=20cont?= =?UTF-8?q?ainers=20in=20the=20census?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The census pushed every value of the document onto its stack. Only lists and objects go there now; numbers are judged as their container is read, and the depth check moves to the container, so a value still nests too deep exactly when it did. The census of virtualenv's zipapp SBOM drops from about 12 to 8 ms. --- src/python_sbom/_validate/_document.py | 46 ++++++++++++++++---------- tests/_validate/test_document.py | 4 +++ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 42e4deb..169a6b3 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -15,6 +15,7 @@ from ._limits import MAX_COMPONENTS, byte_limit if TYPE_CHECKING: + from collections.abc import Iterable from typing import TypeIs # new in 3.13; annotations here are never evaluated from python_sbom._model._types import Component, Document @@ -116,33 +117,44 @@ def _census_problems(document: Json, max_components: int) -> list[str]: # components array, and one array may hold them all, so only the total bounds its work components = 0 refs: Final[Counter[str]] = Counter() - pending: Final[list[tuple[Json, int]]] = [(document, 1)] + # only lists and objects go on the stack, every other value is judged where its container is read, so a scalar + # nests too deep when its container already sits at the limit; the document starts out as the one item of a list + pending: Final[list[tuple[list[Json] | dict[str, Json], int]]] = [([document], 0)] while pending: node, depth = pending.pop() - if depth > _MAX_DEPTH: - return [f"nests more than {_MAX_DEPTH} levels deep, too deep to check"] - if isinstance(node, list): - pending.extend((item, depth + 1) for item in node) - elif isinstance(node, dict): + if isinstance(node, dict): if isinstance(ref := node.get("bom-ref"), str): refs[ref] += 1 - if isinstance(listed := node.get("components"), list): - components += len(listed) - pending.extend((item, depth + 1) for item in node.values()) - elif isinstance(node, float) and not math.isfinite(node): - # json.loads takes NaN and Infinity, which RFC 8259 leaves out of JSON and the schema check reads as null - return [f"holds {node}, which is not a JSON number"] + components += len(listed) if isinstance(listed := node.get("components"), list) else 0 if components > max_components: - problem = ( - f"lists more than {max_components} components, nested ones included;" - " pass a higher --max-components to check it" - ) - return [problem] + return [ + ( + f"lists more than {max_components} components, nested ones included;" + " pass a higher --max-components to check it" + ) + ] + children = node.values() if isinstance(node, dict) else node + if children and depth >= _MAX_DEPTH: + return [f"nests more than {_MAX_DEPTH} levels deep, too deep to check"] + if problem := _push_containers(children, depth + 1, pending): + return problem if duplicates := sorted(ref for ref, count in refs.items() if count > 1): return [f"bom-ref values are not unique: {duplicates}"] return [] +def _push_containers( + children: Iterable[Json], depth: int, pending: list[tuple[list[Json] | dict[str, Json], int]] +) -> list[str]: + for child in children: + if isinstance(child, (list, dict)): + pending.append((child, depth)) + elif isinstance(child, float) and not math.isfinite(child): + # json.loads takes NaN and Infinity, which RFC 8259 leaves out of JSON and the schema reads as null + return [f"holds {child}, which is not a JSON number"] + return [] + + def _has(value: Json, path: tuple[str, ...]) -> bool: for key in path: if not isinstance(value, dict) or key not in value: diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py index eb302e3..fb0f153 100644 --- a/tests/_validate/test_document.py +++ b/tests/_validate/test_document.py @@ -289,6 +289,10 @@ def test_non_finite_numbers_are_rejected(constant: str) -> None: assert validate_document(raw, None) == [f"holds {float(constant)}, which is not a JSON number"] +def test_non_finite_document_is_rejected() -> None: + assert validate_document(b"NaN", None) == ["holds nan, which is not a JSON number"] + + def test_license_policy(describe: Describe) -> None: raw = with_components(describe(), shipped([{"expression": "MIT AND GPL-3.0-only"}])) assert validate_document(raw, frozenset({"MIT"})) == [ From 1425769e5cde299f87488083d6d64144ff44bfb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:39:55 -0800 Subject: [PATCH 106/115] =?UTF-8?q?=E2=9A=A1=20perf:=20read=20build-env=20?= =?UTF-8?q?WHEEL=20and=20requirements=20once?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each distribution's WHEEL went through the email parser for one field, and each Requires-Dist line was parsed and its marker evaluated again for every distribution declaring it. Root-Is-Purelib is now read from the header lines directly, agreeing with the email parser on 104,668 WHEEL files, and a requirement's verdict is cached for the process. --- src/python_sbom/_collect/_facts.py | 48 +++++++++++++++++++++--------- tests/_collect/test_facts.py | 17 +++++++++++ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/python_sbom/_collect/_facts.py b/src/python_sbom/_collect/_facts.py index 9759294..89d6823 100644 --- a/src/python_sbom/_collect/_facts.py +++ b/src/python_sbom/_collect/_facts.py @@ -6,11 +6,12 @@ import sys import time import warnings +from functools import cache from importlib.metadata import distributions +from itertools import takewhile from typing import TYPE_CHECKING, Final from installer.records import InvalidRecordEntry, parse_record_file -from installer.utils import parse_metadata_file from packaging.direct_url import DirectUrl, DirectUrlValidationError from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name @@ -76,9 +77,19 @@ def _installed(path: Sequence[str] | None) -> list[InstalledDistribution]: def _recorded_files(distribution: Distribution) -> list[RecordRow]: # a platform wheel's files are built for this machine's OS and architecture, and an editable install's point at a # checkout, so listing either would tie the document to one machine - if parse_metadata_file(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] != "true" or _editable( - distribution - ): + # Root-Is-Purelib as the email parser takes a field, its first occurrence among the header lines, as written after + # the colon, without parsing the rest of WHEEL + headers = takewhile(bool, (distribution.read_text("WHEEL") or "").splitlines()) + fields = (line.partition(":") for line in headers) + purelib = next( + ( + value.lstrip(" \t") == "true" + for field, colon, value in fields + if colon and field.lower() == "root-is-purelib" + ), + False, + ) + if not purelib or _editable(distribution): return [] # RECORD read once and as written: Distribution.files would stat every row to drop files missing on disk, and build # a path object for each, while the SBOM lists what the install recorded @@ -117,19 +128,28 @@ def _depends_on(name: str, distribution: Distribution, metadata: CoreMetadata, i names: Final[set[str]] = set() # an .egg-info states its requirements in requires.txt, which Distribution.requires reads for line in metadata.values("requires_dist") or distribution.requires or []: - try: - requirement = Requirement(line) - except InvalidRequirement as error: - # old distributions declare requirements, such as "pytz (>dev)", that today's syntax rejects - warnings.warn(f"left out the requirement {line!r} of {name}: {error}", stacklevel=2) - continue - if (required := canonicalize_name(requirement.name)) in installed and ( - requirement.marker is None or requirement.marker.evaluate() - ): - names.add(required) + match _required(line): + case InvalidRequirement() as error: + # old distributions declare requirements, such as "pytz (>dev)", that today's syntax rejects + warnings.warn(f"left out the requirement {line!r} of {name}: {error}", stacklevel=2) + case str() as required if required in installed: + names.add(required) return sorted(names) +@cache +def _required(line: str) -> str | InvalidRequirement | None: + # the normalized name a requirement needs here, None when its marker rules it out, or why it does not parse; the + # same requirements recur across a build environment's distributions, and the verdict holds for the process + try: + requirement = Requirement(line) + except InvalidRequirement as error: + return error + if requirement.marker is not None and not requirement.marker.evaluate(): + return None + return canonicalize_name(requirement.name) + + __all__ = [ "collect_facts", ] diff --git a/tests/_collect/test_facts.py b/tests/_collect/test_facts.py index 149fe7a..ac54dc6 100644 --- a/tests/_collect/test_facts.py +++ b/tests/_collect/test_facts.py @@ -122,6 +122,23 @@ def test_platform_distribution_lists_no_files(tmp_path: Path, install: Install) assert facts_from(tmp_path).distributions[0].files == [] +@pytest.mark.parametrize( + ("wheel", "listed"), + [ + pytest.param("Wheel-Version: 1.0\nroot-is-purelib:\ttrue\n", True, id="any-case-and-blank"), + pytest.param("Wheel-Version: 1.0\nRoot-Is-Purelib: true\nRoot-Is-Purelib: false\n", True, id="first-wins"), + pytest.param("Wheel-Version: 1.0\n", False, id="absent"), + pytest.param("Wheel-Version: 1.0\n\nRoot-Is-Purelib: true\n", False, id="after-the-headers"), + pytest.param("Root-Is-Purelib: true \n", False, id="trailing-space"), + ], +) +def test_wheel_file_decides_purelib(tmp_path: Path, install: Install, wheel: str, listed: bool) -> None: + # read as the email parser reads the field: its first occurrence, among the headers, as written after the colon + install(tmp_path, "Name: alpha\nVersion: 1.0\n", {}) + (tmp_path / "alpha-1.0.dist-info" / "WHEEL").write_text(wheel, encoding="utf-8") + assert bool(facts_from(tmp_path).distributions[0].files) is listed + + def test_editable_distribution_lists_no_files(tmp_path: Path, install: Install) -> None: editable = '{"url": "file:///checkout", "dir_info": {"editable": true}}' install(tmp_path, "Name: local\nVersion: 1.0\n", {"local.pth": b"/checkout/src\n"}, direct_url=editable) From f189bf1b4173b31f0baa92575e6ca6f98fe43150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:40:53 -0800 Subject: [PATCH 107/115] =?UTF-8?q?=E2=9C=A8=20feat:=20deflate=20the=20SBO?= =?UTF-8?q?M=20archive=20--embed=20adds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SBOM followed the compression most of the archive's members used, so virtualenv's zipapp, which stores all of them, took it stored, megabytes of JSON. It now always goes in deflated at level 9, with the fixed time and attributes as before, and the other members keep their bytes. --- docs/changelog.d/+python-sbom.feature.rst | 7 ++-- docs/extend.rst | 8 ++-- src/python_sbom/_wheel/_archive.py | 18 ++------- tests/_cli/test_archive.py | 38 +----------------- tests/_wheel/test_archive.py | 47 ++++++++++++----------- 5 files changed, 37 insertions(+), 81 deletions(-) diff --git a/docs/changelog.d/+python-sbom.feature.rst b/docs/changelog.d/+python-sbom.feature.rst index d5e6604..35eac74 100644 --- a/docs/changelog.d/+python-sbom.feature.rst +++ b/docs/changelog.d/+python-sbom.feature.rst @@ -1,5 +1,6 @@ Add the ``python-sbom`` hatchling build hook, which writes a CycloneDX 1.6 SBOM into each wheel, the Python API behind it, and the ``python-sbom`` command: ``embed`` writes the same SBOM into wheels other build backends build, ``archive`` -describes another zip archive a project publishes, such as a zipapp, ``spdx`` renders an SBOM as SPDX, and ``validate`` -checks the SBOMs in wheels and archives against the official CycloneDX schema files python-sbom ships, with -jsonschema-rs, and rejects one listing more than 100,000 components unless ``--max-components`` allows more. +describes another zip archive a project publishes, such as a zipapp, and with ``--embed`` adds that SBOM to it, +deflated, ``spdx`` renders an SBOM as SPDX, and ``validate`` checks the SBOMs in wheels and archives against the +official CycloneDX schema files python-sbom ships, with jsonschema-rs, and rejects one listing more than 100,000 +components unless ``--max-components`` allows more. diff --git a/docs/extend.rst b/docs/extend.rst index a6c8d66..9ec9dcf 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -107,7 +107,7 @@ files: The document takes its options, source revision and toolchain the way the wheel's does, so both name the project alike. Its build time comes from ``SOURCE_DATE_EPOCH``, or the current time without it; set ``SOURCE_DATE_EPOCH`` for both builds to give them one time. ``--embed`` adds the document to the archive's top level, where ``python-sbom validate -virtualenv.pyz`` finds it. It goes in compressed the way most of the archive's members are: by their compression method -and, for deflate, at the level their option bits name, which Python's zipfile leaves at zlib's default. The entry -carries a fixed time and Unix attributes, so adding it to one archive gives the same bytes on every system with the same -zlib. +virtualenv.pyz`` finds it. It goes in deflated at level 9 whatever the archive's other members use, since a zipapp may +store them all and the SBOM, megabytes of JSON, would then add its full size. The entry carries a fixed time and Unix +attributes and the other members keep their bytes, so adding it to one archive gives the same bytes on every system with +the same zlib. diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index c298b50..e96346b 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -31,9 +31,6 @@ _CHUNK: Final[int] = 1024 * 1024 # 1980-01-01, the earliest time a zip entry holds _ZIP_EPOCH: Final[int] = 315532800 -# the zlib level each setting of the deflate option bits in a member's flags stands for (APPNOTE 4.4.4: normal, maximum, -# fast, super fast), as Info-ZIP sets them; None is zlib's default -_DEFLATE_LEVELS: Final[dict[int, int | None]] = {0b000: None, 0b010: 9, 0b100: 2, 0b110: 1} def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: @@ -164,18 +161,9 @@ def append_member(archive: Path, name: str, content: bytes, epoch: int) -> None: if name in handle.namelist(): msg = f"{archive.name} already holds {name}" raise SbomError(msg) - # compressed by the method most of the archive's files use, at the level their deflate option bits name, since - # the zip format records no level itself and Python's zipfile leaves the bits at normal, zlib's default; - # directories and empty files count for none, as zip tools store them whatever they compress - files: Final[list[zipfile.ZipInfo]] = [ - info for info in handle.infolist() if info.file_size and not info.is_dir() - ] - method: Final[int] = next( - iter(Counter(info.compress_type for info in files).most_common(1)), (zipfile.ZIP_STORED, 0) - )[0] - flags: Final[int] = next((info.flag_bits for info in files if info.compress_type == method), 0) - level: Final[int | None] = _DEFLATE_LEVELS[flags & 0b110] if method == zipfile.ZIP_DEFLATED else None - handle.writestr(_entry(name, epoch), content, compress_type=method, compresslevel=level) + # deflated at the highest level whatever the other members use, since a zipapp may store them all and the SBOM + # then adds megabytes; a fixed level, time and attributes keep the bytes the same for one input on every system + handle.writestr(_entry(name, epoch), content, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) __all__ = [ diff --git a/tests/_cli/test_archive.py b/tests/_cli/test_archive.py index 114ab9f..ce2eb12 100644 --- a/tests/_cli/test_archive.py +++ b/tests/_cli/test_archive.py @@ -2,9 +2,7 @@ import json import stat -import struct import zipfile -import zlib from typing import TYPE_CHECKING import pytest @@ -54,45 +52,11 @@ def test_archive_embeds_sbom(project: Path, zipapp: Path, tmp_path: Path) -> Non assert (entry.create_system, entry.external_attr, entry.compress_type, zipfile.ZipFile(zipapp).read(entry)) == ( 3, (stat.S_IFREG | 0o644) << 16, - zipfile.ZIP_STORED, + zipfile.ZIP_DEFLATED, (tmp_path / "out.json").read_bytes(), ) -@pytest.mark.parametrize( - ("flags", "level"), - [ - pytest.param(0b000, zlib.Z_DEFAULT_COMPRESSION, id="normal"), - pytest.param(0b010, 9, id="maximum"), - pytest.param(0b100, 2, id="fast"), - pytest.param(0b110, 1, id="super-fast"), - ], -) -def test_archive_embeds_sbom_compressed_as_its_members(project: Path, flagged_app: Path, level: int) -> None: - main(["archive", str(flagged_app), "--root", str(project), "--build-env", str(project), "--embed"]) - with zipfile.ZipFile(flagged_app) as handle: - entry = handle.getinfo("app.pyz.cdx.json") - content = handle.read(entry) - data = flagged_app.read_bytes() - name_length, extra_length = struct.unpack(" Path: - with zipfile.ZipFile(app := tmp_path / "app.pyz", "w", zipfile.ZIP_DEFLATED) as handle: - handle.writestr("__main__.py", "import app\n") - handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") - # zipfile clears the option bits it writes, so they go straight into the central directory, where readers take them - data = bytearray(app.read_bytes()) - for at in (index for index in range(len(data)) if data[index : index + 4] == b"PK\x01\x02"): - struct.pack_into(" None: arguments = ["archive", str(zipapp), "--root", str(project), "--build-env", str(project), "--embed"] main(arguments) diff --git a/tests/_wheel/test_archive.py b/tests/_wheel/test_archive.py index c98d371..ad55254 100644 --- a/tests/_wheel/test_archive.py +++ b/tests/_wheel/test_archive.py @@ -5,6 +5,7 @@ import sys import tracemalloc import zipfile +import zlib from contextlib import ExitStack from io import BytesIO from pathlib import Path @@ -281,29 +282,31 @@ def test_embed_ends_an_unterminated_last_row(tmp_path: Path, write_wheel: WriteW ] -@pytest.mark.parametrize( - ("members", "method"), - [ - pytest.param([("a.py", b"x" * 100, zipfile.ZIP_DEFLATED)], zipfile.ZIP_DEFLATED, id="deflated"), - pytest.param([("a.py", b"x" * 100, zipfile.ZIP_STORED)], zipfile.ZIP_STORED, id="stored"), - # Info-ZIP stores directories and empty files whatever it compresses, so they outvote no one - pytest.param( - [("d/", b"", zipfile.ZIP_STORED), ("e", b"", zipfile.ZIP_STORED), ("a.py", b"x", zipfile.ZIP_DEFLATED)], - zipfile.ZIP_DEFLATED, - id="directories-and-empty-files-skipped", - ), - pytest.param([], zipfile.ZIP_STORED, id="empty-archive"), - ], -) -def test_append_member_compresses_as_the_files_are( - tmp_path: Path, members: list[tuple[str, bytes, int]], method: int -) -> None: - with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w") as handle: - for name, content, compression in members: - handle.writestr(name, content, compression) - append_member(archive, "app.pyz.cdx.json", b"{}\n", 1_700_000_000) +def test_append_member_deflates_at_level_nine(tmp_path: Path) -> None: + # the archive stores its members, as virtualenv's zipapp does, yet the SBOM goes in deflated, and the members keep + # their bytes + with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w", zipfile.ZIP_STORED) as handle: + handle.writestr("__main__.py", b"import app\n" * 100) + before = _stored(archive, "__main__.py") + content = b'{"components": []}\n' * 100 + append_member(archive, "app.pyz.cdx.json", content, 1_700_000_000) entry = zipfile.ZipFile(archive).getinfo("app.pyz.cdx.json") - assert (entry.compress_type, entry.create_system, entry.date_time) == (method, 3, (2023, 11, 14, 22, 13, 20)) + compressor = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS) + assert ( + entry.compress_type, + entry.create_system, + entry.date_time, + entry.external_attr, + _stored(archive, "app.pyz.cdx.json"), + _stored(archive, "__main__.py"), + ) == ( + zipfile.ZIP_DEFLATED, + 3, + (2023, 11, 14, 22, 13, 20), + (stat.S_IFREG | 0o644) << 16, + compressor.compress(content) + compressor.flush(), + before, + ) def test_append_member_refuses_a_second_copy(tmp_path: Path) -> None: From 9b29173ea01c08abdb7a721a1e4a46da3c2b76d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 08:53:16 -0800 Subject: [PATCH 108/115] =?UTF-8?q?=F0=9F=A7=AA=20test(spdx):=20nest=20dee?= =?UTF-8?q?p=20enough=20for=20every=20stack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/_cli/test_spdx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_cli/test_spdx.py b/tests/_cli/test_spdx.py index 8207a85..ecfa172 100644 --- a/tests/_cli/test_spdx.py +++ b/tests/_cli/test_spdx.py @@ -45,7 +45,7 @@ def test_spdx_refuses_malformed_document(tmp_path: Path, capsys: pytest.CaptureF b"{", "is not JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)", id="json" ), pytest.param(b'{"name": "\xff"}', "is not UTF-8: invalid start byte at byte 10", id="utf-8"), - pytest.param(b"[" * 100_000, "nests too deep for the JSON parser to read", id="nesting"), + pytest.param(b"[" * 1_000_000 + b"]" * 1_000_000, "nests too deep for the JSON parser to read", id="nesting"), ], ) def test_spdx_refuses_unreadable_source( From aab7c01eaae36de00e7411b8e74bffb667848713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 09:08:15 -0800 Subject: [PATCH 109/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20give?= =?UTF-8?q?=20archive=20the=20wheel's=20layering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The archive command now only parses options and prints, like embed; the orchestrator reads, builds and writes, and the archive's document is built by a pure model function as the wheel's is. The pipeline module folds into embed, its only user. --- src/python_sbom/_cli/_archive.py | 24 ++--- src/python_sbom/_generate/_archive.py | 123 ++++++++----------------- src/python_sbom/_generate/_embed.py | 38 +++++++- src/python_sbom/_generate/_pipeline.py | 46 --------- src/python_sbom/_model/_archive.py | 108 ++++++++++++++++++++++ tests/_generate/test_embed.py | 24 ++++- tests/_generate/test_pipeline.py | 30 ------ tests/conftest.py | 2 +- 8 files changed, 214 insertions(+), 181 deletions(-) delete mode 100644 src/python_sbom/_generate/_pipeline.py create mode 100644 src/python_sbom/_model/_archive.py delete mode 100644 tests/_generate/test_pipeline.py diff --git a/src/python_sbom/_cli/_archive.py b/src/python_sbom/_cli/_archive.py index 7a4ab84..a99bb91 100644 --- a/src/python_sbom/_cli/_archive.py +++ b/src/python_sbom/_cli/_archive.py @@ -4,11 +4,7 @@ from pathlib import Path, PurePosixPath from typing import Protocol -from python_sbom._collect._facts import collect_facts -from python_sbom._collect._revision import revision_reader -from python_sbom._formats._json import dumps -from python_sbom._generate._archive import describe_archive -from python_sbom._wheel._archive import append_member +from python_sbom._generate._archive import archive_sbom from ._parser import BuildOptions @@ -21,21 +17,19 @@ class _Options(BuildOptions, Protocol): def run(options: _Options) -> bool: - root = Path.cwd() if options.root is None else options.root - facts = collect_facts(root, revision_reader(options.commit), path=options.build_env) - document = describe_archive( - root, + target, embedded = archive_sbom( + Path.cwd() if options.root is None else options.root, options.archive, - facts=facts, provider=options.content_provider, + commit=options.commit, + build_env=options.build_env, build_scripts=tuple(PurePosixPath(script) for script in options.build_scripts or []), + output=options.output, + embed=options.embed, ) - name = f"{options.archive.name}.cdx.json" - (target := options.output or options.archive.with_name(name)).write_bytes(content := dumps(document)) sys.stdout.write(f"wrote {target}\n") - if options.embed: - append_member(options.archive, name, content, facts.epoch) - sys.stdout.write(f"wrote {name} into {options.archive.name}\n") + if embedded is not None: + sys.stdout.write(f"wrote {embedded} into {options.archive.name}\n") return True diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index 2925169..8da413a 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -1,36 +1,51 @@ from __future__ import annotations import hashlib -from dataclasses import dataclass -from pathlib import PurePosixPath -from typing import TYPE_CHECKING, Final - -from packageurl import PackageURL +from typing import TYPE_CHECKING from python_sbom._collect._archive import member_reader, read_archive from python_sbom._collect._content import resolve_provider +from python_sbom._collect._facts import collect_facts +from python_sbom._collect._revision import revision_reader from python_sbom._collect._settings import settings_from_pyproject from python_sbom._collect._source import copyright_notice +from python_sbom._formats._json import dumps +from python_sbom._model._archive import archive_artifact, bundled_paths from python_sbom._model._bom import assemble -from python_sbom._model._components import ( - bundled_wheel_component, - component_from_metadata, - file_component, - vendored_dependencies, -) from python_sbom._model._content import ProviderContext from python_sbom._model._document import checked_contributions from python_sbom._model._identity import project_identity -from python_sbom._model._inputs import Artifact, BuildScript +from python_sbom._model._inputs import BuildScript +from python_sbom._wheel._archive import append_member if TYPE_CHECKING: from collections.abc import Sequence - from pathlib import Path + from pathlib import Path, PurePosixPath from python_sbom._model._content import ContentProvider - from python_sbom._model._identity import ProjectIdentity - from python_sbom._model._inputs import ArchiveTree, BuildFacts - from python_sbom._model._types import Component, Document + from python_sbom._model._inputs import BuildFacts + from python_sbom._model._types import Document + + +def archive_sbom( # ruff: ignore[too-many-arguments] # every command-line option reaches the build + root: Path, + archive: Path, + *, + provider: ContentProvider | str | None, + commit: str | None, + build_env: Sequence[str] | None, + build_scripts: Sequence[PurePosixPath], + output: Path | None, + embed: bool, +) -> tuple[Path, str | None]: + facts = collect_facts(root, revision_reader(commit), path=build_env) + document = describe_archive(root, archive, facts=facts, provider=provider, build_scripts=build_scripts) + name = f"{archive.name}.cdx.json" + (target := output or archive.with_name(name)).write_bytes(content := dumps(document)) + if not embed: + return target, None + append_member(archive, name, content, facts.epoch) + return target, name def describe_archive( @@ -41,86 +56,26 @@ def describe_archive( provider: ContentProvider | str | None, build_scripts: Sequence[PurePosixPath], ) -> Document: - # the archive's top-level .dist-info names the project; every other .dist-info marks a wheel bundled unpacked into - # its parent directory and every .whl member a bundled wheel; other files belong to the deepest distribution that - # holds them, or, at the top level, to the archive settings = settings_from_pyproject(root) content, provider_scripts = resolve_provider(provider, settings, root) tree = read_archive(archive) project = project_identity(tree.metadata[""], settings) - reader = _Reader( - tree=tree, + context = ProviderContext(root=root, settings=settings, identity=project, read=member_reader(archive)) + artifact = archive_artifact( + archive.name, + tree, project=project, - content=content, - context=ProviderContext(root=root, settings=settings, identity=project, read=member_reader(archive)), - ) - version: Final[str] = tree.metadata[""].version - ref: Final[str] = PackageURL(type="generic", name=archive.name, version=version).to_string() - components: Final[list[Component]] = [reader.distribution(prefix) for prefix in sorted(tree.metadata)] - license_files: Final[list[str]] = tree.metadata[""].values("license_files") - artifact = Artifact( - root={ - "type": "application", - "bom-ref": ref, - "name": archive.name, - "version": version, - "purl": ref, - "components": [file_component(ref, tree.files[name]) for name in tree.loose], - }, - components=components, - dependencies=vendored_dependencies(components), + bundled_info={str(path): content.bundled_wheel(context, path) for path in bundled_paths(tree)}, build_scripts=[ BuildScript(path=path, sha256=hashlib.sha256((root / str(path)).read_bytes()).hexdigest()) for path in dict.fromkeys([*build_scripts, *provider_scripts]) ], - version=version, - sbom_file=f"{archive.name}.cdx.json", - scope=( - "Components are the distributions the archive holds, as wheels or unpacked, each with its files; the" - " archive's own top-level files sit under the root component." - ), - workflow=("archive-build", f"build {archive.name} and this SBOM"), - license_file=next(iter(license_files), None), - copyright=copyright_notice(root, license_files), + copyright_line=copyright_notice(root, tree.metadata[""].values("license_files")), ) - contributions = checked_contributions(content.contributions(reader.context), content) - return assemble(artifact, project, settings, facts, contributions) - - -@dataclass(frozen=True, slots=True, kw_only=True) -class _Reader: - tree: ArchiveTree - project: ProjectIdentity - content: ContentProvider - context: ProviderContext - - def distribution(self, prefix: str) -> Component: - component = component_from_metadata(self.tree.metadata[prefix]) - if prefix: - # a wheel bundled unpacked: the directory tells copies of one release apart, as the path does for a .whl - directory = PurePosixPath(prefix.rstrip("/")) - info = self.content.bundled_wheel(self.context, directory) - component["bom-ref"] = f"bundled:{directory}" - if info.supplier is not None: - component["supplier"] = info.supplier - component["properties"] = [ - {"name": f"{self.project.property_prefix}:bundled-wheel", "value": str(directory)}, - *info.properties, - *component["properties"], - ] - component["components"] = [ - self.wheel(name) if name.endswith(".whl") else file_component(component["bom-ref"], self.tree.files[name]) - for name in self.tree.owned[prefix] - ] - return component - - def wheel(self, name: str) -> Component: - info = self.content.bundled_wheel(self.context, PurePosixPath(name)) - return bundled_wheel_component( - self.tree.bundled[name], self.project.property_prefix, info.properties, info.supplier - ) + return assemble(artifact, project, settings, facts, checked_contributions(content.contributions(context), content)) __all__ = [ + "archive_sbom", "describe_archive", ] diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py index b0e6dcc..2fff5a7 100644 --- a/src/python_sbom/_generate/_embed.py +++ b/src/python_sbom/_generate/_embed.py @@ -1,6 +1,8 @@ from __future__ import annotations import dataclasses +from dataclasses import dataclass +from pathlib import PurePosixPath from typing import TYPE_CHECKING from python_sbom._collect._archive import member_reader @@ -9,18 +11,21 @@ from python_sbom._collect._revision import revision_reader from python_sbom._collect._settings import settings_from_pyproject from python_sbom._collect._source import read_source_tree, wheel_request +from python_sbom._formats._json import dumps +from python_sbom._model._bom import describe from python_sbom._model._content import ProviderContext +from python_sbom._model._document import checked_contributions from python_sbom._model._identity import project_identity +from python_sbom._model._names import sbom_file_name from python_sbom._wheel._archive import member_digest, open_archive, write_sbom -from ._pipeline import build_sbom - if TYPE_CHECKING: from collections.abc import Sequence from pathlib import Path from python_sbom._model._content import ContentProvider - from python_sbom._model._inputs import BuildFacts, SbomRequest + from python_sbom._model._inputs import BuildFacts, SbomRequest, SourceTree + from python_sbom._model._types import Document def embed_sbom( @@ -59,7 +64,34 @@ def embed_into(wheel: Path, request: SbomRequest, content: ContentProvider, fact return write_sbom(wheel, sbom.file_name, sbom.content, facts.epoch) +def build_sbom( + request: SbomRequest, source: SourceTree, facts: BuildFacts, content: ContentProvider, context: ProviderContext +) -> Sbom: + return Sbom( + file_name=sbom_file_name(request.metadata.name), + document=describe( + request, + source, + facts, + {wheel.path: content.bundled_wheel(context, PurePosixPath(wheel.path)) for wheel in source.bundled}, + checked_contributions(content.contributions(context), content), + ), + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Sbom: + file_name: str + document: Document + + @property + def content(self) -> bytes: + return dumps(self.document) + + __all__ = [ + "Sbom", + "build_sbom", "embed_into", "embed_sbom", ] diff --git a/src/python_sbom/_generate/_pipeline.py b/src/python_sbom/_generate/_pipeline.py deleted file mode 100644 index 5428bde..0000000 --- a/src/python_sbom/_generate/_pipeline.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import PurePosixPath -from typing import TYPE_CHECKING - -from python_sbom._formats._json import dumps -from python_sbom._model._bom import describe -from python_sbom._model._document import checked_contributions -from python_sbom._model._names import sbom_file_name - -if TYPE_CHECKING: - from python_sbom._model._content import ContentProvider, ProviderContext - from python_sbom._model._inputs import BuildFacts, SbomRequest, SourceTree - from python_sbom._model._types import Document - - -def build_sbom( - request: SbomRequest, source: SourceTree, facts: BuildFacts, content: ContentProvider, context: ProviderContext -) -> Sbom: - return Sbom( - file_name=sbom_file_name(request.metadata.name), - document=describe( - request, - source, - facts, - {wheel.path: content.bundled_wheel(context, PurePosixPath(wheel.path)) for wheel in source.bundled}, - checked_contributions(content.contributions(context), content), - ), - ) - - -@dataclass(frozen=True, slots=True, kw_only=True) -class Sbom: - file_name: str - document: Document - - @property - def content(self) -> bytes: - return dumps(self.document) - - -__all__ = [ - "Sbom", - "build_sbom", -] diff --git a/src/python_sbom/_model/_archive.py b/src/python_sbom/_model/_archive.py new file mode 100644 index 0000000..babf93f --- /dev/null +++ b/src/python_sbom/_model/_archive.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import TYPE_CHECKING, Final + +from packageurl import PackageURL + +from ._components import bundled_wheel_component, component_from_metadata, file_component, vendored_dependencies +from ._inputs import Artifact + +if TYPE_CHECKING: + from collections.abc import Mapping + + from ._identity import ProjectIdentity + from ._inputs import ArchiveTree, BuildScript, BundledWheelInfo + from ._types import Component + + +def archive_artifact( # ruff: ignore[too-many-arguments] # what the edge read from the project, keyword-only + name: str, + tree: ArchiveTree, + *, + project: ProjectIdentity, + bundled_info: Mapping[str, BundledWheelInfo], + build_scripts: list[BuildScript], + copyright_line: str | None, +) -> Artifact: + # the archive's top-level .dist-info names the project; every other .dist-info marks a wheel bundled unpacked into + # its parent directory and every .whl member a bundled wheel; other files belong to the deepest distribution that + # holds them, or, at the top level, to the archive + version: Final[str] = tree.metadata[""].version + ref: Final[str] = PackageURL(type="generic", name=name, version=version).to_string() + distributions = _Distributions(tree=tree, project=project, bundled_info=bundled_info) + components: Final[list[Component]] = [distributions.component(prefix) for prefix in sorted(tree.metadata)] + return Artifact( + root={ + "type": "application", + "bom-ref": ref, + "name": name, + "version": version, + "purl": ref, + "components": [file_component(ref, tree.files[member]) for member in tree.loose], + }, + components=components, + dependencies=vendored_dependencies(components), + build_scripts=build_scripts, + version=version, + sbom_file=f"{name}.cdx.json", + scope=( + "Components are the distributions the archive holds, as wheels or unpacked, each with its files; the" + " archive's own top-level files sit under the root component." + ), + workflow=("archive-build", f"build {name} and this SBOM"), + license_file=next(iter(tree.metadata[""].values("license_files")), None), + copyright=copyright_line, + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class _Distributions: + tree: ArchiveTree + project: ProjectIdentity + bundled_info: Mapping[str, BundledWheelInfo] + + def component(self, prefix: str) -> Component: + component = component_from_metadata(self.tree.metadata[prefix]) + if prefix: + # a wheel bundled unpacked: the directory tells copies of one release apart, as the path does for a .whl + directory = _directory(prefix) + info = self.bundled_info[str(directory)] + component["bom-ref"] = f"bundled:{directory}" + if info.supplier is not None: + component["supplier"] = info.supplier + component["properties"] = [ + {"name": f"{self.project.property_prefix}:bundled-wheel", "value": str(directory)}, + *info.properties, + *component["properties"], + ] + component["components"] = [ + self._wheel(name) if name.endswith(".whl") else file_component(component["bom-ref"], self.tree.files[name]) + for name in self.tree.owned[prefix] + ] + return component + + def _wheel(self, name: str) -> Component: + info = self.bundled_info[name] + return bundled_wheel_component( + self.tree.bundled[name], self.project.property_prefix, info.properties, info.supplier + ) + + +def _directory(prefix: str) -> PurePosixPath: + return PurePosixPath(prefix.rstrip("/")) + + +def bundled_paths(tree: ArchiveTree) -> list[PurePosixPath]: + # what a content provider is asked about: each wheel bundled unpacked, by its directory, and each .whl member + return [ + *(_directory(prefix) for prefix in sorted(tree.metadata) if prefix), + *map(PurePosixPath, sorted(tree.bundled)), + ] + + +__all__ = [ + "archive_artifact", + "bundled_paths", +] diff --git a/tests/_generate/test_embed.py b/tests/_generate/test_embed.py index 6882e6f..69e8eae 100644 --- a/tests/_generate/test_embed.py +++ b/tests/_generate/test_embed.py @@ -7,15 +7,20 @@ from typing import TYPE_CHECKING, Final from python_sbom import BundledWheelInfo, ContentProvider, ProviderContext -from python_sbom._generate._embed import embed_sbom +from python_sbom._formats._json import dumps +from python_sbom._generate._embed import build_sbom, embed_sbom +from python_sbom._model._inputs import SbomRequest, SourceTree +from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: from pathlib import PurePosixPath import pytest - from conftest import Install, WriteWheel + from conftest import ContextFor, Install, WriteWheel from pytest_mock import MockerFixture + from python_sbom._model._inputs import BuildFacts, Settings + _PINNED: Final[Path] = Path(__file__).parent / "pinned" @@ -125,3 +130,18 @@ def test_embed_output_is_pinned( def _sha256(content: bytes) -> str: return hashlib.sha256(content).hexdigest() + + +def test_sbom_file_name_and_content(settings: Settings, facts: BuildFacts, context_for: ContextFor) -> None: + request = SbomRequest( + root=Path(), + metadata=parse_metadata("Name: My.Project\nVersion: 1.0\n"), + wheel_name="my_project-1.0-py3-none-any.whl", + settings=settings, + build_scripts=(), + ) + context = context_for(Path(), Path()) + sbom = build_sbom( + request, SourceTree(copyright=None, bundled=[], build_scripts=[]), facts, ContentProvider(), context + ) + assert (sbom.file_name, sbom.content) == ("my-project.cdx.json", dumps(sbom.document)) diff --git a/tests/_generate/test_pipeline.py b/tests/_generate/test_pipeline.py deleted file mode 100644 index e65a885..0000000 --- a/tests/_generate/test_pipeline.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import TYPE_CHECKING - -from python_sbom import ContentProvider -from python_sbom._formats._json import dumps -from python_sbom._generate._pipeline import build_sbom -from python_sbom._model._inputs import SbomRequest, SourceTree -from python_sbom._model._metadata import parse_metadata - -if TYPE_CHECKING: - from conftest import ContextFor - - from python_sbom._model._inputs import BuildFacts, Settings - - -def test_sbom_file_name_and_content(settings: Settings, facts: BuildFacts, context_for: ContextFor) -> None: - request = SbomRequest( - root=Path(), - metadata=parse_metadata("Name: My.Project\nVersion: 1.0\n"), - wheel_name="my_project-1.0-py3-none-any.whl", - settings=settings, - build_scripts=(), - ) - context = context_for(Path(), Path()) - sbom = build_sbom( - request, SourceTree(copyright=None, bundled=[], build_scripts=[]), facts, ContentProvider(), context - ) - assert (sbom.file_name, sbom.content) == ("my-project.cdx.json", dumps(sbom.document)) diff --git a/tests/conftest.py b/tests/conftest.py index 7ae0db7..7f14b7c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ from python_sbom import ContentProvider, ProviderContext from python_sbom._collect._archive import member_reader -from python_sbom._generate._pipeline import build_sbom +from python_sbom._generate._embed import build_sbom from python_sbom._model._identity import project_identity from python_sbom._model._inputs import BuildFacts, Interpreter, SbomRequest, Settings, SourceTree from python_sbom._model._metadata import parse_metadata From 4b05ac097ab68dc875d9923b9c875d67509f6be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 09:08:02 -0800 Subject: [PATCH 110/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20move?= =?UTF-8?q?=20bundled=20wheel=20checks=20into=20=5Fwheel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/python_sbom/__init__.py | 4 +- src/python_sbom/_collect/_archive.py | 3 +- src/python_sbom/_collect/_source.py | 136 +------------ src/python_sbom/_model/_bom.py | 3 +- src/python_sbom/_model/_content.py | 48 ++++- src/python_sbom/_model/_document.py | 4 +- src/python_sbom/_model/_inputs.py | 48 +---- src/python_sbom/_wheel/_bundled.py | 147 ++++++++++++++ tests/_collect/test_source.py | 268 +------------------------ tests/_wheel/test_bundled.py | 281 +++++++++++++++++++++++++++ 10 files changed, 489 insertions(+), 453 deletions(-) create mode 100644 src/python_sbom/_wheel/_bundled.py create mode 100644 tests/_wheel/test_bundled.py diff --git a/src/python_sbom/__init__.py b/src/python_sbom/__init__.py index 25363ff..a71cd89 100644 --- a/src/python_sbom/__init__.py +++ b/src/python_sbom/__init__.py @@ -3,9 +3,9 @@ from __future__ import annotations from ._errors import SbomError -from ._model._content import ContentProvider, ProviderContext +from ._model._content import BundledWheelInfo, ContentProvider, Contributions, ProviderContext from ._model._identity import ProjectIdentity -from ._model._inputs import BundledWheelInfo, Contributions, Settings +from ._model._inputs import Settings from ._model._types import ( Component, Contact, diff --git a/src/python_sbom/_collect/_archive.py b/src/python_sbom/_collect/_archive.py index 01d4683..10aab50 100644 --- a/src/python_sbom/_collect/_archive.py +++ b/src/python_sbom/_collect/_archive.py @@ -7,8 +7,7 @@ from python_sbom._model._inputs import ArchiveTree, RecordRow from python_sbom._model._metadata import parse_metadata from python_sbom._wheel._archive import member_digest, open_archive, read_member - -from ._source import read_bundled_wheels +from python_sbom._wheel._bundled import read_bundled_wheels if TYPE_CHECKING: from collections.abc import Callable diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index c6c89fd..019efc0 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -1,34 +1,19 @@ from __future__ import annotations import hashlib -from collections import Counter -from concurrent.futures import ThreadPoolExecutor from pathlib import PurePath from typing import TYPE_CHECKING, Final -from installer.records import InvalidRecordEntry, parse_record_file -from installer.sources import WheelFile -from packaging.requirements import InvalidRequirement, Requirement - from python_sbom._errors import SbomError -from python_sbom._model._inputs import BuildScript, BundledWheel, RecordRow, SbomRequest, SourceTree -from python_sbom._model._metadata import CoreMetadata, parse_metadata -from python_sbom._wheel._archive import open_archive, read_member, read_wheel_metadata -from python_sbom._wheel._record import record_rows +from python_sbom._model._inputs import BuildScript, SbomRequest, SourceTree +from python_sbom._wheel._archive import open_archive, read_wheel_metadata +from python_sbom._wheel._bundled import read_bundled_wheels if TYPE_CHECKING: - import zipfile - from collections.abc import Callable from pathlib import Path, PurePosixPath - from typing import IO from python_sbom._model._inputs import Settings -# a bundled wheel is hashed in chunks of this size -_CHUNK: Final[int] = 1024 * 1024 -# wheels verified at once, each perhaps in memory whole -_WORKERS: Final[int] = 4 - def read_source_tree(request: SbomRequest) -> SourceTree: root = request.root @@ -60,120 +45,6 @@ def copyright_notice(root: Path, license_files: list[str]) -> str | None: return None -def read_bundled_wheels(paths: list[str], open_wheel: Callable[[str], IO[bytes]]) -> list[BundledWheel]: - # inflating and hashing release the GIL, so the wheels verify side by side, a few at a time, as each may be held in - # memory whole; results and the first error come in the order given - def read(path: str) -> BundledWheel: - with open_wheel(path) as stream: - return _read_bundled_wheel(stream, path) - - with ThreadPoolExecutor(max_workers=_WORKERS) as pool: - return list(pool.map(read, paths)) - - -def _read_bundled_wheel(stream: IO[bytes], path: str) -> BundledWheel: - where: Final[str] = f"bundled wheel {path}" - digest: Final = hashlib.sha256() - while chunk := stream.read(_CHUNK): - digest.update(chunk) - stream.seek(0) - with open_archive(stream, where) as archive: - # installer takes the project name from the archive's file name, which an archive read from memory lacks - archive.filename = path - members: Final[list[str]] = sorted(name for name in archive.namelist() if not name.endswith("/")) - wheel, metadata = read_wheel_metadata(archive, where) - dist_info: Final[str] = wheel.dist_info_dir - _check_named(metadata, where, f"{dist_info}/METADATA") - files: Final[list[RecordRow]] = _verified_record(archive, wheel, where) - vendored: Final[list[tuple[str, CoreMetadata]]] = [ - *_vendor_manifest(archive, where, members), - *( - (name, _vendored_metadata(archive, where, name)) - for name in members - if "/_vendor/" in name and name.endswith(".dist-info/METADATA") - ), - ] - return BundledWheel( - path=path, - sha256=digest.hexdigest(), - metadata_name=f"{dist_info}/METADATA", - metadata=metadata, - files=files, - vendored=vendored, - ) - - -def _check_named(metadata: CoreMetadata, where: str, name: str) -> None: - if not (metadata.name and metadata.version): - msg = f"{where}: {name} lacks Name or Version" - raise SbomError(msg) - - -def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> list[RecordRow]: - # the SBOM copies these rows, so a RECORD that misstates the wheel would make the SBOM misstate it too - dist_info: Final[str] = wheel.dist_info_dir - # installer decompresses RECORD whole, so a capped read comes first and bounds installer's read of the same bytes - record: Final[str] = read_member(archive, f"{dist_info}/RECORD", where).decode("utf-8") - try: - rows: Final[list[tuple[str, str, str]]] = list(parse_record_file(record.splitlines())) - # read before installer's check, which takes any hashlib name and fails on one needing a length, such as - # shake_128, with a TypeError - found: Final[list[RecordRow]] = record_rows(rows, where) - except InvalidRecordEntry as error: - msg = f"{where}: RECORD does not parse: {error}" - raise SbomError(msg) from error - # installer checks the last row of a path only, skips rows naming no member, and lets a RECORD.jws or RECORD.p7s - # anywhere under the .dist-info, "../" included, go unlisted, so each of these checks is ours - if repeated := sorted(path for path, count in Counter(path for path, _, _ in rows).items() if count > 1): - msg = f"{where}: RECORD lists {', '.join(repeated)} more than once" - raise SbomError(msg) - members: Final[set[str]] = {info.filename for info in archive.infolist() if not info.is_dir()} - recorded: Final[set[str]] = {path for path, _, _ in rows} - # installer checks the row of RECORD itself, the one row without a digest - if missing := sorted( - members - recorded - {f"{dist_info}/{name}" for name in ("RECORD", "RECORD.jws", "RECORD.p7s")} - ): - msg = f"{where}: RECORD lacks a digest for {', '.join(missing)}" - raise SbomError(msg) - if extra := sorted(recorded - members): - msg = f"{where}: RECORD lists {', '.join(extra)}, which the wheel does not hold" - raise SbomError(msg) - try: - wheel.validate_record() - except WheelFile.validation_error as error: - # installer documents the exception class only, not the list of issues it carries - issues: Final[list[str]] = getattr(error, "issues", [str(error)]) - prefix: Final[str] = f"In {archive.filename}, " - msg = f"{where}: RECORD disagrees with the wheel: {'; '.join(issue.removeprefix(prefix) for issue in issues)}" - raise SbomError(msg) from error - return found - - -def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) -> list[tuple[str, CoreMetadata]]: - # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info - found: Final[list[tuple[str, CoreMetadata]]] = [] - for name in (member for member in members if member.endswith("/_vendor/vendor.txt")): - for line in read_member(archive, name, where).decode("utf-8").splitlines(): - if not (text := line.partition("#")[0].strip()): - continue - try: - pins = list((requirement := Requirement(text)).specifier) - except InvalidRequirement as error: - msg = f"{where}: vendored requirement in {name} does not parse: {text}" - raise SbomError(msg) from error - if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: - msg = f"{where}: vendored requirement in {name} pins no one exact version: {text}" - raise SbomError(msg) - found.append((name, CoreMetadata(raw={"name": requirement.name, "version": pins[0].version}, unparsed={}))) - return found - - -def _vendored_metadata(archive: zipfile.ZipFile, where: str, name: str) -> CoreMetadata: - metadata: Final[CoreMetadata] = parse_metadata(read_member(archive, name, where).decode("utf-8")) - _check_named(metadata, where, name) - return metadata - - def wheel_request(wheel: Path, root: Path, settings: Settings, build_scripts: tuple[PurePosixPath, ...]) -> SbomRequest: with open_archive(wheel, str(wheel)) as archive: _, metadata = read_wheel_metadata(archive, str(wheel)) @@ -184,7 +55,6 @@ def wheel_request(wheel: Path, root: Path, settings: Settings, build_scripts: tu __all__ = [ "copyright_notice", - "read_bundled_wheels", "read_source_tree", "wheel_request", ] diff --git a/src/python_sbom/_model/_bom.py b/src/python_sbom/_model/_bom.py index 0a25d74..cf095e1 100644 --- a/src/python_sbom/_model/_bom.py +++ b/src/python_sbom/_model/_bom.py @@ -17,8 +17,9 @@ if TYPE_CHECKING: from collections.abc import Mapping + from ._content import BundledWheelInfo, Contributions from ._identity import ProjectIdentity - from ._inputs import BuildFacts, BundledWheelInfo, Contributions, SbomRequest, Settings, SourceTree + from ._inputs import BuildFacts, SbomRequest, Settings, SourceTree from ._types import Body, Component, Document, ExternalReference, Metadata, Property diff --git a/src/python_sbom/_model/_content.py b/src/python_sbom/_model/_content.py index 83dee50..cd51765 100644 --- a/src/python_sbom/_model/_content.py +++ b/src/python_sbom/_model/_content.py @@ -1,15 +1,15 @@ from __future__ import annotations +from dataclasses import dataclass, field from typing import TYPE_CHECKING -from ._inputs import BundledWheelInfo, Contributions - if TYPE_CHECKING: from collections.abc import Callable from pathlib import Path, PurePosixPath from ._identity import ProjectIdentity from ._inputs import Settings + from ._types import Component, Dependency, OrganizationalEntity, Property class ProviderContext: @@ -78,7 +78,51 @@ def contributions(self, context: ProviderContext) -> Contributions: # ruff: enable[unused-method-argument, no-self-use] +@dataclass(frozen=True, slots=True, kw_only=True) +class BundledWheelInfo: + """ + What a project adds to the component of one bundled wheel. + + Both fields default to empty, so a provider names only what it adds. + + :param properties: properties after ``:bundled-wheel``, before those from the wheel's metadata + :param supplier: the wheel's supplier, or ``None`` to name none; only the project decides a bundled wheel's + supplier, never the wheel's own metadata + + """ + + properties: list[Property] = field(default_factory=list) + supplier: OrganizationalEntity | None = None + + +@dataclass(frozen=True, slots=True, kw_only=True) +class Contributions: + """ + What a project adds to the document beyond its bundled wheels, merged before references and the serial are derived. + + Each field defaults to empty, so a provider names only what it adds. The build fails when a component or tool lacks + ``type``, ``bom-ref`` or ``name``, a ``bom-ref`` repeats, the ``formula:`` and ``workflow:`` refs of the build's + formulation included, a dependency names one that does not exist, or a ``bom-ref`` gets a second dependency entry, + as the root does if a provider adds one, since python-sbom writes the root's. + + :param components: extra top-level components, such as vendored C libraries; the root depends on each, and a + component's own parts nest under its ``components`` + :param dependencies: extra dependency entries, such as the dependencies of those components + :param root_properties: extra properties of the root component; their names must not start with the project's + prefix, which python-sbom's own properties use + :param tools: extra build tools + + """ + + components: list[Component] = field(default_factory=list) + dependencies: list[Dependency] = field(default_factory=list) + root_properties: list[Property] = field(default_factory=list) + tools: list[Component] = field(default_factory=list) + + __all__ = [ + "BundledWheelInfo", "ContentProvider", + "Contributions", "ProviderContext", ] diff --git a/src/python_sbom/_model/_document.py b/src/python_sbom/_model/_document.py index dad5c83..46c249f 100644 --- a/src/python_sbom/_model/_document.py +++ b/src/python_sbom/_model/_document.py @@ -11,8 +11,8 @@ if TYPE_CHECKING: from collections.abc import Iterable, Iterator - from ._content import ContentProvider - from ._inputs import BuildFacts, Contributions + from ._content import ContentProvider, Contributions + from ._inputs import BuildFacts from ._types import Body, Component, Document, Formula, Property diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py index 14d03d7..61cbc0b 100644 --- a/src/python_sbom/_model/_inputs.py +++ b/src/python_sbom/_model/_inputs.py @@ -1,13 +1,13 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import TYPE_CHECKING if TYPE_CHECKING: from pathlib import Path, PurePosixPath from ._metadata import CoreMetadata - from ._types import Component, Dependency, OrganizationalEntity, Property + from ._types import Component, Dependency, OrganizationalEntity @dataclass(frozen=True, slots=True, kw_only=True) @@ -130,56 +130,12 @@ class SourceTree: build_scripts: list[BuildScript] -@dataclass(frozen=True, slots=True, kw_only=True) -class BundledWheelInfo: - """ - What a project adds to the component of one bundled wheel. - - Both fields default to empty, so a provider names only what it adds. - - :param properties: properties after ``:bundled-wheel``, before those from the wheel's metadata - :param supplier: the wheel's supplier, or ``None`` to name none; only the project decides a bundled wheel's - supplier, never the wheel's own metadata - - """ - - properties: list[Property] = field(default_factory=list) - supplier: OrganizationalEntity | None = None - - -@dataclass(frozen=True, slots=True, kw_only=True) -class Contributions: - """ - What a project adds to the document beyond its bundled wheels, merged before references and the serial are derived. - - Each field defaults to empty, so a provider names only what it adds. The build fails when a component or tool lacks - ``type``, ``bom-ref`` or ``name``, a ``bom-ref`` repeats, the ``formula:`` and ``workflow:`` refs of the build's - formulation included, a dependency names one that does not exist, or a ``bom-ref`` gets a second dependency entry, - as the root does if a provider adds one, since python-sbom writes the root's. - - :param components: extra top-level components, such as vendored C libraries; the root depends on each, and a - component's own parts nest under its ``components`` - :param dependencies: extra dependency entries, such as the dependencies of those components - :param root_properties: extra properties of the root component; their names must not start with the project's - prefix, which python-sbom's own properties use - :param tools: extra build tools - - """ - - components: list[Component] = field(default_factory=list) - dependencies: list[Dependency] = field(default_factory=list) - root_properties: list[Property] = field(default_factory=list) - tools: list[Component] = field(default_factory=list) - - __all__ = [ "ArchiveTree", "Artifact", "BuildFacts", "BuildScript", "BundledWheel", - "BundledWheelInfo", - "Contributions", "InstalledDistribution", "Interpreter", "RecordRow", diff --git a/src/python_sbom/_wheel/_bundled.py b/src/python_sbom/_wheel/_bundled.py new file mode 100644 index 0000000..357cf0e --- /dev/null +++ b/src/python_sbom/_wheel/_bundled.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import hashlib +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING, Final + +from installer.records import InvalidRecordEntry, parse_record_file +from installer.sources import WheelFile +from packaging.requirements import InvalidRequirement, Requirement + +from python_sbom._errors import SbomError +from python_sbom._model._inputs import BundledWheel, RecordRow +from python_sbom._model._metadata import CoreMetadata, parse_metadata + +from ._archive import open_archive, read_member, read_wheel_metadata +from ._record import record_rows + +if TYPE_CHECKING: + import zipfile + from collections.abc import Callable + from typing import IO + + +# a bundled wheel is hashed in chunks of this size +_CHUNK: Final[int] = 1024 * 1024 +# wheels verified at once, each perhaps in memory whole +_WORKERS: Final[int] = 4 + + +def read_bundled_wheels(paths: list[str], open_wheel: Callable[[str], IO[bytes]]) -> list[BundledWheel]: + # inflating and hashing release the GIL, so the wheels verify side by side, a few at a time, as each may be held in + # memory whole; results and the first error come in the order given + def read(path: str) -> BundledWheel: + with open_wheel(path) as stream: + return _read_bundled_wheel(stream, path) + + with ThreadPoolExecutor(max_workers=_WORKERS) as pool: + return list(pool.map(read, paths)) + + +def _read_bundled_wheel(stream: IO[bytes], path: str) -> BundledWheel: + where: Final[str] = f"bundled wheel {path}" + digest: Final = hashlib.sha256() + while chunk := stream.read(_CHUNK): + digest.update(chunk) + stream.seek(0) + with open_archive(stream, where) as archive: + # installer takes the project name from the archive's file name, which an archive read from memory lacks + archive.filename = path + members: Final[list[str]] = sorted(name for name in archive.namelist() if not name.endswith("/")) + wheel, metadata = read_wheel_metadata(archive, where) + dist_info: Final[str] = wheel.dist_info_dir + _check_named(metadata, where, f"{dist_info}/METADATA") + files: Final[list[RecordRow]] = _verified_record(archive, wheel, where) + vendored: Final[list[tuple[str, CoreMetadata]]] = [ + *_vendor_manifest(archive, where, members), + *( + (name, _vendored_metadata(archive, where, name)) + for name in members + if "/_vendor/" in name and name.endswith(".dist-info/METADATA") + ), + ] + return BundledWheel( + path=path, + sha256=digest.hexdigest(), + metadata_name=f"{dist_info}/METADATA", + metadata=metadata, + files=files, + vendored=vendored, + ) + + +def _check_named(metadata: CoreMetadata, where: str, name: str) -> None: + if not (metadata.name and metadata.version): + msg = f"{where}: {name} lacks Name or Version" + raise SbomError(msg) + + +def _verified_record(archive: zipfile.ZipFile, wheel: WheelFile, where: str) -> list[RecordRow]: + # the SBOM copies these rows, so a RECORD that misstates the wheel would make the SBOM misstate it too + dist_info: Final[str] = wheel.dist_info_dir + # installer decompresses RECORD whole, so a capped read comes first and bounds installer's read of the same bytes + record: Final[str] = read_member(archive, f"{dist_info}/RECORD", where).decode("utf-8") + try: + rows: Final[list[tuple[str, str, str]]] = list(parse_record_file(record.splitlines())) + # read before installer's check, which takes any hashlib name and fails on one needing a length, such as + # shake_128, with a TypeError + found: Final[list[RecordRow]] = record_rows(rows, where) + except InvalidRecordEntry as error: + msg = f"{where}: RECORD does not parse: {error}" + raise SbomError(msg) from error + # installer checks the last row of a path only, skips rows naming no member, and lets a RECORD.jws or RECORD.p7s + # anywhere under the .dist-info, "../" included, go unlisted, so each of these checks is ours + if repeated := sorted(path for path, count in Counter(path for path, _, _ in rows).items() if count > 1): + msg = f"{where}: RECORD lists {', '.join(repeated)} more than once" + raise SbomError(msg) + members: Final[set[str]] = {info.filename for info in archive.infolist() if not info.is_dir()} + recorded: Final[set[str]] = {path for path, _, _ in rows} + # installer checks the row of RECORD itself, the one row without a digest + if missing := sorted( + members - recorded - {f"{dist_info}/{name}" for name in ("RECORD", "RECORD.jws", "RECORD.p7s")} + ): + msg = f"{where}: RECORD lacks a digest for {', '.join(missing)}" + raise SbomError(msg) + if extra := sorted(recorded - members): + msg = f"{where}: RECORD lists {', '.join(extra)}, which the wheel does not hold" + raise SbomError(msg) + try: + wheel.validate_record() + except WheelFile.validation_error as error: + # installer documents the exception class only, not the list of issues it carries + issues: Final[list[str]] = getattr(error, "issues", [str(error)]) + prefix: Final[str] = f"In {archive.filename}, " + msg = f"{where}: RECORD disagrees with the wheel: {'; '.join(issue.removeprefix(prefix) for issue in issues)}" + raise SbomError(msg) from error + return found + + +def _vendor_manifest(archive: zipfile.ZipFile, where: str, members: list[str]) -> list[tuple[str, CoreMetadata]]: + # pip lists what it vendors in _vendor/vendor.txt instead of shipping each vendored .dist-info + found: Final[list[tuple[str, CoreMetadata]]] = [] + for name in (member for member in members if member.endswith("/_vendor/vendor.txt")): + for line in read_member(archive, name, where).decode("utf-8").splitlines(): + if not (text := line.partition("#")[0].strip()): + continue + try: + pins = list((requirement := Requirement(text)).specifier) + except InvalidRequirement as error: + msg = f"{where}: vendored requirement in {name} does not parse: {text}" + raise SbomError(msg) from error + if len(pins) != 1 or pins[0].operator != "==" or "*" in pins[0].version: + msg = f"{where}: vendored requirement in {name} pins no one exact version: {text}" + raise SbomError(msg) + found.append((name, CoreMetadata(raw={"name": requirement.name, "version": pins[0].version}, unparsed={}))) + return found + + +def _vendored_metadata(archive: zipfile.ZipFile, where: str, name: str) -> CoreMetadata: + metadata: Final[CoreMetadata] = parse_metadata(read_member(archive, name, where).decode("utf-8")) + _check_named(metadata, where, name) + return metadata + + +__all__ = [ + "read_bundled_wheels", +] diff --git a/tests/_collect/test_source.py b/tests/_collect/test_source.py index 97dd385..329d359 100644 --- a/tests/_collect/test_source.py +++ b/tests/_collect/test_source.py @@ -1,20 +1,16 @@ from __future__ import annotations -import base64 import dataclasses import hashlib import re -import zipfile -from io import BytesIO -from itertools import starmap from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Final import pytest -from python_sbom._collect._source import read_bundled_wheels, read_source_tree, wheel_request +from python_sbom._collect._source import read_source_tree, wheel_request from python_sbom._errors import SbomError -from python_sbom._model._inputs import BuildScript, BundledWheel, RecordRow, SbomRequest +from python_sbom._model._inputs import BuildScript, SbomRequest from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: @@ -24,19 +20,7 @@ from python_sbom._model._inputs import Settings -_INIT: Final[str] = "pip/__init__.py" -_METADATA: Final[str] = "pip-1.0.dist-info/METADATA" -_PIP: Final[dict[str, bytes]] = {_INIT: b"", _METADATA: b"Name: pip\nVersion: 1.0\n"} - - -# defined before the parametrized cases below, which build their RECORD rows with it at import time -def row(path: str, content: bytes, algorithm: str = "sha256", *, encoded: str = "", size: int | None = None) -> str: - encoded = encoded or base64.urlsafe_b64encode(hashlib.new(algorithm, content).digest()).rstrip(b"=").decode() - return f"{path},{algorithm}={encoded},{len(content) if size is None else size}\n" - - -_METADATA_ROW: Final[str] = row(_METADATA, _PIP[_METADATA]) -_RECORD_ROW: Final[str] = "pip-1.0.dist-info/RECORD,,\n" +_PIP: Final[dict[str, bytes]] = {"pip/__init__.py": b"", "pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n"} @pytest.mark.parametrize( @@ -110,235 +94,6 @@ def test_build_script_outside_project(tmp_path: Path, settings: Settings) -> Non assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] -def test_bundled_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: - wheel = write_wheel( - tmp_path / "src" / "pip-1.0-py3-none-any.whl", - { - **_PIP, - "pip/_vendor/vendor.txt": b"# pinned\n\nurllib3==1.26.4 # http\n", - "pip/_vendor/rich-13.0.dist-info/METADATA": b"Name: rich\nVersion: 13.0\n", - }, - ) - bundled = _bundled(wheel.read_bytes(), "src/pip-1.0-py3-none-any.whl") - assert (bundled.path, bundled.sha256, bundled.metadata_name, bundled.metadata.name, bundled.files[0]) == ( - "src/pip-1.0-py3-none-any.whl", - hashlib.sha256(wheel.read_bytes()).hexdigest(), - "pip-1.0.dist-info/METADATA", - "pip", - RecordRow(path="pip/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"").hexdigest(), size="0"), - ) - assert [(source, metadata.raw) for source, metadata in bundled.vendored] == [ - ("pip/_vendor/vendor.txt", {"name": "urllib3", "version": "1.26.4"}), - ("pip/_vendor/rich-13.0.dist-info/METADATA", {"name": "rich", "version": "13.0"}), - ] - - -@pytest.mark.parametrize( - ("files", "message"), - [ - pytest.param( - {"pip/__init__.py": b""}, - r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", - id="no-dist-info", - ), - pytest.param({"pip-1.0.dist-info/WHEEL": b""}, "has no pip-1.0.dist-info/METADATA", id="no-metadata"), - pytest.param( - {"aaa-0.dist-info/METADATA": b"Name: aaa\nVersion: 0\n", **_PIP}, - r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", - id="two-dist-info", - ), - pytest.param({"pip-1.0.dist-info/METADATA": b"Name: pip\n"}, "METADATA lacks Name or Version", id="no-version"), - pytest.param( - {"pip-1.0.dist-info/METADATA": _PIP["pip-1.0.dist-info/METADATA"]}, - "has no pip-1.0.dist-info/RECORD", - id="no-record", - ), - pytest.param( - {"pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n" + b" " * 16 * 1024 * 1024}, - "METADATA is over the 16777216 byte limit", - id="oversized-metadata", - ), - pytest.param( - {**_PIP, "pip-1.0.dist-info/RECORD": b"\n" * (16 * 1024 * 1024 + 1)}, - "RECORD is over the 16777216 byte limit", - id="oversized-record", - ), - ], -) -def test_malformed_bundled_wheel( - tmp_path: Path, write_wheel: WriteWheel, files: Mapping[str, bytes], message: str -) -> None: - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - with pytest.raises(SbomError, match=message): - _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") - - -@pytest.mark.parametrize( - ("name", "message"), - [ - pytest.param("pip.whl", "Not a valid wheel filename: pip.whl", id="unparsable"), - pytest.param( - "other-1.0-py3-none-any.whl", r"Wheel \.dist-info directory doesn't match wheel filename", id="other" - ), - ], -) -def test_bundled_wheel_file_name_names_the_project( - tmp_path: Path, write_wheel: WriteWheel, name: str, message: str -) -> None: - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", _PIP) - with pytest.raises(SbomError, match=f"bundled wheel {name} is not a wheel: {message}"): - _bundled(wheel.read_bytes(), name) - - -def test_corrupt_bundled_wheel() -> None: - with pytest.raises(SbomError, match=r"bundled wheel pip\.whl is not a valid zip archive"): - _bundled(b"not a zip", "pip.whl") - - -def test_bundled_wheel_with_duplicate_member(tmp_path: Path) -> None: - # installers extract the first copy, so a RECORD that hashes the second one would hide the installed code - with zipfile.ZipFile(wheel := tmp_path / "pip-1.0-py3-none-any.whl", "w") as archive: - archive.writestr(_INIT, b"import os\n") - with pytest.warns(UserWarning, match="Duplicate name"): - archive.writestr(_INIT, b"") - archive.writestr(_METADATA, _PIP[_METADATA]) - archive.writestr("pip-1.0.dist-info/RECORD", f"{row(_INIT, b'')}{_METADATA_ROW}") - with pytest.raises(SbomError, match=f"holds more than one member named {_INIT}, ignoring case"): - _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") - - -@pytest.mark.parametrize( - ("algorithm", "expected"), - [ - pytest.param("sha256", "SHA-256", id="sha256"), - pytest.param("sha384", "SHA-384", id="sha384"), - pytest.param("sha512", "SHA-512", id="sha512"), - ], -) -def test_bundled_wheel_record_algorithm(tmp_path: Path, write_wheel: WriteWheel, algorithm: str, expected: str) -> None: - record = "".join(row(path, content, algorithm) for path, content in _PIP.items()) + _RECORD_ROW - files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - assert [ - (file.path, file.algorithm, file.digest) - for file in _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files - ] == [(path, expected, hashlib.new(algorithm, content).hexdigest()) for path, content in _PIP.items()] - - -@pytest.mark.parametrize( - ("record", "message"), - [ - pytest.param( - f"{_INIT},sha256=x\n", "RECORD does not parse: Row Index 0: expected 3 elements, got 2", id="columns" - ), - pytest.param( - f"{_METADATA_ROW}\n{_RECORD_ROW}", - "RECORD does not parse: Row Index 1: expected 3 elements, got 0", - id="blank-line", - ), - pytest.param( - f"{_INIT},,0\n{_METADATA_ROW}{_RECORD_ROW}", - f"RECORD disagrees with the wheel: hash / size of {_INIT} is not included in RECORD", - id="no-digest", - ), - pytest.param( - f"{row(_INIT, b'')}{_METADATA_ROW}", - "RECORD disagrees with the wheel: pip-1.0.dist-info/RECORD is not mentioned in RECORD", - id="record-not-recorded", - ), - pytest.param( - f"{row(_INIT, b'', 'sha224')}{_METADATA_ROW}{_RECORD_ROW}", - f"RECORD row for {_INIT}: unknown hash algorithm 'sha224'", - id="algorithm-cyclonedx-lacks", - ), - # installer would call digest() without the length shake_128 needs and raise a TypeError - pytest.param( - f"{_INIT},shake_128=AAAA,0\n{_METADATA_ROW}{_RECORD_ROW}", - f"RECORD row for {_INIT}: unknown hash algorithm 'shake_128'", - id="algorithm-needing-a-length", - ), - pytest.param(_METADATA_ROW, f"RECORD lacks a digest for {_INIT}", id="member-not-recorded"), - pytest.param( - f"{row(_INIT, b'', encoded='A' * 43)}{row(_INIT, b'')}{_METADATA_ROW}", - f"RECORD lists {_INIT} more than once", - id="listed-twice", - ), - pytest.param( - f"{row(_INIT, b'')}{row(_INIT, b'', 'sha512', encoded='A' * 86)}{_METADATA_ROW}", - f"RECORD lists {_INIT} more than once", - id="listed-twice-by-other-algorithm", - ), - pytest.param( - f"{row(_INIT, b'')}{row('pip/ghost.py', b'', 'md5', size=99)}{_METADATA_ROW}", - "RECORD lists pip/ghost.py, which the wheel does not hold", - id="recorded-not-member", - ), - pytest.param( - f"{row(_INIT, b'', size=999999)}{_METADATA_ROW}{_RECORD_ROW}", - f"RECORD disagrees with the wheel: hash / size of {_INIT} didn't match RECORD", - id="size-mismatch", - ), - pytest.param( - f"{row(_INIT, b'', 'sha512', encoded='A' * 86)}{_METADATA_ROW}{_RECORD_ROW}", - f"RECORD disagrees with the wheel: hash / size of {_INIT} didn't match RECORD", - id="digest-mismatch", - ), - ], -) -def test_record_disagrees_with_wheel(tmp_path: Path, write_wheel: WriteWheel, record: str, message: str) -> None: - files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - with pytest.raises(SbomError, match=message): - _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") - - -def test_record_signatures_need_no_row(tmp_path: Path, write_wheel: WriteWheel) -> None: - record = "".join(starmap(row, _PIP.items())) + _RECORD_ROW - files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), "pip-1.0.dist-info/RECORD.jws": b"{}"} - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - assert [file.path for file in _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files] == list(_PIP) - - -@pytest.mark.parametrize( - "signature", - [ - pytest.param("pip-1.0.dist-info/sub/RECORD.jws", id="nested"), - pytest.param("pip-1.0.dist-info/../pip/RECORD.p7s", id="parent"), - ], -) -def test_signature_away_from_record_needs_a_row(tmp_path: Path, write_wheel: WriteWheel, signature: str) -> None: - # installer lets these go unlisted, though a signature only sits beside RECORD - record = "".join(starmap(row, _PIP.items())) + _RECORD_ROW - files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), signature: b"import os\n"} - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) - with pytest.raises(SbomError, match=f"RECORD lacks a digest for {re.escape(signature)}"): - _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") - - -def test_vendored_metadata_needs_name_and_version(tmp_path: Path, write_wheel: WriteWheel) -> None: - files = {**_PIP, "pip/_vendor/rich-13.0.dist-info/METADATA": b"Metadata-Version: 2.1\n"} - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files) - with pytest.raises(SbomError, match=r"pip/_vendor/rich-13\.0\.dist-info/METADATA lacks Name or Version"): - _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") - - -@pytest.mark.parametrize( - ("manifest", "message"), - [ - pytest.param(b"urllib3 ==\n", "does not parse: urllib3 ==", id="unparsable"), - pytest.param(b"urllib3>=1\n", "pins no one exact version: urllib3>=1", id="range"), - pytest.param(b"urllib3==1.*\n", "pins no one exact version: urllib3==1.*", id="wildcard"), - pytest.param(b"urllib3==1,!=1.1\n", "pins no one exact version: urllib3==1,!=1.1", id="two-specifiers"), - ], -) -def test_vendor_manifest_needs_exact_pins( - tmp_path: Path, write_wheel: WriteWheel, manifest: bytes, message: str -) -> None: - wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", {**_PIP, "pip/_vendor/vendor.txt": manifest}) - with pytest.raises(SbomError, match=message): - _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") - - def test_wheel_request(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: wheel = write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"Name: native\n"}) request = wheel_request(wheel, tmp_path, settings, (PurePosixPath("build.py"),)) @@ -368,20 +123,3 @@ def test_wheel_request_needs_a_wheel( archive = write_wheel(tmp_path / name, {"native/__init__.py": b""}, record=False) with pytest.raises(SbomError, match=message): wheel_request(archive, Path(), settings, ()) - - -def _bundled(data: bytes, path: str) -> BundledWheel: - return read_bundled_wheels([path], lambda _: BytesIO(data))[0] - - -def test_bundled_wheels_come_in_the_order_given(tmp_path: Path, write_wheel: WriteWheel) -> None: - # read side by side, yet reported in order, the first broken wheel's error included - good = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", _PIP).read_bytes() - wheels = {"pip-1.0-py3-none-any.whl": good, "b.whl": b"not a zip", "c.whl": b"neither"} - paths = ["pip-1.0-py3-none-any.whl", "b.whl", "c.whl"] - assert [bundled.path for bundled in read_bundled_wheels(paths[:1] * 2, lambda name: BytesIO(wheels[name]))] == [ - paths[0], - paths[0], - ] - with pytest.raises(SbomError, match=r"^bundled wheel b\.whl is not a valid zip archive"): - read_bundled_wheels(paths, lambda name: BytesIO(wheels[name])) diff --git a/tests/_wheel/test_bundled.py b/tests/_wheel/test_bundled.py new file mode 100644 index 0000000..b3226d0 --- /dev/null +++ b/tests/_wheel/test_bundled.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import base64 +import hashlib +import re +import zipfile +from io import BytesIO +from itertools import starmap +from pathlib import Path +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom._errors import SbomError +from python_sbom._model._inputs import BundledWheel, RecordRow +from python_sbom._wheel._bundled import read_bundled_wheels + +if TYPE_CHECKING: + from collections.abc import Mapping + + from conftest import WriteWheel + +_INIT: Final[str] = "pip/__init__.py" +_METADATA: Final[str] = "pip-1.0.dist-info/METADATA" +_PIP: Final[dict[str, bytes]] = {_INIT: b"", _METADATA: b"Name: pip\nVersion: 1.0\n"} + + +# defined before the parametrized cases below, which build their RECORD rows with it at import time +def row(path: str, content: bytes, algorithm: str = "sha256", *, encoded: str = "", size: int | None = None) -> str: + encoded = encoded or base64.urlsafe_b64encode(hashlib.new(algorithm, content).digest()).rstrip(b"=").decode() + return f"{path},{algorithm}={encoded},{len(content) if size is None else size}\n" + + +_METADATA_ROW: Final[str] = row(_METADATA, _PIP[_METADATA]) +_RECORD_ROW: Final[str] = "pip-1.0.dist-info/RECORD,,\n" + + +def test_bundled_wheel(tmp_path: Path, write_wheel: WriteWheel) -> None: + wheel = write_wheel( + tmp_path / "src" / "pip-1.0-py3-none-any.whl", + { + **_PIP, + "pip/_vendor/vendor.txt": b"# pinned\n\nurllib3==1.26.4 # http\n", + "pip/_vendor/rich-13.0.dist-info/METADATA": b"Name: rich\nVersion: 13.0\n", + }, + ) + bundled = _bundled(wheel.read_bytes(), "src/pip-1.0-py3-none-any.whl") + assert (bundled.path, bundled.sha256, bundled.metadata_name, bundled.metadata.name, bundled.files[0]) == ( + "src/pip-1.0-py3-none-any.whl", + hashlib.sha256(wheel.read_bytes()).hexdigest(), + "pip-1.0.dist-info/METADATA", + "pip", + RecordRow(path="pip/__init__.py", algorithm="SHA-256", digest=hashlib.sha256(b"").hexdigest(), size="0"), + ) + assert [(source, metadata.raw) for source, metadata in bundled.vendored] == [ + ("pip/_vendor/vendor.txt", {"name": "urllib3", "version": "1.26.4"}), + ("pip/_vendor/rich-13.0.dist-info/METADATA", {"name": "rich", "version": "13.0"}), + ] + + +@pytest.mark.parametrize( + ("files", "message"), + [ + pytest.param( + {"pip/__init__.py": b""}, + r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", + id="no-dist-info", + ), + pytest.param({"pip-1.0.dist-info/WHEEL": b""}, "has no pip-1.0.dist-info/METADATA", id="no-metadata"), + pytest.param( + {"aaa-0.dist-info/METADATA": b"Name: aaa\nVersion: 0\n", **_PIP}, + r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", + id="two-dist-info", + ), + pytest.param({"pip-1.0.dist-info/METADATA": b"Name: pip\n"}, "METADATA lacks Name or Version", id="no-version"), + pytest.param( + {"pip-1.0.dist-info/METADATA": _PIP["pip-1.0.dist-info/METADATA"]}, + "has no pip-1.0.dist-info/RECORD", + id="no-record", + ), + pytest.param( + {"pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n" + b" " * 16 * 1024 * 1024}, + "METADATA is over the 16777216 byte limit", + id="oversized-metadata", + ), + pytest.param( + {**_PIP, "pip-1.0.dist-info/RECORD": b"\n" * (16 * 1024 * 1024 + 1)}, + "RECORD is over the 16777216 byte limit", + id="oversized-record", + ), + ], +) +def test_malformed_bundled_wheel( + tmp_path: Path, write_wheel: WriteWheel, files: Mapping[str, bytes], message: str +) -> None: + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + with pytest.raises(SbomError, match=message): + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +@pytest.mark.parametrize( + ("name", "message"), + [ + pytest.param("pip.whl", "Not a valid wheel filename: pip.whl", id="unparsable"), + pytest.param( + "other-1.0-py3-none-any.whl", r"Wheel \.dist-info directory doesn't match wheel filename", id="other" + ), + ], +) +def test_bundled_wheel_file_name_names_the_project( + tmp_path: Path, write_wheel: WriteWheel, name: str, message: str +) -> None: + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", _PIP) + with pytest.raises(SbomError, match=f"bundled wheel {name} is not a wheel: {message}"): + _bundled(wheel.read_bytes(), name) + + +def test_corrupt_bundled_wheel() -> None: + with pytest.raises(SbomError, match=r"bundled wheel pip\.whl is not a valid zip archive"): + _bundled(b"not a zip", "pip.whl") + + +def test_bundled_wheel_with_duplicate_member(tmp_path: Path) -> None: + # installers extract the first copy, so a RECORD that hashes the second one would hide the installed code + with zipfile.ZipFile(wheel := tmp_path / "pip-1.0-py3-none-any.whl", "w") as archive: + archive.writestr(_INIT, b"import os\n") + with pytest.warns(UserWarning, match="Duplicate name"): + archive.writestr(_INIT, b"") + archive.writestr(_METADATA, _PIP[_METADATA]) + archive.writestr("pip-1.0.dist-info/RECORD", f"{row(_INIT, b'')}{_METADATA_ROW}") + with pytest.raises(SbomError, match=f"holds more than one member named {_INIT}, ignoring case"): + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +@pytest.mark.parametrize( + ("algorithm", "expected"), + [ + pytest.param("sha256", "SHA-256", id="sha256"), + pytest.param("sha384", "SHA-384", id="sha384"), + pytest.param("sha512", "SHA-512", id="sha512"), + ], +) +def test_bundled_wheel_record_algorithm(tmp_path: Path, write_wheel: WriteWheel, algorithm: str, expected: str) -> None: + record = "".join(row(path, content, algorithm) for path, content in _PIP.items()) + _RECORD_ROW + files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + assert [ + (file.path, file.algorithm, file.digest) + for file in _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files + ] == [(path, expected, hashlib.new(algorithm, content).hexdigest()) for path, content in _PIP.items()] + + +@pytest.mark.parametrize( + ("record", "message"), + [ + pytest.param( + f"{_INIT},sha256=x\n", "RECORD does not parse: Row Index 0: expected 3 elements, got 2", id="columns" + ), + pytest.param( + f"{_METADATA_ROW}\n{_RECORD_ROW}", + "RECORD does not parse: Row Index 1: expected 3 elements, got 0", + id="blank-line", + ), + pytest.param( + f"{_INIT},,0\n{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD disagrees with the wheel: hash / size of {_INIT} is not included in RECORD", + id="no-digest", + ), + pytest.param( + f"{row(_INIT, b'')}{_METADATA_ROW}", + "RECORD disagrees with the wheel: pip-1.0.dist-info/RECORD is not mentioned in RECORD", + id="record-not-recorded", + ), + pytest.param( + f"{row(_INIT, b'', 'sha224')}{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD row for {_INIT}: unknown hash algorithm 'sha224'", + id="algorithm-cyclonedx-lacks", + ), + # installer would call digest() without the length shake_128 needs and raise a TypeError + pytest.param( + f"{_INIT},shake_128=AAAA,0\n{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD row for {_INIT}: unknown hash algorithm 'shake_128'", + id="algorithm-needing-a-length", + ), + pytest.param(_METADATA_ROW, f"RECORD lacks a digest for {_INIT}", id="member-not-recorded"), + pytest.param( + f"{row(_INIT, b'', encoded='A' * 43)}{row(_INIT, b'')}{_METADATA_ROW}", + f"RECORD lists {_INIT} more than once", + id="listed-twice", + ), + pytest.param( + f"{row(_INIT, b'')}{row(_INIT, b'', 'sha512', encoded='A' * 86)}{_METADATA_ROW}", + f"RECORD lists {_INIT} more than once", + id="listed-twice-by-other-algorithm", + ), + pytest.param( + f"{row(_INIT, b'')}{row('pip/ghost.py', b'', 'md5', size=99)}{_METADATA_ROW}", + "RECORD lists pip/ghost.py, which the wheel does not hold", + id="recorded-not-member", + ), + pytest.param( + f"{row(_INIT, b'', size=999999)}{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD disagrees with the wheel: hash / size of {_INIT} didn't match RECORD", + id="size-mismatch", + ), + pytest.param( + f"{row(_INIT, b'', 'sha512', encoded='A' * 86)}{_METADATA_ROW}{_RECORD_ROW}", + f"RECORD disagrees with the wheel: hash / size of {_INIT} didn't match RECORD", + id="digest-mismatch", + ), + ], +) +def test_record_disagrees_with_wheel(tmp_path: Path, write_wheel: WriteWheel, record: str, message: str) -> None: + files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode()} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + with pytest.raises(SbomError, match=message): + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +def test_record_signatures_need_no_row(tmp_path: Path, write_wheel: WriteWheel) -> None: + record = "".join(starmap(row, _PIP.items())) + _RECORD_ROW + files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), "pip-1.0.dist-info/RECORD.jws": b"{}"} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + assert [file.path for file in _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl").files] == list(_PIP) + + +@pytest.mark.parametrize( + "signature", + [ + pytest.param("pip-1.0.dist-info/sub/RECORD.jws", id="nested"), + pytest.param("pip-1.0.dist-info/../pip/RECORD.p7s", id="parent"), + ], +) +def test_signature_away_from_record_needs_a_row(tmp_path: Path, write_wheel: WriteWheel, signature: str) -> None: + # installer lets these go unlisted, though a signature only sits beside RECORD + record = "".join(starmap(row, _PIP.items())) + _RECORD_ROW + files = {**_PIP, "pip-1.0.dist-info/RECORD": record.encode(), signature: b"import os\n"} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files, record=False) + with pytest.raises(SbomError, match=f"RECORD lacks a digest for {re.escape(signature)}"): + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +def test_vendored_metadata_needs_name_and_version(tmp_path: Path, write_wheel: WriteWheel) -> None: + files = {**_PIP, "pip/_vendor/rich-13.0.dist-info/METADATA": b"Metadata-Version: 2.1\n"} + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", files) + with pytest.raises(SbomError, match=r"pip/_vendor/rich-13\.0\.dist-info/METADATA lacks Name or Version"): + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +@pytest.mark.parametrize( + ("manifest", "message"), + [ + pytest.param(b"urllib3 ==\n", "does not parse: urllib3 ==", id="unparsable"), + pytest.param(b"urllib3>=1\n", "pins no one exact version: urllib3>=1", id="range"), + pytest.param(b"urllib3==1.*\n", "pins no one exact version: urllib3==1.*", id="wildcard"), + pytest.param(b"urllib3==1,!=1.1\n", "pins no one exact version: urllib3==1,!=1.1", id="two-specifiers"), + ], +) +def test_vendor_manifest_needs_exact_pins( + tmp_path: Path, write_wheel: WriteWheel, manifest: bytes, message: str +) -> None: + wheel = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", {**_PIP, "pip/_vendor/vendor.txt": manifest}) + with pytest.raises(SbomError, match=message): + _bundled(wheel.read_bytes(), "pip-1.0-py3-none-any.whl") + + +def _bundled(data: bytes, path: str) -> BundledWheel: + return read_bundled_wheels([path], lambda _: BytesIO(data))[0] + + +def test_bundled_wheels_come_in_the_order_given(tmp_path: Path, write_wheel: WriteWheel) -> None: + # read side by side, yet reported in order, the first broken wheel's error included + good = write_wheel(tmp_path / "pip-1.0-py3-none-any.whl", _PIP).read_bytes() + wheels = {"pip-1.0-py3-none-any.whl": good, "b.whl": b"not a zip", "c.whl": b"neither"} + paths = ["pip-1.0-py3-none-any.whl", "b.whl", "c.whl"] + assert [bundled.path for bundled in read_bundled_wheels(paths[:1] * 2, lambda name: BytesIO(wheels[name]))] == [ + paths[0], + paths[0], + ] + with pytest.raises(SbomError, match=r"^bundled wheel b\.whl is not a valid zip archive"): + read_bundled_wheels(paths, lambda name: BytesIO(wheels[name])) From eb528e9d4a325380f2f07d79cf13e73b4d4013fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 09:09:40 -0800 Subject: [PATCH 111/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20name?= =?UTF-8?q?=20bundled=20wheels=20while=20reading=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The built wheel's .whl members are read by _wheel and handed to the source tree reader, so embed no longer rebuilds the source tree to rename them. --- docs/explanation.rst | 4 +--- src/python_sbom/_collect/_source.py | 11 +++++++++-- src/python_sbom/_generate/_embed.py | 15 ++------------- src/python_sbom/_model/_archive.py | 3 ++- src/python_sbom/_wheel/_archive.py | 7 +++++++ tests/_collect/test_source.py | 10 +++++----- tests/_wheel/test_bundled.py | 2 +- 7 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/explanation.rst b/docs/explanation.rst index 36b1be4..de8028a 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -28,7 +28,7 @@ that command runs. collect[_collect
settings, facts, revision, source tree, archives, providers] validate[_validate
limits, schema, references, licenses] formats[_formats
CycloneDX JSON, SPDX 2.3] - wheel[_wheel
RECORD rows, member reads, rewrite] + wheel[_wheel
RECORD rows, member reads, bundled wheel checks, rewrite] model[_model
inputs, types, identity, document building] errors[_errors
SbomError] version[version] @@ -36,10 +36,8 @@ that command runs. hook -. lazy .-> generate hook -. types .-> model cli --> generate - cli --> collect cli --> validate cli --> formats - cli --> wheel cli -. types .-> model api --> cli api --> model diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index 019efc0..3791a5e 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import hashlib from pathlib import PurePath from typing import TYPE_CHECKING, Final @@ -10,12 +11,15 @@ from python_sbom._wheel._bundled import read_bundled_wheels if TYPE_CHECKING: + from collections.abc import Mapping from pathlib import Path, PurePosixPath from python_sbom._model._inputs import Settings -def read_source_tree(request: SbomRequest) -> SourceTree: +def read_source_tree(request: SbomRequest, *, inside: Mapping[str, str]) -> SourceTree: + # inside maps a wheel's sha256 to its path in the built artifact: a bundled wheel the artifact holds is named by + # that path, as an archive SBOM names it, so both documents give one wheel the same identity root = request.root wheels: Final[set[str]] = set() for pattern in request.settings.bundled_wheels: @@ -26,7 +30,10 @@ def read_source_tree(request: SbomRequest) -> SourceTree: wheels.update(wheel.relative_to(root).as_posix() for wheel in root.glob(pattern)) return SourceTree( copyright=copyright_notice(root, request.metadata.values("license_files")), - bundled=read_bundled_wheels(sorted(wheels), lambda path: (root / path).open("rb")), + bundled=[ + dataclasses.replace(wheel, path=inside.get(wheel.sha256, wheel.path)) + for wheel in read_bundled_wheels(sorted(wheels), lambda path: (root / path).open("rb")) + ], # str, so a Windows drive in an absolute path survives the join with the native root build_scripts=[ BuildScript(path=script, sha256=hashlib.sha256((root / str(script)).read_bytes()).hexdigest()) diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py index 2fff5a7..74c466a 100644 --- a/src/python_sbom/_generate/_embed.py +++ b/src/python_sbom/_generate/_embed.py @@ -1,6 +1,5 @@ from __future__ import annotations -import dataclasses from dataclasses import dataclass from pathlib import PurePosixPath from typing import TYPE_CHECKING @@ -17,7 +16,7 @@ from python_sbom._model._document import checked_contributions from python_sbom._model._identity import project_identity from python_sbom._model._names import sbom_file_name -from python_sbom._wheel._archive import member_digest, open_archive, write_sbom +from python_sbom._wheel._archive import wheel_members, write_sbom if TYPE_CHECKING: from collections.abc import Sequence @@ -47,17 +46,7 @@ def embed_sbom( def embed_into(wheel: Path, request: SbomRequest, content: ContentProvider, facts: BuildFacts) -> str: - source = read_source_tree(request) - with open_archive(wheel, str(wheel)) as archive: - inside = {member_digest(archive, name, "sha256"): name for name in archive.namelist() if name.endswith(".whl")} - # a bundled wheel the built wheel holds is named by its path in there, as an archive SBOM names it, so both - # documents give one wheel the same identity - source = dataclasses.replace( - source, - bundled=[ - dataclasses.replace(bundled, path=inside.get(bundled.sha256, bundled.path)) for bundled in source.bundled - ], - ) + source = read_source_tree(request, inside=wheel_members(wheel)) project = project_identity(request.metadata, request.settings) context = ProviderContext(root=request.root, settings=request.settings, identity=project, read=member_reader(wheel)) sbom = build_sbom(request, source, facts, content, context) diff --git a/src/python_sbom/_model/_archive.py b/src/python_sbom/_model/_archive.py index babf93f..2e825b3 100644 --- a/src/python_sbom/_model/_archive.py +++ b/src/python_sbom/_model/_archive.py @@ -12,8 +12,9 @@ if TYPE_CHECKING: from collections.abc import Mapping + from ._content import BundledWheelInfo from ._identity import ProjectIdentity - from ._inputs import ArchiveTree, BuildScript, BundledWheelInfo + from ._inputs import ArchiveTree, BuildScript from ._types import Component diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index e96346b..4907be0 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -148,6 +148,12 @@ def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _M return bytes(content) +def wheel_members(wheel: Path) -> dict[str, str]: + # the .whl members an artifact holds, by their sha256 + with open_archive(wheel, str(wheel)) as archive: + return {member_digest(archive, name, "sha256"): name for name in archive.namelist() if name.endswith(".whl")} + + def member_digest(archive: zipfile.ZipFile, name: str | zipfile.ZipInfo, algorithm: str) -> str: digest: Final = hashlib.new(algorithm) with archive.open(name) as member: @@ -172,5 +178,6 @@ def append_member(archive: Path, name: str, content: bytes, epoch: int) -> None: "open_archive", "read_member", "read_wheel_metadata", + "wheel_members", "write_sbom", ] diff --git a/tests/_collect/test_source.py b/tests/_collect/test_source.py index 329d359..19ce26a 100644 --- a/tests/_collect/test_source.py +++ b/tests/_collect/test_source.py @@ -38,7 +38,7 @@ def test_copyright(tmp_path: Path, settings: Settings, files: Mapping[str, str], expected: str | None) -> None: for name, text in files.items(): (tmp_path / name).write_text(text, encoding="utf-8") - source = read_source_tree(request(tmp_path, settings, "License-File: LICENSE\nLicense-File: NOTICE\n")) + source = read_source_tree(request(tmp_path, settings, "License-File: LICENSE\nLicense-File: NOTICE\n"), inside={}) assert source.copyright == expected @@ -56,7 +56,7 @@ def test_bundled_wheels_sorted_once(tmp_path: Path, settings: Settings, write_wh for directory in ("b", "a"): write_wheel(tmp_path / "src" / directory / "pip-1.0-py3-none-any.whl", _PIP) settings = dataclasses.replace(settings, bundled_wheels=("src/*/*.whl", "src/a/*.whl")) - paths = [wheel.path for wheel in read_source_tree(request(tmp_path, settings)).bundled] + paths = [wheel.path for wheel in read_source_tree(request(tmp_path, settings), inside={}).bundled] assert paths == ["src/a/pip-1.0-py3-none-any.whl", "src/b/pip-1.0-py3-none-any.whl"] @@ -76,13 +76,13 @@ def test_bundled_wheel_pattern_outside_project( pattern = pattern.replace("ROOT", tmp_path.as_posix()) settings = dataclasses.replace(settings, bundled_wheels=(pattern,)) with pytest.raises(SbomError, match=f"`bundled-wheels` pattern {re.escape(repr(pattern))} must be relative"): - read_source_tree(request(project.parent, settings)) + read_source_tree(request(project.parent, settings), inside={}) def test_build_scripts_hashed(tmp_path: Path, settings: Settings) -> None: (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") script = PurePosixPath("hatch_build.py") - source = read_source_tree(request(tmp_path, settings, scripts=(script,))) + source = read_source_tree(request(tmp_path, settings, scripts=(script,)), inside={}) assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] @@ -90,7 +90,7 @@ def test_build_script_outside_project(tmp_path: Path, settings: Settings) -> Non # a custom hook path may be absolute, which on Windows carries a drive the join with the root must keep (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") script = PurePosixPath((tmp_path / "hatch_build.py").as_posix()) - source = read_source_tree(request(tmp_path / "project", settings, scripts=(script,))) + source = read_source_tree(request(tmp_path / "project", settings, scripts=(script,)), inside={}) assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] diff --git a/tests/_wheel/test_bundled.py b/tests/_wheel/test_bundled.py index b3226d0..d05a42f 100644 --- a/tests/_wheel/test_bundled.py +++ b/tests/_wheel/test_bundled.py @@ -6,7 +6,6 @@ import zipfile from io import BytesIO from itertools import starmap -from pathlib import Path from typing import TYPE_CHECKING, Final import pytest @@ -17,6 +16,7 @@ if TYPE_CHECKING: from collections.abc import Mapping + from pathlib import Path from conftest import WriteWheel From dd1eb38d52267ff0386a4d11db6c68fd17d058ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 09:14:11 -0800 Subject: [PATCH 112/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20ask=20?= =?UTF-8?q?providers=20in=20component=20order?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keeps the order the archive path asked content providers in before the model split, names the digest lookup for what it returns, and lets the benchmark find the bundled wheel reader in _wheel. --- bench/_cases.py | 7 ++++++- src/python_sbom/_generate/_embed.py | 4 ++-- src/python_sbom/_model/_archive.py | 13 +++++++++---- src/python_sbom/_wheel/_archive.py | 5 ++--- tests/_generate/test_archive.py | 14 ++++++++++++++ 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/bench/_cases.py b/bench/_cases.py index 4fd4eb0..4d93dd0 100644 --- a/bench/_cases.py +++ b/bench/_cases.py @@ -145,6 +145,11 @@ def _bundled(key: str) -> Callable[[dict[str, str], Path], Step]: def make(fixtures: dict[str, str], scratch: Path) -> Step: # ruff: ignore[unused-function-argument] # one signature from python_sbom._collect import _source # ruff: ignore[import-outside-top-level] # see _validate + try: + from python_sbom._wheel import _bundled as bundled # ruff: ignore[import-outside-top-level] # see _validate + except ImportError: # revisions before the move keep the reader in _collect._source + bundled = _source + wheel = Path(fixtures[key]) data = wheel.read_bytes() # revisions before the batch reader take one wheel's bytes; later ones a list and a way to open each @@ -154,7 +159,7 @@ def run() -> None: if single is not None: single(data, wheel.name) else: - _source.read_bundled_wheels([wheel.name], lambda _name: io.BytesIO(data)) + bundled.read_bundled_wheels([wheel.name], lambda _name: io.BytesIO(data)) return Step(run=run) diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py index 74c466a..089da3b 100644 --- a/src/python_sbom/_generate/_embed.py +++ b/src/python_sbom/_generate/_embed.py @@ -16,7 +16,7 @@ from python_sbom._model._document import checked_contributions from python_sbom._model._identity import project_identity from python_sbom._model._names import sbom_file_name -from python_sbom._wheel._archive import wheel_members, write_sbom +from python_sbom._wheel._archive import bundled_wheels_by_digest, write_sbom if TYPE_CHECKING: from collections.abc import Sequence @@ -46,7 +46,7 @@ def embed_sbom( def embed_into(wheel: Path, request: SbomRequest, content: ContentProvider, facts: BuildFacts) -> str: - source = read_source_tree(request, inside=wheel_members(wheel)) + source = read_source_tree(request, inside=bundled_wheels_by_digest(wheel)) project = project_identity(request.metadata, request.settings) context = ProviderContext(root=request.root, settings=request.settings, identity=project, read=member_reader(wheel)) sbom = build_sbom(request, source, facts, content, context) diff --git a/src/python_sbom/_model/_archive.py b/src/python_sbom/_model/_archive.py index 2e825b3..bda21d5 100644 --- a/src/python_sbom/_model/_archive.py +++ b/src/python_sbom/_model/_archive.py @@ -18,7 +18,7 @@ from ._types import Component -def archive_artifact( # ruff: ignore[too-many-arguments] # what the edge read from the project, keyword-only +def archive_artifact( # ruff: ignore[too-many-arguments] # keyword-only, each a separate fact the edge read name: str, tree: ArchiveTree, *, @@ -96,10 +96,15 @@ def _directory(prefix: str) -> PurePosixPath: def bundled_paths(tree: ArchiveTree) -> list[PurePosixPath]: - # what a content provider is asked about: each wheel bundled unpacked, by its directory, and each .whl member + # what a content provider is asked about, in the order the components list them: each wheel bundled unpacked, by its + # directory, followed by the .whl members it holds return [ - *(_directory(prefix) for prefix in sorted(tree.metadata) if prefix), - *map(PurePosixPath, sorted(tree.bundled)), + path + for prefix in sorted(tree.metadata) + for path in ( + *([_directory(prefix)] if prefix else []), + *(PurePosixPath(name) for name in tree.owned[prefix] if name.endswith(".whl")), + ) ] diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index 4907be0..0a42054 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -148,8 +148,7 @@ def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _M return bytes(content) -def wheel_members(wheel: Path) -> dict[str, str]: - # the .whl members an artifact holds, by their sha256 +def bundled_wheels_by_digest(wheel: Path) -> dict[str, str]: with open_archive(wheel, str(wheel)) as archive: return {member_digest(archive, name, "sha256"): name for name in archive.namelist() if name.endswith(".whl")} @@ -174,10 +173,10 @@ def append_member(archive: Path, name: str, content: bytes, epoch: int) -> None: __all__ = [ "append_member", + "bundled_wheels_by_digest", "member_digest", "open_archive", "read_member", "read_wheel_metadata", - "wheel_members", "write_sbom", ] diff --git a/tests/_generate/test_archive.py b/tests/_generate/test_archive.py index 96fc8e2..249cf02 100644 --- a/tests/_generate/test_archive.py +++ b/tests/_generate/test_archive.py @@ -232,3 +232,17 @@ def test_archive_without_project_distribution(tmp_path: Path) -> None: describe_archive( tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None)), provider=None, build_scripts=() ) + + +def test_archive_asks_provider_in_component_order(project: Path, archive: Path) -> None: + asked: Final[list[str]] = [] + + class Recording(ContentProvider): + def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> BundledWheelInfo: + asked.append(str(wheel)) + return super().bundled_wheel(context, wheel) + + describe_archive( + project, archive, facts=collect_facts(project, revision_reader(None)), provider=Recording(), build_scripts=() + ) + assert asked == [_WHEEL, _LIBRARY] From 79f8dde98ed16e5a3ff23abbbbff682a6657038a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 11:42:37 -0800 Subject: [PATCH 113/115] =?UTF-8?q?=F0=9F=90=9B=20fix(archive):=20refuse?= =?UTF-8?q?=20or=20describe=20odd=20bundled=20layouts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two .dist-info in one directory left one distribution out of the archive SBOM and gave its files to the other, so that layout is now refused. Identical bundled wheel copies took one path in the built wheel and broke the build on a repeated bom-ref; each copy now takes its own. A .whl at the top level of an archive crashed generation; it is now described as a bundled wheel under the root component. --- src/python_sbom/_collect/_archive.py | 19 +++++++++++++------ src/python_sbom/_collect/_source.py | 12 +++++++----- src/python_sbom/_model/_archive.py | 28 +++++++++++++++++----------- src/python_sbom/_wheel/_archive.py | 8 ++++++-- tests/_generate/test_archive.py | 26 ++++++++++++++++++++++++++ tests/_generate/test_embed.py | 21 +++++++++++++++++++++ 6 files changed, 90 insertions(+), 24 deletions(-) diff --git a/src/python_sbom/_collect/_archive.py b/src/python_sbom/_collect/_archive.py index 10aab50..c680269 100644 --- a/src/python_sbom/_collect/_archive.py +++ b/src/python_sbom/_collect/_archive.py @@ -27,11 +27,18 @@ def read_archive(archive: Path) -> ArchiveTree: members: Final[list[str]] = sorted( info.filename for info in handle.infolist() if not info.is_dir() and info.filename != f"{where}.cdx.json" ) - metadata: Final[dict[str, CoreMetadata]] = { - _directory(name): parse_metadata(read_member(handle, name, where).decode("utf-8")) - for name in members - if name.endswith(".dist-info/METADATA") - } + metadata: Final[dict[str, CoreMetadata]] = {} + for name in members: + if not name.endswith(".dist-info/METADATA"): + continue + # files belong to a distribution by the directory that holds its .dist-info, which two cannot share + if (directory := _directory(name)) in metadata: + msg = ( + f"{where} holds more than one .dist-info in {directory or 'its top level'}, so their files cannot" + " be told apart" + ) + raise SbomError(msg) + metadata[directory] = parse_metadata(read_member(handle, name, where).decode("utf-8")) if (top := sum(name.count("/") == 1 and name.endswith(".dist-info/METADATA") for name in members)) != 1: msg = f"{where} holds {top} top-level .dist-info/METADATA, not the one naming the project" raise SbomError(msg) @@ -56,7 +63,7 @@ def read_archive(archive: Path) -> ArchiveTree: else: loose.append(name) # in the order the components list them, so the first broken wheel is the one reported - wheels: Final[list[str]] = [] + wheels: Final[list[str]] = [name for name in loose if name.endswith(".whl")] for prefix in sorted(metadata): wheels.extend(name for name in owned[prefix] if name.endswith(".whl")) return ArchiveTree( diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index 3791a5e..ad16c02 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -11,15 +11,16 @@ from python_sbom._wheel._bundled import read_bundled_wheels if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Iterator, Mapping, Sequence from pathlib import Path, PurePosixPath from python_sbom._model._inputs import Settings -def read_source_tree(request: SbomRequest, *, inside: Mapping[str, str]) -> SourceTree: - # inside maps a wheel's sha256 to its path in the built artifact: a bundled wheel the artifact holds is named by - # that path, as an archive SBOM names it, so both documents give one wheel the same identity +def read_source_tree(request: SbomRequest, *, inside: Mapping[str, Sequence[str]]) -> SourceTree: + # inside maps a wheel's sha256 to its paths in the built artifact: a bundled wheel the artifact holds is named by + # its path there, as an archive SBOM names it, so both documents give one wheel the same identity; identical copies + # take those paths in turn, so each keeps an identity of its own root = request.root wheels: Final[set[str]] = set() for pattern in request.settings.bundled_wheels: @@ -28,10 +29,11 @@ def read_source_tree(request: SbomRequest, *, inside: Mapping[str, str]) -> Sour msg = f"`bundled-wheels` pattern {pattern!r} must be relative to the project, without `..`" raise SbomError(msg) wheels.update(wheel.relative_to(root).as_posix() for wheel in root.glob(pattern)) + destinations: Final[dict[str, Iterator[str]]] = {digest: iter(paths) for digest, paths in inside.items()} return SourceTree( copyright=copyright_notice(root, request.metadata.values("license_files")), bundled=[ - dataclasses.replace(wheel, path=inside.get(wheel.sha256, wheel.path)) + dataclasses.replace(wheel, path=next(destinations.get(wheel.sha256, iter(())), wheel.path)) for wheel in read_bundled_wheels(sorted(wheels), lambda path: (root / path).open("rb")) ], # str, so a Windows drive in an absolute path survives the join with the native root diff --git a/src/python_sbom/_model/_archive.py b/src/python_sbom/_model/_archive.py index bda21d5..2357858 100644 --- a/src/python_sbom/_model/_archive.py +++ b/src/python_sbom/_model/_archive.py @@ -41,7 +41,10 @@ def archive_artifact( # ruff: ignore[too-many-arguments] # keyword-only, each a "name": name, "version": version, "purl": ref, - "components": [file_component(ref, tree.files[member]) for member in tree.loose], + "components": [ + distributions.wheel(member) if member.endswith(".whl") else file_component(ref, tree.files[member]) + for member in tree.loose + ], }, components=components, dependencies=vendored_dependencies(components), @@ -79,12 +82,12 @@ def component(self, prefix: str) -> Component: *component["properties"], ] component["components"] = [ - self._wheel(name) if name.endswith(".whl") else file_component(component["bom-ref"], self.tree.files[name]) + self.wheel(name) if name.endswith(".whl") else file_component(component["bom-ref"], self.tree.files[name]) for name in self.tree.owned[prefix] ] return component - def _wheel(self, name: str) -> Component: + def wheel(self, name: str) -> Component: info = self.bundled_info[name] return bundled_wheel_component( self.tree.bundled[name], self.project.property_prefix, info.properties, info.supplier @@ -96,15 +99,18 @@ def _directory(prefix: str) -> PurePosixPath: def bundled_paths(tree: ArchiveTree) -> list[PurePosixPath]: - # what a content provider is asked about, in the order the components list them: each wheel bundled unpacked, by its - # directory, followed by the .whl members it holds + # what a content provider is asked about, in the order the components list them: the archive's own top-level .whl + # members, then each wheel bundled unpacked, by its directory, followed by the .whl members it holds return [ - path - for prefix in sorted(tree.metadata) - for path in ( - *([_directory(prefix)] if prefix else []), - *(PurePosixPath(name) for name in tree.owned[prefix] if name.endswith(".whl")), - ) + *(PurePosixPath(name) for name in tree.loose if name.endswith(".whl")), + *( + path + for prefix in sorted(tree.metadata) + for path in ( + *([_directory(prefix)] if prefix else []), + *(PurePosixPath(name) for name in tree.owned[prefix] if name.endswith(".whl")), + ) + ), ] diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_archive.py index 0a42054..583ee07 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_archive.py @@ -148,9 +148,13 @@ def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _M return bytes(content) -def bundled_wheels_by_digest(wheel: Path) -> dict[str, str]: +def bundled_wheels_by_digest(wheel: Path) -> dict[str, list[str]]: + found: Final[dict[str, list[str]]] = {} with open_archive(wheel, str(wheel)) as archive: - return {member_digest(archive, name, "sha256"): name for name in archive.namelist() if name.endswith(".whl")} + for name in sorted(archive.namelist()): + if name.endswith(".whl"): + found.setdefault(member_digest(archive, name, "sha256"), []).append(name) + return found def member_digest(archive: zipfile.ZipFile, name: str | zipfile.ZipInfo, algorithm: str) -> str: diff --git a/tests/_generate/test_archive.py b/tests/_generate/test_archive.py index 249cf02..3a50b77 100644 --- a/tests/_generate/test_archive.py +++ b/tests/_generate/test_archive.py @@ -246,3 +246,29 @@ def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> Bundl project, archive, facts=collect_facts(project, revision_reader(None)), provider=Recording(), build_scripts=() ) assert asked == [_WHEEL, _LIBRARY] + + +def test_archive_top_level_wheel_belongs_to_the_archive(project: Path, tmp_path: Path, write_wheel: WriteWheel) -> None: + wheel = write_wheel( + tmp_path / "pip-1.0-py3-none-any.whl", {"pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n"} + ) + with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w") as handle: + handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") + handle.writestr(wheel.name, wheel.read_bytes()) + root = describe_archive( + project, archive, facts=collect_facts(project, revision_reader(None)), provider=None, build_scripts=() + )["metadata"]["component"] + assert [(child["bom-ref"], child["name"]) for child in root.get("components", [])] == [ + (f"bundled:{wheel.name}", "pip") + ] + + +def test_archive_refuses_distributions_sharing_a_directory(project: Path, tmp_path: Path) -> None: + with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w") as handle: + handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") + for name in ("bar", "foo"): + handle.writestr(f"vendor/{name}-1.0.dist-info/METADATA", f"Name: {name}\nVersion: 1.0\n") + with pytest.raises(SbomError, match=r"app\.pyz holds more than one \.dist-info in vendor/, so their files cannot"): + describe_archive( + project, archive, facts=collect_facts(project, revision_reader(None)), provider=None, build_scripts=() + ) diff --git a/tests/_generate/test_embed.py b/tests/_generate/test_embed.py index 69e8eae..324bdc3 100644 --- a/tests/_generate/test_embed.py +++ b/tests/_generate/test_embed.py @@ -60,6 +60,27 @@ def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> Bundl return BundledWheelInfo(properties=[{"name": f"{context.identity.property_prefix}:table", "value": value}]) +def test_identical_bundled_copies_keep_their_own_paths(tmp_path: Path, write_wheel: WriteWheel) -> None: + (root := tmp_path / "project").mkdir() + (root / "pyproject.toml").write_text("[tool.python-sbom]\nbundled-wheels = ['src/*/wheels/*.whl']\n") + pip = {"pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\nLicense-Expression: MIT\n"} + for package in ("a", "b"): + content = write_wheel(root / "src" / package / "wheels" / "pip-1.0-py3-none-any.whl", pip).read_bytes() + wheel = write_wheel( + tmp_path / "dist" / "pkg-1.0-py3-none-any.whl", + { + "a/wheels/pip-1.0-py3-none-any.whl": content, + "b/wheels/pip-1.0-py3-none-any.whl": content, + "pkg-1.0.dist-info/METADATA": b"Name: pkg\nVersion: 1.0\n", + }, + ) + written = embed_sbom(wheel, root, provider=None, commit=None, build_env=[str(root)]) + assert [component["bom-ref"] for component in json.loads(zipfile.ZipFile(wheel).read(written))["components"]] == [ + "bundled:a/wheels/pip-1.0-py3-none-any.whl", + "bundled:b/wheels/pip-1.0-py3-none-any.whl", + ] + + def test_embed_output_is_pinned( tmp_path: Path, write_wheel: WriteWheel, install: Install, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture ) -> None: From e8ca8f60b922d086186183660a117d7e42d2e15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 15:38:54 -0800 Subject: [PATCH 114/115] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20share?= =?UTF-8?q?=20artifact=20reads=20and=20graph=20rules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the second external review and restructures for review. The built artifact is now the only source of the bytes an SBOM describes: one reader keeps it open for the whole build, and bundled wheels are found in it by digest, so a source wheel changed or left out since the build is refused. One module defines what an artifact holds and ships and checks references, so generation, validation and the license policy judge the same set: top-level archive wheels meet the policy and get their vendored edges, and a repeated dependency entry is refused. A provider module another project already imported is refused. The wheel and archive pipelines now read alike: open the artifact, read it into a tree, map the tree to an Artifact with a pure function, assemble, write. Zip reading and writing are separate modules, one helper hashes, and the archive SBOM file name, bundled order and bundled marking each live in one place. Also: archive refuses to write over its own input, SPDX ids carry the full digest, RECORD rows match by parsed path, the scheduled check runs, malformed tools fail cleanly, archive ownership looks up ancestors, and docs/contributing.rst names what each layer guarantees and the test that holds it. --- .github/workflows/check.yaml | 2 +- bench/_cases.py | 27 ++--- docs/contributing.rst | 36 +++++- docs/explanation.rst | 15 +-- src/python_sbom/_cli/_spdx.py | 28 ++--- src/python_sbom/_collect/__init__.py | 2 +- src/python_sbom/_collect/_archive.py | 112 ++++++++---------- src/python_sbom/_collect/_content.py | 28 +++++ src/python_sbom/_collect/_source.py | 67 ++++------- src/python_sbom/_collect/_wheel.py | 55 +++++++++ src/python_sbom/_formats/_spdx.py | 41 +++---- src/python_sbom/_generate/_archive.py | 47 ++++---- src/python_sbom/_generate/_embed.py | 80 ++++++------- src/python_sbom/_model/_archive.py | 79 ++++++------ src/python_sbom/_model/_bom.py | 97 +-------------- src/python_sbom/_model/_components.py | 40 ++++--- src/python_sbom/_model/_document.py | 43 ++----- src/python_sbom/_model/_graph.py | 64 ++++++++++ src/python_sbom/_model/_inputs.py | 21 ++-- src/python_sbom/_model/_names.py | 6 + src/python_sbom/_model/_wheel.py | 89 ++++++++++++++ src/python_sbom/_validate/_artifact.py | 7 +- src/python_sbom/_validate/_document.py | 71 ++++++----- src/python_sbom/_validate/_licenses.py | 17 +-- src/python_sbom/_validate/_limits.py | 9 +- src/python_sbom/_validate/_schemas.py | 4 +- src/python_sbom/_wheel/__init__.py | 2 +- src/python_sbom/_wheel/_bundled.py | 13 +- src/python_sbom/_wheel/_reader.py | 59 +++++++++ src/python_sbom/_wheel/_record.py | 5 +- .../_wheel/{_archive.py => _write.py} | 99 ++-------------- src/python_sbom/_wheel/_zip.py | 94 +++++++++++++++ src/python_sbom/hatch/_hook.py | 8 +- tests/_cli/test_archive.py | 59 +++++++++ tests/_cli/test_spdx.py | 21 +++- tests/_collect/test_archive.py | 32 ++--- tests/_collect/test_content.py | 26 ++++ tests/_collect/test_source.py | 91 ++++---------- tests/_collect/test_wheel.py | 112 ++++++++++++++++++ tests/_formats/test_spdx.py | 20 +++- tests/_generate/test_archive.py | 31 ++++- tests/_generate/test_embed.py | 23 +--- tests/_model/test_bom.py | 15 +-- tests/_model/test_components.py | 11 +- tests/_validate/test_document.py | 22 ++-- tests/_validate/test_schemas.py | 2 +- tests/_wheel/test_reader.py | 36 ++++++ tests/_wheel/test_record.py | 28 +++-- .../_wheel/{test_archive.py => test_write.py} | 108 +++++++---------- tests/_wheel/test_zip.py | 82 +++++++++++++ tests/conftest.py | 67 +++++++---- 51 files changed, 1309 insertions(+), 844 deletions(-) create mode 100644 src/python_sbom/_collect/_wheel.py create mode 100644 src/python_sbom/_model/_graph.py create mode 100644 src/python_sbom/_model/_wheel.py create mode 100644 src/python_sbom/_wheel/_reader.py rename src/python_sbom/_wheel/{_archive.py => _write.py} (51%) create mode 100644 src/python_sbom/_wheel/_zip.py create mode 100644 tests/_collect/test_wheel.py create mode 100644 tests/_wheel/test_reader.py rename tests/_wheel/{test_archive.py => test_write.py} (80%) create mode 100644 tests/_wheel/test_zip.py diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index ad01f2c..e0d659e 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -15,7 +15,7 @@ permissions: jobs: test: name: ๐Ÿงช test - if: github.event_name != 'schedule' || github.repository_owner == 'gaborbernat' + if: github.event_name != 'schedule' || github.repository == 'tox-dev/python-sbom' uses: tox-dev/github-workflows/.github/workflows/check.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 permissions: contents: read diff --git a/bench/_cases.py b/bench/_cases.py index 4d93dd0..4f6505a 100644 --- a/bench/_cases.py +++ b/bench/_cases.py @@ -9,6 +9,8 @@ from __future__ import annotations import contextlib +import importlib +import importlib.util import inspect import io import os @@ -143,25 +145,16 @@ def run() -> None: def _bundled(key: str) -> Callable[[dict[str, str], Path], Step]: def make(fixtures: dict[str, str], scratch: Path) -> Step: # ruff: ignore[unused-function-argument] # one signature - from python_sbom._collect import _source # ruff: ignore[import-outside-top-level] # see _validate - - try: - from python_sbom._wheel import _bundled as bundled # ruff: ignore[import-outside-top-level] # see _validate - except ImportError: # revisions before the move keep the reader in _collect._source - bundled = _source - + # revisions before the move keep the reader in _collect._source, and those before the batch reader take one + # wheel's bytes rather than a list and a way to open each + moved = importlib.util.find_spec("python_sbom._wheel._bundled") is not None + module = importlib.import_module("python_sbom._wheel._bundled" if moved else "python_sbom._collect._source") wheel = Path(fixtures[key]) data = wheel.read_bytes() - # revisions before the batch reader take one wheel's bytes; later ones a list and a way to open each - single = getattr(_source, "read_bundled_wheel", None) - - def run() -> None: - if single is not None: - single(data, wheel.name) - else: - bundled.read_bundled_wheels([wheel.name], lambda _name: io.BytesIO(data)) - - return Step(run=run) + if (single := getattr(module, "read_bundled_wheel", None)) is not None: + return Step(run=lambda: single(data, wheel.name)) + batch = module.read_bundled_wheels + return Step(run=lambda: batch([wheel.name], lambda _name: io.BytesIO(data))) return make diff --git a/docs/contributing.rst b/docs/contributing.rst index 6309c52..0f2c0f0 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -4,10 +4,11 @@ Contributing How the code fits together -------------------------- -Every SBOM comes out of one pipeline. An adapter, the hatchling hook or a command of the command line, gathers what the -edge knows: the request (the wheel's core metadata and file name, the project directory, the options and the build -scripts), the source tree (license files and bundled wheels), and the build facts (time, source revision and toolchain). -It hands them, with a content provider, to the pipeline, which builds the document with pure functions in ``_model``. +Each artifact has one pipeline in ``_generate``, and the two read alike. An adapter, the hatchling hook or a command of +the command line, gathers the options, the content provider, the build scripts and the build facts (time, source +revision and toolchain). The pipeline opens the artifact once, reads it into a tree (``_collect/_wheel.py`` or +``_collect/_archive.py``), maps the tree to an ``Artifact`` with a pure function (``_model/_wheel.py`` or +``_model/_archive.py``), and hands that to ``_model/_bom.py``, which adds what every artifact of the project shares. :doc:`explanation` draws the packages and their imports. To record a new field, read it at the edge into ``_model/_inputs.py`` and write it in ``_model``; to add a subcommand, @@ -15,6 +16,33 @@ write a ``run`` function in a new ``_cli/_.py`` and register its argume ``"python_sbom._cli._:run"`` handler in ``_cli/_parser.py``, which loads the module only when the command runs. Tests mirror the source: ``tests//test_.py``. +What each layer guarantees +-------------------------- + +A change is easiest to review against the guarantee of the layer it touches, and the test named here is the one that +fails when that guarantee breaks. Each test crosses a boundary on purpose: unit tests of one layer did not catch the +mismatches between layers these guard against. + +- ``_wheel`` reads an artifact through one open handle per build, checked once for clashing member names, and never + trusts a member's stated size. The bytes it hands on are the bytes the artifact ships. + ``tests/_wheel/test_reader.py``, and ``test_embed_rewrites_record_by_parsed_path`` for rewriting RECORD. +- ``_collect`` turns the project and the artifact into inputs. Source configuration only selects: a bundled wheel is + described from the artifact's copy, matched by digest, and a selected wheel the artifact does not ship is refused. A + provider module another project already imported in the same process is refused rather than reused. + ``test_bundled_wheel_not_shipped_is_refused``, ``test_provider_of_another_project_is_not_reused``. +- ``_model`` builds documents with pure functions. ``_model/_graph.py`` defines the sets every layer judges alike: + ``artifact_contents`` is what the artifact holds or declares wherever the document nests it, ``shipped_libraries`` is + what it carries bytes for, and ``reference_problems`` is the one check of repeated and unknown references that + generation refuses and validation reports. ``test_archive_bundled_wheel_judged_wherever_it_sits``. +- ``_validate`` reads hostile input: bounded size, depth and component count before the schema runs, then references, + the SPDX rendering and the license policy over ``artifact_contents``. + ``test_repeated_dependency_entry_cannot_hide_an_invalid_one``. +- ``_formats`` renders without losing identity: every SPDX id is unique because it ends in the full digest of its + ``bom-ref``. ``test_spdx_ids_of_refs_differing_in_dropped_characters_stay_distinct``. +- ``_generate`` orders reading, building and writing: an artifact stays open while its document is built, is closed + before it is rewritten, and is never overwritten by its own SBOM. ``test_archive_refuses_output_over_archive``, in + ``tests/_cli`` since the command line is where an output path reaches it. + Updating the schemas -------------------- diff --git a/docs/explanation.rst b/docs/explanation.rst index de8028a..3b15a12 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -5,10 +5,10 @@ Architecture ------------ Each concern has its own subpackage, and only the adapters know a build system or the command line. The hatchling hook -and the ``embed`` and ``archive`` commands gather what the edge knows: the request (core metadata, file name, options, -build scripts), the source tree or the archive's members, and the build facts (time, revision, toolchain). They hand -those, with a :class:`~python_sbom.ContentProvider` and its :class:`~python_sbom.ProviderContext`, to the pipeline for -their artifact. Each pipeline maps what it read to components, and one function of the model adds what every artifact +and the ``embed`` and ``archive`` commands gather the options, the build scripts and the build facts (time, revision, +toolchain), and hand them, with a :class:`~python_sbom.ContentProvider`, to the pipeline for their artifact. Each +pipeline opens its artifact once, reads it, gives the provider a :class:`~python_sbom.ProviderContext` that reads from +the same open artifact, and maps what it read to components; one function of the model then adds what every artifact of the project shares: supplier, source links, revision, tools, dependencies and formulation. The diagram follows the imports between the subpackages: a solid arrow points at what a package imports when it loads, a @@ -25,11 +25,11 @@ that command runs. end api[python_sbom
provider API, __main__] generate[_generate
wheel and archive pipelines] - collect[_collect
settings, facts, revision, source tree, archives, providers] + collect[_collect
settings, facts, revision, project tree, wheels, archives, providers] validate[_validate
limits, schema, references, licenses] formats[_formats
CycloneDX JSON, SPDX 2.3] - wheel[_wheel
RECORD rows, member reads, bundled wheel checks, rewrite] - model[_model
inputs, types, identity, document building] + wheel[_wheel
artifact reader, zip reads and writes, RECORD, bundled wheel checks] + model[_model
inputs, types, identity, graph rules, document building] errors[_errors
SbomError] version[version] hook --> collect @@ -38,6 +38,7 @@ that command runs. cli --> generate cli --> validate cli --> formats + cli --> version cli -. types .-> model api --> cli api --> model diff --git a/src/python_sbom/_cli/_spdx.py b/src/python_sbom/_cli/_spdx.py index c2320e8..ad6de8a 100644 --- a/src/python_sbom/_cli/_spdx.py +++ b/src/python_sbom/_cli/_spdx.py @@ -1,11 +1,11 @@ from __future__ import annotations -import json from typing import TYPE_CHECKING, Protocol from python_sbom._errors import SbomError -from python_sbom._formats._spdx import to_spdx -from python_sbom._validate._document import is_document +from python_sbom._formats._json import dumps +from python_sbom._formats._spdx import render_spdx +from python_sbom._validate._document import is_document, load_json from python_sbom._validate._limits import MAX_COMPONENTS, byte_limit if TYPE_CHECKING: @@ -22,15 +22,15 @@ class _Options(Protocol): def run(options: _Options) -> bool: document = _read(options.source) try: - rendered = to_spdx(document) - # ValueError: a timestamp fromisoformat rejects; RecursionError: nesting json.dumps, a few frames deeper than - # json.loads, gives up on - except (KeyError, TypeError, ValueError, RecursionError) as error: + rendered = dumps(render_spdx(document)) + # ValueError: a timestamp fromisoformat rejects; AttributeError: a tools value that is neither an array nor an + # object; RecursionError: nesting json.dumps, a few frames deeper than json.loads, gives up on + except (AttributeError, KeyError, TypeError, ValueError, RecursionError) as error: # the CLI renders without the schema check, which needs the validate extra msg = f"{options.source} is not a CycloneDX document python-sbom can render: {error!r}" raise SbomError(msg) from error # bytes, so the line endings are the same on every operating system - options.target.write_bytes(rendered.encode()) + options.target.write_bytes(rendered) return True @@ -46,15 +46,9 @@ def _read(source: Path) -> Document: msg = f"{source} is over {limit} bytes, the size limit of an SBOM python-sbom reads" raise SbomError(msg) try: - loaded = json.loads(raw.decode("utf-8")) - except UnicodeDecodeError as error: - msg = f"{source} is not UTF-8: {error.reason} at byte {error.start}" - raise SbomError(msg) from error - except json.JSONDecodeError as error: - msg = f"{source} is not JSON: {error}" - raise SbomError(msg) from error - except RecursionError as error: - msg = f"{source} nests too deep for the JSON parser to read" + loaded = load_json(raw) + except SbomError as error: + msg = f"{source} {error}" raise SbomError(msg) from error if not is_document(loaded): msg = f"{source} is not a CycloneDX document python-sbom can render" diff --git a/src/python_sbom/_collect/__init__.py b/src/python_sbom/_collect/__init__.py index 8083326..3ee5431 100644 --- a/src/python_sbom/_collect/__init__.py +++ b/src/python_sbom/_collect/__init__.py @@ -1 +1 @@ -"""Facts gathered at the edge: settings, the build environment, the source revision and the project tree.""" +"""What the edge reads: settings, build facts, the source revision, the project tree, the artifact and the provider.""" diff --git a/src/python_sbom/_collect/_archive.py b/src/python_sbom/_collect/_archive.py index c680269..f11b47c 100644 --- a/src/python_sbom/_collect/_archive.py +++ b/src/python_sbom/_collect/_archive.py @@ -1,102 +1,94 @@ from __future__ import annotations -from io import BytesIO from typing import TYPE_CHECKING, Final from python_sbom._errors import SbomError +from python_sbom._model._archive import bundled_members from python_sbom._model._inputs import ArchiveTree, RecordRow from python_sbom._model._metadata import parse_metadata -from python_sbom._wheel._archive import member_digest, open_archive, read_member -from python_sbom._wheel._bundled import read_bundled_wheels +from python_sbom._model._names import archive_sbom_file_name if TYPE_CHECKING: - from collections.abc import Callable - from pathlib import Path, PurePosixPath + from collections.abc import Container from python_sbom._model._metadata import CoreMetadata + from python_sbom._wheel._reader import ArtifactReader -# a bundled wheel is read whole, to hash it and open it as a zip, so its size is capped as metadata's is -_WHEEL_LIMIT: Final[int] = 512 * 1024 * 1024 - -def read_archive(archive: Path) -> ArchiveTree: +def read_archive(artifact: ArtifactReader) -> ArchiveTree: # every member is read from the zip by its name, never from a file system path, so a name such as /etc/hostname or # ../outside names a member, not a file of the host - where: Final[str] = archive.name - with open_archive(archive, where) as handle: - members: Final[list[str]] = sorted( - info.filename for info in handle.infolist() if not info.is_dir() and info.filename != f"{where}.cdx.json" - ) - metadata: Final[dict[str, CoreMetadata]] = {} - for name in members: - if not name.endswith(".dist-info/METADATA"): - continue - # files belong to a distribution by the directory that holds its .dist-info, which two cannot share - if (directory := _directory(name)) in metadata: - msg = ( - f"{where} holds more than one .dist-info in {directory or 'its top level'}, so their files cannot" - " be told apart" - ) - raise SbomError(msg) - metadata[directory] = parse_metadata(read_member(handle, name, where).decode("utf-8")) - if (top := sum(name.count("/") == 1 and name.endswith(".dist-info/METADATA") for name in members)) != 1: - msg = f"{where} holds {top} top-level .dist-info/METADATA, not the one naming the project" - raise SbomError(msg) - files: Final[dict[str, RecordRow]] = { - name: RecordRow( - path=name, - algorithm="SHA-256", - digest=member_digest(handle, name, "sha256"), - size=str(handle.getinfo(name).file_size), + where: Final[str] = artifact.path.name + members: Final[list[str]] = sorted( + info.filename + for info in artifact.archive.infolist() + if not info.is_dir() and info.filename != archive_sbom_file_name(where) + ) + metadata: Final[dict[str, CoreMetadata]] = {} + for name in members: + if not name.endswith(".dist-info/METADATA"): + continue + # files belong to a distribution by the directory that holds its .dist-info, which two cannot share + if (directory := _distribution_prefix(name)) in metadata: + msg = ( + f"{where} holds more than one .dist-info in {directory or 'its top level'}, so their files cannot" + " be told apart" ) - for name in members - if not name.endswith(".whl") - } + raise SbomError(msg) + metadata[directory] = parse_metadata(artifact.read(name).decode("utf-8")) + if "" not in metadata: + msg = f"{where} holds no top-level .dist-info/METADATA naming the project" + raise SbomError(msg) + files: Final[dict[str, RecordRow]] = { + name: RecordRow( + path=name, + algorithm="SHA-256", + digest=artifact.digest(name), + size=str(artifact.archive.getinfo(name).file_size), + ) + for name in members + if not name.endswith(".whl") + } owned: Final[dict[str, list[str]]] = {prefix: [] for prefix in metadata} loose: Final[list[str]] = [] for name in members: - # the deepest distribution directory that holds the file; the project's own holds every other subdirectory - if owner := max((prefix for prefix in metadata if prefix and name.startswith(prefix)), key=len, default=""): + # the project's own distribution holds every subdirectory no other one does + if owner := _owner(name, metadata): owned[owner].append(name) elif "/" in name: owned[""].append(name) else: loose.append(name) # in the order the components list them, so the first broken wheel is the one reported - wheels: Final[list[str]] = [name for name in loose if name.endswith(".whl")] - for prefix in sorted(metadata): - wheels.extend(name for name in owned[prefix] if name.endswith(".whl")) + wheels: Final[list[str]] = [member for member in bundled_members(loose, owned) if not member.endswith("/")] return ArchiveTree( metadata=metadata, owned=owned, loose=loose, files=files, - bundled=dict(zip(wheels, read_bundled_wheels(wheels, lambda name: BytesIO(_read(archive, name))), strict=True)), + bundled=dict(zip(wheels, artifact.bundled_wheels(wheels), strict=True)), ) -def _directory(name: str) -> str: +def _distribution_prefix(name: str) -> str: # each distribution by the directory that holds its .dist-info, the project's own at the top level return f"{name.rsplit('/', 2)[0]}/" if name.count("/") > 1 else "" -def _read(archive: Path, name: str) -> bytes: - with open_archive(archive, archive.name) as handle: - return read_member(handle, name, archive.name, _WHEEL_LIMIT) - - -def member_reader(artifact: Path) -> Callable[[PurePosixPath], bytes]: - # what a content provider reads: a member of the built wheel or archive, through the same checks as every read - def read(path: PurePosixPath) -> bytes: - with open_archive(artifact, str(artifact)) as handle: - if str(path) not in handle.namelist(): - raise KeyError(str(path)) - return read_member(handle, str(path), str(artifact)) - - return read +def _owner(name: str, distributions: Container[str]) -> str | None: + # the deepest distribution directory that holds the member, found by looking up each of its ancestor directories, + # deepest first, so the work grows with the member's depth rather than the number of distributions + parents: Final[list[str]] = name.split("/")[:-1] + return next( + ( + prefix + for depth in range(len(parents), 0, -1) + if (prefix := f"{'/'.join(parents[:depth])}/") in distributions + ), + None, + ) __all__ = [ - "member_reader", "read_archive", ] diff --git a/src/python_sbom/_collect/_content.py b/src/python_sbom/_collect/_content.py index 04dc728..efbcbec 100644 --- a/src/python_sbom/_collect/_content.py +++ b/src/python_sbom/_collect/_content.py @@ -3,6 +3,7 @@ import inspect import pkgutil import sys +from importlib.machinery import PathFinder from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Final @@ -33,6 +34,12 @@ def _load(spec: str, root: Path) -> ContentProvider: if not (module_name and attribute): msg = f"content provider {spec!r} must be `module:attribute`" raise SbomError(msg) + if (imported := _imported_elsewhere(module_name, root)) is not None: + msg = ( + f"content provider {spec!r} cannot load from {root}: this process already imported {imported}, a module of" + " the same name, and would run that instead; build each such project in a process of its own" + ) + raise SbomError(msg) sys.path.insert(0, str(root)) try: provider = pkgutil.resolve_name(spec) @@ -50,6 +57,27 @@ def _load(spec: str, root: Path) -> ContentProvider: return provider +def _imported_elsewhere(module_name: str, root: Path) -> str | None: + # the import cache is per process, so a hatch build of a second project with a provider module of the same name + # would run the first project's; each package on the way down to the module, and the module itself, that the project + # provides is compared with what is already imported under its name; namespace packages merge, so only modules and + # regular packages can clash + search: list[str] | None = [str(root)] + parts: Final[list[str]] = module_name.split(".") + for depth in range(1, len(parts) + 1): + if search is None or (found := PathFinder.find_spec(".".join(parts[:depth]), search)) is None: + return None + imported = getattr(sys.modules.get(found.name), "__file__", None) + if ( + found.origin is not None + and imported is not None + and Path(imported).resolve() != Path(found.origin).resolve() + ): + return imported + search = None if found.submodule_search_locations is None else list(found.submodule_search_locations) + return None + + def _provider_scripts(provider: ContentProvider, root: Path) -> tuple[PurePosixPath, ...]: # each class of the provider defined in the project shapes the document, a base class or mixin as much as the class # itself, so the SBOM lists their modules among the build tools; python-sbom's own ContentProvider adds nothing diff --git a/src/python_sbom/_collect/_source.py b/src/python_sbom/_collect/_source.py index ad16c02..a30b16e 100644 --- a/src/python_sbom/_collect/_source.py +++ b/src/python_sbom/_collect/_source.py @@ -1,49 +1,19 @@ from __future__ import annotations -import dataclasses -import hashlib from pathlib import PurePath from typing import TYPE_CHECKING, Final from python_sbom._errors import SbomError -from python_sbom._model._inputs import BuildScript, SbomRequest, SourceTree -from python_sbom._wheel._archive import open_archive, read_wheel_metadata -from python_sbom._wheel._bundled import read_bundled_wheels +from python_sbom._model._inputs import BuildScript +from python_sbom._wheel._zip import sha256_hex if TYPE_CHECKING: - from collections.abc import Iterator, Mapping, Sequence + from collections.abc import Iterable from pathlib import Path, PurePosixPath from python_sbom._model._inputs import Settings -def read_source_tree(request: SbomRequest, *, inside: Mapping[str, Sequence[str]]) -> SourceTree: - # inside maps a wheel's sha256 to its paths in the built artifact: a bundled wheel the artifact holds is named by - # its path there, as an archive SBOM names it, so both documents give one wheel the same identity; identical copies - # take those paths in turn, so each keeps an identity of its own - root = request.root - wheels: Final[set[str]] = set() - for pattern in request.settings.bundled_wheels: - # a wheel outside the project is none the project ships, and pathlib refuses an absolute pattern outright - if (path := PurePath(pattern)).anchor or ".." in path.parts: - msg = f"`bundled-wheels` pattern {pattern!r} must be relative to the project, without `..`" - raise SbomError(msg) - wheels.update(wheel.relative_to(root).as_posix() for wheel in root.glob(pattern)) - destinations: Final[dict[str, Iterator[str]]] = {digest: iter(paths) for digest, paths in inside.items()} - return SourceTree( - copyright=copyright_notice(root, request.metadata.values("license_files")), - bundled=[ - dataclasses.replace(wheel, path=next(destinations.get(wheel.sha256, iter(())), wheel.path)) - for wheel in read_bundled_wheels(sorted(wheels), lambda path: (root / path).open("rb")) - ], - # str, so a Windows drive in an absolute path survives the join with the native root - build_scripts=[ - BuildScript(path=script, sha256=hashlib.sha256((root / str(script)).read_bytes()).hexdigest()) - for script in request.build_scripts - ], - ) - - def copyright_notice(root: Path, license_files: list[str]) -> str | None: # the first Copyright line of the license files the project names, as they sit in the project for license_file in license_files: @@ -54,16 +24,31 @@ def copyright_notice(root: Path, license_files: list[str]) -> str | None: return None -def wheel_request(wheel: Path, root: Path, settings: Settings, build_scripts: tuple[PurePosixPath, ...]) -> SbomRequest: - with open_archive(wheel, str(wheel)) as archive: - _, metadata = read_wheel_metadata(archive, str(wheel)) - return SbomRequest( - root=root, metadata=metadata, wheel_name=wheel.name, settings=settings, build_scripts=build_scripts - ) +def selected_wheels(root: Path, settings: Settings) -> list[str]: + wheels: Final[set[str]] = set() + for pattern in settings.bundled_wheels: + # a wheel outside the project is none the project ships, and pathlib refuses an absolute pattern outright + if (path := PurePath(pattern)).anchor or ".." in path.parts: + msg = f"`bundled-wheels` pattern {pattern!r} must be relative to the project, without `..`" + raise SbomError(msg) + wheels.update(wheel.relative_to(root).as_posix() for wheel in root.glob(pattern)) + return sorted(wheels) + + +def hash_build_scripts(root: Path, paths: Iterable[PurePosixPath]) -> list[BuildScript]: + # str, so a Windows drive in an absolute path survives the join with the native root; a script named twice, by the + # caller and by the provider, is listed once + return [BuildScript(path=path, sha256=file_sha256(root / str(path))) for path in dict.fromkeys(paths)] + + +def file_sha256(path: Path) -> str: + with path.open("rb") as stream: + return sha256_hex(stream) __all__ = [ "copyright_notice", - "read_source_tree", - "wheel_request", + "file_sha256", + "hash_build_scripts", + "selected_wheels", ] diff --git a/src/python_sbom/_collect/_wheel.py b/src/python_sbom/_collect/_wheel.py new file mode 100644 index 0000000..3a0fbd0 --- /dev/null +++ b/src/python_sbom/_collect/_wheel.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from python_sbom._errors import SbomError +from python_sbom._model._inputs import WheelTree +from python_sbom._wheel._zip import read_wheel_metadata + +from ._source import copyright_notice, file_sha256, hash_build_scripts, selected_wheels + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + from pathlib import Path, PurePosixPath + + from python_sbom._model._inputs import Settings + from python_sbom._wheel._reader import ArtifactReader + + +def read_wheel( + artifact: ArtifactReader, root: Path, settings: Settings, build_scripts: Iterable[PurePosixPath] +) -> WheelTree: + _, metadata = read_wheel_metadata(artifact.archive, str(artifact.path)) + return WheelTree( + name=artifact.path.name, + metadata=metadata, + copyright=copyright_notice(root, metadata.values("license_files")), + bundled=artifact.bundled_wheels(_shipped_copies(root, settings, artifact)), + build_scripts=hash_build_scripts(root, build_scripts), + ) + + +def _shipped_copies(root: Path, settings: Settings, artifact: ArtifactReader) -> list[str]: + # the bundled-wheels option selects wheels in the source tree, but the SBOM describes the bytes the artifact ships: + # each selected wheel is found in the artifact by its digest and read from there, so a wheel changed or left out + # since the build is refused rather than described; identical copies take the artifact's matching members in turn + shipped: Final[dict[str, list[str]]] = {} + for name in sorted(artifact.members): + if name.endswith(".whl"): + shipped.setdefault(artifact.digest(name), []).append(name) + copies: Final[dict[str, Iterator[str]]] = {digest: iter(names) for digest, names in shipped.items()} + found: Final[list[str]] = [] + for source in selected_wheels(root, settings): + if (member := next(copies.get(file_sha256(root / source), iter(())), None)) is None: + msg = ( + f"bundled wheel {source} is not in {artifact.path.name} byte for byte: it changed since the build, or" + " the build left it out" + ) + raise SbomError(msg) + found.append(member) + return found + + +__all__ = [ + "read_wheel", +] diff --git a/src/python_sbom/_formats/_spdx.py b/src/python_sbom/_formats/_spdx.py index 14c8aa7..dde2be2 100644 --- a/src/python_sbom/_formats/_spdx.py +++ b/src/python_sbom/_formats/_spdx.py @@ -6,12 +6,10 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Final, NotRequired, TypedDict -from python_sbom._model._document import walk_components +from python_sbom._model._graph import walk_components from python_sbom._model._names import SBOM_FILE, SBOM_REPOSITORY, property_prefix, sbom_property from python_sbom.version import __version__ -from ._json import dumps - if TYPE_CHECKING: from python_sbom._model._types import ( Component, @@ -107,21 +105,17 @@ class Spdx(TypedDict): hasExtractedLicensingInfos: NotRequired[list[_ExtractedLicense]] -def to_spdx(document: Document) -> str: - return dumps(render_spdx(document)).decode() - - -def render_spdx(bom: Document) -> Spdx: - metadata: Final[Metadata] = bom["metadata"] +def render_spdx(document: Document) -> Spdx: + metadata: Final[Metadata] = document["metadata"] root: Final[Component] = metadata["component"] project: Final[str] = property_prefix(root["name"]) recorded: Final[dict[str, str]] = {prop["name"]: prop.get("value", "") for prop in metadata.get("properties", [])} name: Final[str] = f"{root['name']}-{root['version']}" if "version" in root else root["name"] # without a serial number the content names the document, so rendering it twice gives the same namespace source: Final[str] = ( - bom["serialNumber"] - if "serialNumber" in bom - else f"sha256:{hashlib.sha256(json.dumps(bom, sort_keys=True).encode()).hexdigest()}" + document["serialNumber"] + if "serialNumber" in document + else f"sha256:{hashlib.sha256(json.dumps(document, sort_keys=True).encode()).hexdigest()}" ) repository: Final[str] = recorded.get(sbom_property(project, SBOM_REPOSITORY), "https://spdx.org/spdxdocs") sbom_file: Final[str] = recorded.get(sbom_property(project, SBOM_FILE), "CycloneDX JSON") @@ -129,10 +123,10 @@ def render_spdx(bom: Document) -> Spdx: # file components mirror wheel RECORD entries, which carry no SHA-1 and so cannot become SPDX files components: Final[list[Component]] = [ component - for component in walk_components([root, *bom.get("components", []), *tools]) + for component in walk_components([root, *document.get("components", []), *tools]) if component["type"] != "file" ] - packages: Final[list[_Package]] = [_package(component, f"{project}:") for component in components] + packages: Final[list[_Package]] = [_package(component, project) for component in components] spdx: Final[Spdx] = { "spdxVersion": "SPDX-2.3", "dataLicense": "CC0-1.0", @@ -143,7 +137,7 @@ def render_spdx(bom: Document) -> Spdx: " the build formulation.", "creationInfo": _creation_info(metadata, tool_creators), "packages": packages, - "relationships": _relationships(bom, components, tools), + "relationships": _relationships(document, components, tools), } # SPDX requires the text of every LicenseRef a package declares; the metadata names them, it holds no text references: Final[set[str]] = set() @@ -173,7 +167,7 @@ def _creator(name: str, version: str | None) -> str: return f"Tool: {name}" if version is None else f"Tool: {name}-{version}" -def _package(component: Component, property_prefix: str) -> _Package: +def _package(component: Component, project: str) -> _Package: # a component carries either one SPDX expression, or the free-form names from License/classifier metadata plus the # SPDX ids concluded from them licenses: Final[list[License]] = component.get("licenses", []) @@ -210,7 +204,7 @@ def _package(component: Component, property_prefix: str) -> _Package: if comment := "\n".join( f"{prop['name']}: {prop.get('value', '')}" for prop in component.get("properties", []) - if prop["name"].startswith(property_prefix) + if prop["name"].startswith(f"{project}:") ): package["comment"] = comment if purl := component.get("purl"): @@ -225,10 +219,10 @@ def _package(component: Component, property_prefix: str) -> _Package: def _spdx_id(bom_ref: str) -> str: - # SPDX ids allow letters, digits, "." and "-"; the digest keeps two refs that differ only in other characters, - # such as the markers of two conditional requirements, apart + # the readable part drops every other character, so refs that differ only in those share it; only the full digest, + # not a prefix of it, keeps their ids apart readable: Final[str] = re.sub(r"[^A-Za-z0-9.]+", "-", bom_ref).strip("-") - return f"SPDXRef-{readable}-{hashlib.sha256(bom_ref.encode()).hexdigest()[:8]}" + return f"SPDXRef-{readable}-{hashlib.sha256(bom_ref.encode()).hexdigest()}" def _creation_info(metadata: Metadata, tool_creators: list[str]) -> _CreationInfo: @@ -243,14 +237,14 @@ def _creation_info(metadata: Metadata, tool_creators: list[str]) -> _CreationInf return {"created": created.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "creators": creators} -def _relationships(bom: Document, components: list[Component], tools: list[Component]) -> list[_Relationship]: - root_id: Final[str] = _spdx_id(bom["metadata"]["component"]["bom-ref"]) +def _relationships(document: Document, components: list[Component], tools: list[Component]) -> list[_Relationship]: + root_id: Final[str] = _spdx_id(document["metadata"]["component"]["bom-ref"]) relationships: Final[list[_Relationship]] = [_relationship("SPDXRef-DOCUMENT", "DESCRIBES", root_id)] for parent in components: parent_id = _spdx_id(parent["bom-ref"]) children = (child for child in parent.get("components", []) if child["type"] != "file") relationships.extend(_relationship(parent_id, "CONTAINS", _spdx_id(child["bom-ref"])) for child in children) - for entry in bom.get("dependencies", []): + for entry in document.get("dependencies", []): dependent_id = _spdx_id(entry["ref"]) relationships.extend( _relationship(dependent_id, "DEPENDS_ON", _spdx_id(dependency)) for dependency in entry.get("dependsOn", []) @@ -266,5 +260,4 @@ def _relationship(element: str, kind: str, related: str) -> _Relationship: __all__ = [ "Spdx", "render_spdx", - "to_spdx", ] diff --git a/src/python_sbom/_generate/_archive.py b/src/python_sbom/_generate/_archive.py index 8da413a..1b14a98 100644 --- a/src/python_sbom/_generate/_archive.py +++ b/src/python_sbom/_generate/_archive.py @@ -1,22 +1,23 @@ from __future__ import annotations -import hashlib from typing import TYPE_CHECKING -from python_sbom._collect._archive import member_reader, read_archive +from python_sbom._collect._archive import read_archive from python_sbom._collect._content import resolve_provider from python_sbom._collect._facts import collect_facts from python_sbom._collect._revision import revision_reader from python_sbom._collect._settings import settings_from_pyproject -from python_sbom._collect._source import copyright_notice +from python_sbom._collect._source import copyright_notice, hash_build_scripts +from python_sbom._errors import SbomError from python_sbom._formats._json import dumps from python_sbom._model._archive import archive_artifact, bundled_paths from python_sbom._model._bom import assemble from python_sbom._model._content import ProviderContext from python_sbom._model._document import checked_contributions from python_sbom._model._identity import project_identity -from python_sbom._model._inputs import BuildScript -from python_sbom._wheel._archive import append_member +from python_sbom._model._names import archive_sbom_file_name +from python_sbom._wheel._reader import open_artifact +from python_sbom._wheel._write import append_member if TYPE_CHECKING: from collections.abc import Sequence @@ -38,10 +39,13 @@ def archive_sbom( # ruff: ignore[too-many-arguments] # every command-line optio output: Path | None, embed: bool, ) -> tuple[Path, str | None]: + name = archive_sbom_file_name(archive.name) + if (target := output or archive.with_name(name)).exists() and target.samefile(archive): + msg = f"{target} is the archive {archive} itself, refusing to overwrite it with the SBOM" + raise SbomError(msg) facts = collect_facts(root, revision_reader(commit), path=build_env) document = describe_archive(root, archive, facts=facts, provider=provider, build_scripts=build_scripts) - name = f"{archive.name}.cdx.json" - (target := output or archive.with_name(name)).write_bytes(content := dumps(document)) + target.write_bytes(content := dumps(document)) if not embed: return target, None append_member(archive, name, content, facts.epoch) @@ -58,21 +62,20 @@ def describe_archive( ) -> Document: settings = settings_from_pyproject(root) content, provider_scripts = resolve_provider(provider, settings, root) - tree = read_archive(archive) - project = project_identity(tree.metadata[""], settings) - context = ProviderContext(root=root, settings=settings, identity=project, read=member_reader(archive)) - artifact = archive_artifact( - archive.name, - tree, - project=project, - bundled_info={str(path): content.bundled_wheel(context, path) for path in bundled_paths(tree)}, - build_scripts=[ - BuildScript(path=path, sha256=hashlib.sha256((root / str(path)).read_bytes()).hexdigest()) - for path in dict.fromkeys([*build_scripts, *provider_scripts]) - ], - copyright_line=copyright_notice(root, tree.metadata[""].values("license_files")), - ) - return assemble(artifact, project, settings, facts, checked_contributions(content.contributions(context), content)) + with open_artifact(archive) as artifact: + tree = read_archive(artifact) + project = project_identity(tree.metadata[""], settings) + context = ProviderContext(root=root, settings=settings, identity=project, read=artifact.provider_read) + built = archive_artifact( + archive.name, + tree, + project=project, + bundled_info={str(path): content.bundled_wheel(context, path) for path in bundled_paths(tree)}, + build_scripts=hash_build_scripts(root, [*build_scripts, *provider_scripts]), + copyright_line=copyright_notice(root, tree.metadata[""].values("license_files")), + ) + contributions = checked_contributions(content.contributions(context), content) + return assemble(built, project, settings, facts, contributions) __all__ = [ diff --git a/src/python_sbom/_generate/_embed.py b/src/python_sbom/_generate/_embed.py index 089da3b..4f1dcfc 100644 --- a/src/python_sbom/_generate/_embed.py +++ b/src/python_sbom/_generate/_embed.py @@ -1,30 +1,28 @@ from __future__ import annotations -from dataclasses import dataclass from pathlib import PurePosixPath from typing import TYPE_CHECKING -from python_sbom._collect._archive import member_reader from python_sbom._collect._content import resolve_provider from python_sbom._collect._facts import collect_facts from python_sbom._collect._revision import revision_reader from python_sbom._collect._settings import settings_from_pyproject -from python_sbom._collect._source import read_source_tree, wheel_request +from python_sbom._collect._wheel import read_wheel from python_sbom._formats._json import dumps -from python_sbom._model._bom import describe +from python_sbom._model._bom import assemble from python_sbom._model._content import ProviderContext from python_sbom._model._document import checked_contributions from python_sbom._model._identity import project_identity -from python_sbom._model._names import sbom_file_name -from python_sbom._wheel._archive import bundled_wheels_by_digest, write_sbom +from python_sbom._model._wheel import wheel_artifact +from python_sbom._wheel._reader import open_artifact +from python_sbom._wheel._write import write_sbom if TYPE_CHECKING: from collections.abc import Sequence from pathlib import Path from python_sbom._model._content import ContentProvider - from python_sbom._model._inputs import BuildFacts, SbomRequest, SourceTree - from python_sbom._model._types import Document + from python_sbom._model._inputs import BuildFacts, Settings def embed_sbom( @@ -37,50 +35,40 @@ def embed_sbom( ) -> str: settings = settings_from_pyproject(root) content, scripts = resolve_provider(provider, settings, root) - return embed_into( - wheel, - wheel_request(wheel, root, settings, scripts), - content, - collect_facts(root, revision_reader(commit), path=build_env), - ) - - -def embed_into(wheel: Path, request: SbomRequest, content: ContentProvider, facts: BuildFacts) -> str: - source = read_source_tree(request, inside=bundled_wheels_by_digest(wheel)) - project = project_identity(request.metadata, request.settings) - context = ProviderContext(root=request.root, settings=request.settings, identity=project, read=member_reader(wheel)) - sbom = build_sbom(request, source, facts, content, context) - return write_sbom(wheel, sbom.file_name, sbom.content, facts.epoch) + facts = collect_facts(root, revision_reader(commit), path=build_env) + return embed_into(wheel, root, settings=settings, content=content, build_scripts=scripts, facts=facts) -def build_sbom( - request: SbomRequest, source: SourceTree, facts: BuildFacts, content: ContentProvider, context: ProviderContext -) -> Sbom: - return Sbom( - file_name=sbom_file_name(request.metadata.name), - document=describe( - request, - source, - facts, - {wheel.path: content.bundled_wheel(context, PurePosixPath(wheel.path)) for wheel in source.bundled}, - checked_contributions(content.contributions(context), content), - ), +def embed_into( # ruff: ignore[too-many-arguments] # what the hook and the command line each gather their own way + wheel: Path, + root: Path, + *, + settings: Settings, + content: ContentProvider, + build_scripts: Sequence[PurePosixPath], + facts: BuildFacts, +) -> str: + # the wheel stays open while its document is built, for its metadata, its bundled wheels and the provider to read, + # and is closed before it is rewritten + with open_artifact(wheel) as artifact: + tree = read_wheel(artifact, root, settings, build_scripts) + project = project_identity(tree.metadata, settings) + context = ProviderContext(root=root, settings=settings, identity=project, read=artifact.provider_read) + built = wheel_artifact( + tree, + project=project, + settings=settings, + bundled_info={ + bundled.path: content.bundled_wheel(context, PurePosixPath(bundled.path)) for bundled in tree.bundled + }, + ) + contributions = checked_contributions(content.contributions(context), content) + return write_sbom( + wheel, built.sbom_file, dumps(assemble(built, project, settings, facts, contributions)), facts.epoch ) -@dataclass(frozen=True, slots=True, kw_only=True) -class Sbom: - file_name: str - document: Document - - @property - def content(self) -> bytes: - return dumps(self.document) - - __all__ = [ - "Sbom", - "build_sbom", "embed_into", "embed_sbom", ] diff --git a/src/python_sbom/_model/_archive.py b/src/python_sbom/_model/_archive.py index 2357858..f335220 100644 --- a/src/python_sbom/_model/_archive.py +++ b/src/python_sbom/_model/_archive.py @@ -6,8 +6,16 @@ from packageurl import PackageURL -from ._components import bundled_wheel_component, component_from_metadata, file_component, vendored_dependencies +from ._components import ( + bundled_wheel_component, + component_from_metadata, + file_component, + mark_bundled, + vendored_dependencies, +) +from ._graph import artifact_contents from ._inputs import Artifact +from ._names import archive_sbom_file_name if TYPE_CHECKING: from collections.abc import Mapping @@ -33,24 +41,25 @@ def archive_artifact( # ruff: ignore[too-many-arguments] # keyword-only, each a version: Final[str] = tree.metadata[""].version ref: Final[str] = PackageURL(type="generic", name=name, version=version).to_string() distributions = _Distributions(tree=tree, project=project, bundled_info=bundled_info) + root: Final[Component] = { + "type": "application", + "bom-ref": ref, + "name": name, + "version": version, + "purl": ref, + "components": [ + distributions.wheel(member) if member.endswith(".whl") else file_component(ref, tree.files[member]) + for member in tree.loose + ], + } components: Final[list[Component]] = [distributions.component(prefix) for prefix in sorted(tree.metadata)] return Artifact( - root={ - "type": "application", - "bom-ref": ref, - "name": name, - "version": version, - "purl": ref, - "components": [ - distributions.wheel(member) if member.endswith(".whl") else file_component(ref, tree.files[member]) - for member in tree.loose - ], - }, + root=root, components=components, - dependencies=vendored_dependencies(components), + dependencies=vendored_dependencies(artifact_contents(root, components)), build_scripts=build_scripts, version=version, - sbom_file=f"{name}.cdx.json", + sbom_file=archive_sbom_file_name(name), scope=( "Components are the distributions the archive holds, as wheels or unpacked, each with its files; the" " archive's own top-level files sit under the root component." @@ -70,17 +79,9 @@ class _Distributions: def component(self, prefix: str) -> Component: component = component_from_metadata(self.tree.metadata[prefix]) if prefix: - # a wheel bundled unpacked: the directory tells copies of one release apart, as the path does for a .whl - directory = _directory(prefix) - info = self.bundled_info[str(directory)] - component["bom-ref"] = f"bundled:{directory}" - if info.supplier is not None: - component["supplier"] = info.supplier - component["properties"] = [ - {"name": f"{self.project.property_prefix}:bundled-wheel", "value": str(directory)}, - *info.properties, - *component["properties"], - ] + # a wheel bundled unpacked, named by its directory as a .whl is by its path + directory = prefix.removesuffix("/") + mark_bundled(component, directory, self.project.property_prefix, self.bundled_info[directory]) component["components"] = [ self.wheel(name) if name.endswith(".whl") else file_component(component["bom-ref"], self.tree.files[name]) for name in self.tree.owned[prefix] @@ -88,33 +89,29 @@ def component(self, prefix: str) -> Component: return component def wheel(self, name: str) -> Component: - info = self.bundled_info[name] - return bundled_wheel_component( - self.tree.bundled[name], self.project.property_prefix, info.properties, info.supplier - ) + return bundled_wheel_component(self.tree.bundled[name], self.project.property_prefix, self.bundled_info[name]) -def _directory(prefix: str) -> PurePosixPath: - return PurePosixPath(prefix.rstrip("/")) +def bundled_paths(tree: ArchiveTree) -> list[PurePosixPath]: + return [PurePosixPath(member.removesuffix("/")) for member in bundled_members(tree.loose, tree.owned)] -def bundled_paths(tree: ArchiveTree) -> list[PurePosixPath]: - # what a content provider is asked about, in the order the components list them: the archive's own top-level .whl - # members, then each wheel bundled unpacked, by its directory, followed by the .whl members it holds +def bundled_members(loose: list[str], owned: Mapping[str, list[str]]) -> list[str]: + # the bundled wheels in the order the components list them, which is the order they are read and a provider is asked + # about them: the archive's own top-level .whl members, then each wheel bundled unpacked, by its directory ending in + # "/", followed by the .whl members it holds return [ - *(PurePosixPath(name) for name in tree.loose if name.endswith(".whl")), + *(name for name in loose if name.endswith(".whl")), *( - path - for prefix in sorted(tree.metadata) - for path in ( - *([_directory(prefix)] if prefix else []), - *(PurePosixPath(name) for name in tree.owned[prefix] if name.endswith(".whl")), - ) + member + for prefix in sorted(owned) + for member in ([prefix] if prefix else []) + [name for name in owned[prefix] if name.endswith(".whl")] ), ] __all__ = [ "archive_artifact", + "bundled_members", "bundled_paths", ] diff --git a/src/python_sbom/_model/_bom.py b/src/python_sbom/_model/_bom.py index cf095e1..d335732 100644 --- a/src/python_sbom/_model/_bom.py +++ b/src/python_sbom/_model/_bom.py @@ -4,102 +4,18 @@ from typing import TYPE_CHECKING, Final from urllib.parse import urlsplit -from packaging.requirements import Requirement - -from ._components import bundled_wheel_component, component_from_metadata, script_tool, vendored_dependencies +from ._components import script_tool from ._document import check_refs, contributed_properties, formulation, timestamp, with_header -from ._identity import project_identity -from ._inputs import Artifact -from ._markers import applies_without_extras -from ._names import SBOM_FILE, SBOM_REPOSITORY, purl, pypi_url, sbom_file_name, sbom_property +from ._names import SBOM_FILE, SBOM_REPOSITORY, sbom_property from ._tools import tool_components if TYPE_CHECKING: - from collections.abc import Mapping - - from ._content import BundledWheelInfo, Contributions + from ._content import Contributions from ._identity import ProjectIdentity - from ._inputs import BuildFacts, SbomRequest, Settings, SourceTree + from ._inputs import Artifact, BuildFacts, Settings from ._types import Body, Component, Document, ExternalReference, Metadata, Property -def describe( - request: SbomRequest, - source: SourceTree, - facts: BuildFacts, - bundled_info: Mapping[str, BundledWheelInfo], - contributions: Contributions, -) -> Document: - metadata, settings = request.metadata, request.settings - project = project_identity(metadata, settings) - bundled = [ - bundled_wheel_component( - wheel, - project.property_prefix, - bundled_info[wheel.path].properties, - bundled_info[wheel.path].supplier, - ) - for wheel in source.bundled - ] - root = component_from_metadata(metadata) - root["type"] = "application" - name, version = metadata.name, metadata.version - root["externalReferences"] = [ - *root["externalReferences"], - {"type": "distribution", "url": pypi_url(name, version)}, - {"type": "attestation", "url": f"https://pypi.org/integrity/{name}/{version}/{request.wheel_name}/provenance"}, - ] - artifact = Artifact( - root=root, - components=[ - *bundled, - # hatchling repeats a Requires-Dist when a hook re-adds a dependency, and each bom-ref must be unique - *( - _declared_dependency(text, requirement, project.property_prefix) - for text in dict.fromkeys(metadata.values("requires_dist")) - if applies_without_extras(requirement := Requirement(text)) - ), - ], - dependencies=vendored_dependencies(bundled), - build_scripts=source.build_scripts, - version=version, - sbom_file=sbom_file_name(name), - scope=_scope(settings), - workflow=("wheel-build", "build the wheel and this SBOM"), - license_file=next(iter(metadata.values("license_files")), None), - copyright=source.copyright, - ) - return assemble(artifact, project, settings, facts, contributions) - - -def _declared_dependency(text: str, requirement: Requirement, property_prefix: str) -> Component: - component: Final[Component] = { - "type": "library", - # the same distribution can be declared more than once with different markers, so the purl is not unique - "bom-ref": f"requires-dist:{text}", - "name": requirement.name, - "purl": purl(requirement.name), - "externalReferences": [{"type": "distribution", "url": pypi_url(requirement.name)}], - "properties": [{"name": f"{property_prefix}:requires-dist", "value": text}], - } - if requirement.marker is not None: - component["properties"].append({ - "name": f"{property_prefix}:environment-marker", - "value": str(requirement.marker), - }) - return component - - -def _scope(settings: Settings) -> str: - declared = ( - "the direct runtime dependencies declared in the wheel's core metadata, which carry no version because the" - " installer resolves them" - ) - if directories := " and ".join(sorted({str(PurePosixPath(pattern).parent) for pattern in settings.bundled_wheels})): - return f"Components are the wheels bundled as data files under {directories} and {declared}." - return f"Components are {declared}." - - def assemble( artifact: Artifact, project: ProjectIdentity, settings: Settings, facts: BuildFacts, contributions: Contributions ) -> Document: @@ -165,9 +81,9 @@ def _provenance( "url": f"{project.repository}/tree/{revision.commit}", "comment": comment, }) - properties.append({"name": f"{project.property_prefix}:vcs-commit", "value": revision.commit}) + properties.append({"name": sbom_property(project.property_prefix, "vcs-commit"), "value": revision.commit}) if revision.dirty: - properties.append({"name": f"{project.property_prefix}:vcs-dirty", "value": "true"}) + properties.append({"name": sbom_property(project.property_prefix, "vcs-dirty"), "value": "true"}) root["externalReferences"] = references root["properties"] = properties return root @@ -218,5 +134,4 @@ def _metadata( __all__ = [ "assemble", - "describe", ] diff --git a/src/python_sbom/_model/_components.py b/src/python_sbom/_model/_components.py index 0895457..319b039 100644 --- a/src/python_sbom/_model/_components.py +++ b/src/python_sbom/_model/_components.py @@ -7,38 +7,29 @@ from packaging.utils import canonicalize_name -from ._document import walk_components +from ._graph import walk_components from ._iri import iri_reference from ._licenses import licenses -from ._names import purl, pypi_url, reference_type, tool_ref +from ._names import purl, pypi_url, reference_type, sbom_property, tool_ref from ._types import Dependency if TYPE_CHECKING: from pathlib import PurePosixPath + from ._content import BundledWheelInfo from ._inputs import BundledWheel, RecordRow from ._metadata import CoreMetadata - from ._types import Component, Contact, ExternalReference, Identity, OrganizationalEntity, Property + from ._types import Component, Contact, ExternalReference, Identity, Property -def bundled_wheel_component( - wheel: BundledWheel, property_prefix: str, properties: list[Property], supplier: OrganizationalEntity | None -) -> Component: +def bundled_wheel_component(wheel: BundledWheel, property_prefix: str, info: BundledWheelInfo) -> Component: component = component_from_metadata(wheel.metadata) - # the purl names the package; the path tells apart copies of one release bundled in two places - component["bom-ref"] = f"bundled:{wheel.path}" - if supplier is not None: - component["supplier"] = supplier + mark_bundled(component, wheel.path, property_prefix, info) component["hashes"] = [{"alg": "SHA-256", "content": wheel.sha256}] component["externalReferences"] = [ *component["externalReferences"], {"type": "distribution", "url": pypi_url(wheel.metadata.name, wheel.metadata.version)}, ] - component["properties"] = [ - {"name": f"{property_prefix}:bundled-wheel", "value": wheel.path}, - *properties, - *component["properties"], - ] component["evidence"] = { "identity": [ _evidence("purl", "manifest-analysis", wheel.metadata_name), @@ -49,7 +40,10 @@ def bundled_wheel_component( for source, metadata in wheel.vendored: child = component_from_metadata(metadata) child["bom-ref"] = f"{component['bom-ref']}#vendored/{child['bom-ref']}" - child["properties"] = [*child["properties"], {"name": f"{property_prefix}:vendored-manifest", "value": source}] + child["properties"] = [ + *child["properties"], + {"name": sbom_property(property_prefix, "vendored-manifest"), "value": source}, + ] child["evidence"] = {"identity": [_evidence("purl", "manifest-analysis", source)]} # a vendored .dist-info comes after pip's manifest line naming the same package, so its metadata wins; keyed on # the PEP 503 name, since the manifest and the .dist-info may spell it differently @@ -125,6 +119,19 @@ def _project_references(metadata: CoreMetadata) -> tuple[list[ExternalReference] return references, unlinked +def mark_bundled(component: Component, path: str, property_prefix: str, info: BundledWheelInfo) -> None: + # the purl names the package; the path, of a .whl or of the directory a wheel is unpacked into, tells apart copies + # of one release bundled in two places + component["bom-ref"] = f"bundled:{path}" + if info.supplier is not None: + component["supplier"] = info.supplier + component["properties"] = [ + {"name": sbom_property(property_prefix, "bundled-wheel"), "value": path}, + *info.properties, + *component["properties"], + ] + + def _evidence(field: str, technique: str, value: str) -> Identity: return {"field": field, "confidence": 1, "methods": [{"technique": technique, "confidence": 1, "value": value}]} @@ -169,6 +176,7 @@ def script_tool(path: PurePosixPath, sha256: str, version: str, link: str | None "bundled_wheel_component", "component_from_metadata", "file_component", + "mark_bundled", "script_tool", "vendored_dependencies", ] diff --git a/src/python_sbom/_model/_document.py b/src/python_sbom/_model/_document.py index 46c249f..42e4d74 100644 --- a/src/python_sbom/_model/_document.py +++ b/src/python_sbom/_model/_document.py @@ -2,15 +2,14 @@ import json import uuid -from collections import Counter from datetime import UTC, datetime from typing import TYPE_CHECKING, Final from python_sbom._errors import SbomError -if TYPE_CHECKING: - from collections.abc import Iterable, Iterator +from ._graph import reference_problems, walk_components +if TYPE_CHECKING: from ._content import ContentProvider, Contributions from ._inputs import BuildFacts from ._types import Body, Component, Document, Formula, Property @@ -25,35 +24,16 @@ def check_refs(body: Body) -> None: *body["metadata"]["tools"]["components"], ]) ] - refs: Final[Counter[str]] = Counter(component["bom-ref"] for component in components) # formulas and workflows share the bom-ref namespace, so a component cannot take theirs - for formula in body.get("formulation", []): - refs.update([formula["bom-ref"], *(workflow["bom-ref"] for workflow in formula["workflows"])]) - if repeated := sorted(ref for ref, count in refs.items() if count > 1): - msg = f"bom-ref values repeat: {', '.join(repeated)}" - raise SbomError(msg) - # one entry per component, as validation requires, so a provider adds to a component's dependencies, not a second - # entry for it - entries: Final[Counter[str]] = Counter(entry["ref"] for entry in body["dependencies"]) - if repeated := sorted(ref for ref, count in entries.items() if count > 1): - msg = f"dependencies list more than one entry for: {', '.join(repeated)}" - raise SbomError(msg) - named: Final[set[str]] = set(entries) - for entry in body["dependencies"]: - named.update(entry.get("dependsOn", [])) - if unknown := sorted(named - {component["bom-ref"] for component in components}): - msg = f"dependencies name bom-refs no component has: {', '.join(unknown)}" - raise SbomError(msg) - - -def walk_components(components: Iterable[Component]) -> Iterator[Component]: - # depth first, each component before the ones it nests, in document order; iterative, so a deeply nested document - # cannot exhaust the stack - pending: Final[list[Component]] = list(reversed(list(components))) - while pending: - component = pending.pop() - yield component - pending.extend(reversed(component.get("components", []))) + formulas: Final[list[str]] = [ + ref + for formula in body.get("formulation", []) + for ref in (formula["bom-ref"], *(workflow["bom-ref"] for workflow in formula["workflows"])) + ] + if problems := reference_problems( + [*(component["bom-ref"] for component in components), *formulas], body["dependencies"] + ): + raise SbomError(problems[0]) def with_header(body: Body, serial_namespace: uuid.UUID) -> Document: @@ -116,6 +96,5 @@ def timestamp(facts: BuildFacts) -> str: "contributed_properties", "formulation", "timestamp", - "walk_components", "with_header", ] diff --git a/src/python_sbom/_model/_graph.py b/src/python_sbom/_model/_graph.py new file mode 100644 index 0000000..79dbff1 --- /dev/null +++ b/src/python_sbom/_model/_graph.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from collections import Counter +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + + from ._types import Component, Dependency + + +def artifact_contents(root: Component, components: list[Component]) -> list[Component]: + # what the artifact holds or declares, wherever the document places it: the files and wheels nested under the root, + # which sit in an archive's top level, and the components; not the root itself, nor the tools that built it, so + # generation, the license policy and validation all judge the same set + return [*root.get("components", []), *components] + + +def shipped_libraries(contents: list[Component]) -> list[Component]: + # what the artifact carries bytes for: a declared requirement resolves at install time, and a library a bundled + # wheel vendors ships under that wheel's terms + return [ + component + for component in walk_components(contents) + if component["type"] == "library" + and ("hashes" in component or any(child["type"] == "file" for child in component.get("components", []))) + ] + + +def walk_components(components: Iterable[Component]) -> Iterator[Component]: + # depth first, each component before the ones it nests, in document order; iterative, so a deeply nested document + # cannot exhaust the stack + pending: Final[list[Component]] = list(reversed(list(components))) + while pending: + component = pending.pop() + yield component + pending.extend(reversed(component.get("components", []))) + + +def reference_problems(refs: list[str], dependencies: list[Dependency]) -> list[str]: + # what generation refuses and validation reports alike: a bom-ref given twice, which formulas and workflows share + # with components; a second dependency entry for one ref, which a reader taking them as a mapping would let a valid + # entry hide an invalid one with, while the SPDX rendering keeps the edges of both; and an entry naming no component + if twice := repeated(refs): + return [f"bom-ref values repeat: {', '.join(twice)}"] + if twice := repeated(entry["ref"] for entry in dependencies): + return [f"dependencies list more than one entry for: {', '.join(twice)}"] + named: Final[set[str]] = {ref for entry in dependencies for ref in (entry["ref"], *entry.get("dependsOn", []))} + if unknown := sorted(named - set(refs)): + return [f"dependencies name bom-refs no component has: {', '.join(unknown)}"] + return [] + + +def repeated(values: Iterable[str]) -> list[str]: + return sorted(value for value, count in Counter(values).items() if count > 1) + + +__all__ = [ + "artifact_contents", + "reference_problems", + "repeated", + "shipped_libraries", + "walk_components", +] diff --git a/src/python_sbom/_model/_inputs.py b/src/python_sbom/_model/_inputs.py index 61cbc0b..e4928f4 100644 --- a/src/python_sbom/_model/_inputs.py +++ b/src/python_sbom/_model/_inputs.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from pathlib import Path, PurePosixPath + from pathlib import PurePosixPath from ._metadata import CoreMetadata from ._types import Component, Dependency, OrganizationalEntity @@ -70,15 +70,6 @@ class Settings: content_provider: str | None -@dataclass(frozen=True, slots=True, kw_only=True) -class SbomRequest: - root: Path - metadata: CoreMetadata - wheel_name: str - settings: Settings - build_scripts: tuple[PurePosixPath, ...] - - @dataclass(frozen=True, slots=True, kw_only=True) class BundledWheel: path: str @@ -124,7 +115,12 @@ class Artifact: @dataclass(frozen=True, slots=True, kw_only=True) -class SourceTree: +class WheelTree: + # what a built wheel holds, read from the wheel without unpacking it: its file name, its core metadata and the + # wheels it bundles; and what the project adds: the copyright line of its license files and the scripts that built + # it + name: str + metadata: CoreMetadata copyright: str | None bundled: list[BundledWheel] build_scripts: list[BuildScript] @@ -140,7 +136,6 @@ class SourceTree: "Interpreter", "RecordRow", "Revision", - "SbomRequest", "Settings", - "SourceTree", + "WheelTree", ] diff --git a/src/python_sbom/_model/_names.py b/src/python_sbom/_model/_names.py index fdb33bc..e9aeb72 100644 --- a/src/python_sbom/_model/_names.py +++ b/src/python_sbom/_model/_names.py @@ -52,6 +52,11 @@ def property_prefix(name: str) -> str: return canonicalize_name(name) +def archive_sbom_file_name(archive: str) -> str: + # next to the archive, and inside it when embedded, where reading the archive again must skip it + return f"{archive}.cdx.json" + + def purl(name: str, version: str | None = None) -> str: # normalized as PEP 503 does, so every spelling of a project, jaraco.context or jaraco_context, gets one purl and a # component's bom-ref agrees with its purl; packageurl only lowercases and turns underscores into dashes @@ -74,6 +79,7 @@ def reference_type(label: str) -> str: __all__ = [ "SBOM_FILE", "SBOM_REPOSITORY", + "archive_sbom_file_name", "property_prefix", "purl", "pypi_url", diff --git a/src/python_sbom/_model/_wheel.py b/src/python_sbom/_model/_wheel.py new file mode 100644 index 0000000..9a46b18 --- /dev/null +++ b/src/python_sbom/_model/_wheel.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from pathlib import PurePosixPath +from typing import TYPE_CHECKING, Final + +from packaging.requirements import Requirement + +from ._components import bundled_wheel_component, component_from_metadata, vendored_dependencies +from ._inputs import Artifact +from ._markers import applies_without_extras +from ._names import purl, pypi_url, sbom_file_name, sbom_property + +if TYPE_CHECKING: + from collections.abc import Mapping + + from ._content import BundledWheelInfo + from ._identity import ProjectIdentity + from ._inputs import Settings, WheelTree + from ._types import Component + + +def wheel_artifact( + tree: WheelTree, *, project: ProjectIdentity, settings: Settings, bundled_info: Mapping[str, BundledWheelInfo] +) -> Artifact: + metadata = tree.metadata + bundled = [ + bundled_wheel_component(wheel, project.property_prefix, bundled_info[wheel.path]) for wheel in tree.bundled + ] + root = component_from_metadata(metadata) + root["type"] = "application" + name, version = metadata.name, metadata.version + root["externalReferences"] = [ + *root["externalReferences"], + {"type": "distribution", "url": pypi_url(name, version)}, + {"type": "attestation", "url": f"https://pypi.org/integrity/{name}/{version}/{tree.name}/provenance"}, + ] + return Artifact( + root=root, + components=[ + *bundled, + # hatchling repeats a Requires-Dist when a hook re-adds a dependency, and each bom-ref must be unique + *( + _declared_dependency(text, requirement, project.property_prefix) + for text in dict.fromkeys(metadata.values("requires_dist")) + if applies_without_extras(requirement := Requirement(text)) + ), + ], + dependencies=vendored_dependencies(bundled), + build_scripts=tree.build_scripts, + version=version, + sbom_file=sbom_file_name(name), + scope=_scope(settings), + workflow=("wheel-build", "build the wheel and this SBOM"), + license_file=next(iter(metadata.values("license_files")), None), + copyright=tree.copyright, + ) + + +def _declared_dependency(text: str, requirement: Requirement, property_prefix: str) -> Component: + component: Final[Component] = { + "type": "library", + # the same distribution can be declared more than once with different markers, so the purl is not unique + "bom-ref": f"requires-dist:{text}", + "name": requirement.name, + "purl": purl(requirement.name), + "externalReferences": [{"type": "distribution", "url": pypi_url(requirement.name)}], + "properties": [{"name": sbom_property(property_prefix, "requires-dist"), "value": text}], + } + if requirement.marker is not None: + component["properties"].append({ + "name": sbom_property(property_prefix, "environment-marker"), + "value": str(requirement.marker), + }) + return component + + +def _scope(settings: Settings) -> str: + declared = ( + "the direct runtime dependencies declared in the wheel's core metadata, which carry no version because the" + " installer resolves them" + ) + if directories := " and ".join(sorted({str(PurePosixPath(pattern).parent) for pattern in settings.bundled_wheels})): + return f"Components are the wheels bundled as data files under {directories} and {declared}." + return f"Components are {declared}." + + +__all__ = [ + "wheel_artifact", +] diff --git a/src/python_sbom/_validate/_artifact.py b/src/python_sbom/_validate/_artifact.py index 99b54eb..02c514b 100644 --- a/src/python_sbom/_validate/_artifact.py +++ b/src/python_sbom/_validate/_artifact.py @@ -3,7 +3,7 @@ import zipfile from typing import IO, TYPE_CHECKING, Final -from python_sbom._wheel._archive import open_archive +from python_sbom._wheel._zip import CHUNK, open_archive from ._document import validate_document from ._limits import MAX_COMPONENTS, byte_limit @@ -11,9 +11,6 @@ if TYPE_CHECKING: from pathlib import Path -# what one read may decompress, so a compressed member cannot run far past the limit before the reader notices -_CHUNK: Final[int] = 1024 * 1024 - def validate(path: Path, allowed_licenses: frozenset[str] | None, max_components: int = MAX_COMPONENTS) -> list[str]: limit: Final[int] = byte_limit(max_components) @@ -37,7 +34,7 @@ def validate(path: Path, allowed_licenses: frozenset[str] | None, max_components def _read(source: IO[bytes], limit: int) -> bytes: # stops within a chunk past the limit: enough for validate_document to refuse the SBOM without reading all of it content: Final[bytearray] = bytearray() - while len(content) <= limit and (chunk := source.read(_CHUNK)): + while len(content) <= limit and (chunk := source.read(CHUNK)): content.extend(chunk) return bytes(content) diff --git a/src/python_sbom/_validate/_document.py b/src/python_sbom/_validate/_document.py index 169a6b3..e07e329 100644 --- a/src/python_sbom/_validate/_document.py +++ b/src/python_sbom/_validate/_document.py @@ -3,16 +3,15 @@ import json import math import re -from collections import Counter from importlib.util import find_spec -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import TYPE_CHECKING, Any, Final -from python_sbom._errors import MissingExtraError +from python_sbom._errors import MissingExtraError, SbomError from python_sbom._formats._spdx import render_spdx -from python_sbom._model._document import walk_components +from python_sbom._model._graph import artifact_contents, reference_problems, repeated, walk_components from python_sbom._model._names import SBOM_FILE -from ._limits import MAX_COMPONENTS, byte_limit +from ._limits import MAX_COMPONENTS, MAX_TEXT, Json, byte_limit if TYPE_CHECKING: from collections.abc import Iterable @@ -20,8 +19,6 @@ from python_sbom._model._types import Component, Document -# a JSON value, as json.loads hands it over -Json: TypeAlias = "str | int | float | bool | list[Json] | dict[str, Json] | None" # the schema leaves these optional, yet every document python-sbom writes carries them and the checks below read them _REQUIRED: Final[tuple[tuple[str, ...], ...]] = ( ("metadata", "timestamp"), @@ -34,8 +31,6 @@ _SERIAL: Final[re.Pattern[str]] = re.compile(r"urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") # the modules of the validate extra _EXTRA: Final[tuple[str, ...]] = ("jsonschema_rs", "license_expression", "spdx_tools") -# an exception's message, or a schema error's, can quote the hostile value that caused it, which runs to megabytes -MAX_TEXT: Final[int] = 200 # the schema validator recurses once per level and crashes the process past a few thousand levels on a small thread # stack; the SBOMs python-sbom and other tools write nest about a dozen levels deep _MAX_DEPTH: Final[int] = 128 @@ -55,14 +50,8 @@ def validate_document( return _problems(raw, allowed_licenses, max_components) except MissingExtraError: raise - except UnicodeDecodeError as error: - return [f"is not UTF-8: {error.reason} at byte {error.start}"] - except json.JSONDecodeError as error: - return [f"is not JSON: {error}"] - except RecursionError: - # the JSON parser recurses once per level and gives up near the recursion limit, long before the depth check - # below sees the document - return ["nests too deep for the JSON parser to read"] + except SbomError as error: + return [str(error)] except Exception as error: # ruff: ignore[blind-except] # a hostile document may trip any check, see above return [f"could not be checked, {type(error).__name__}: {str(error)[:MAX_TEXT]}"] @@ -74,7 +63,7 @@ def _problems(raw: bytes, allowed_licenses: frozenset[str] | None, max_component missing_extra: Final[str] = "validation needs the validate extra: pip install 'python-sbom[validate]'" if any(find_spec(name) is None for name in _EXTRA): raise MissingExtraError(missing_extra) - loaded: Final = json.loads(raw.decode("utf-8")) + loaded: Final = load_json(raw) if census := _census_problems(loaded, max_components): return census # find_spec sees only the top-level modules, not a dependency of theirs that is missing or broken @@ -107,16 +96,35 @@ def _problems(raw: bytes, allowed_licenses: frozenset[str] | None, max_component from ._licenses import license_problems # ruff: ignore[import-outside-top-level] # see above except ImportError as error: raise MissingExtraError(missing_extra) from error - problems.extend(license_problems(loaded["components"], allowed_licenses)) + problems.extend( + license_problems(artifact_contents(loaded["metadata"]["component"], loaded["components"]), allowed_licenses) + ) return problems +def load_json(raw: bytes) -> Any: # ruff: ignore[any-type] # json.loads' own type: any shape until checked + # why bytes are no JSON value, worded once for validate and for spdx, which renders a document without validating it + try: + return json.loads(raw.decode("utf-8")) + except UnicodeDecodeError as error: + msg = f"is not UTF-8: {error.reason} at byte {error.start}" + raise SbomError(msg) from error + except json.JSONDecodeError as error: + msg = f"is not JSON: {error}" + raise SbomError(msg) from error + except RecursionError as error: + # the JSON parser recurses once per level and gives up near the recursion limit, long before the depth check + # sees the document + msg = "nests too deep for the JSON parser to read" + raise SbomError(msg) from error + + def _census_problems(document: Json, max_components: int) -> list[str]: # counts the components and bom-refs and measures the depth in one linear pass before the schema vouches for any # shape; iterative, so deep nesting cannot exhaust the stack. Nested components count too: the schema checks every # components array, and one array may hold them all, so only the total bounds its work components = 0 - refs: Final[Counter[str]] = Counter() + refs: Final[list[str]] = [] # only lists and objects go on the stack, every other value is judged where its container is read, so a scalar # nests too deep when its container already sits at the limit; the document starts out as the one item of a list pending: Final[list[tuple[list[Json] | dict[str, Json], int]]] = [([document], 0)] @@ -124,7 +132,7 @@ def _census_problems(document: Json, max_components: int) -> list[str]: node, depth = pending.pop() if isinstance(node, dict): if isinstance(ref := node.get("bom-ref"), str): - refs[ref] += 1 + refs.append(ref) components += len(listed) if isinstance(listed := node.get("components"), list) else 0 if components > max_components: return [ @@ -138,8 +146,10 @@ def _census_problems(document: Json, max_components: int) -> list[str]: return [f"nests more than {_MAX_DEPTH} levels deep, too deep to check"] if problem := _push_containers(children, depth + 1, pending): return problem - if duplicates := sorted(ref for ref, count in refs.items() if count > 1): - return [f"bom-ref values are not unique: {duplicates}"] + # every bom-ref in the document, wherever it sits, as CycloneDX requires them unique and documents other tools wrote + # get no reference check beyond this + if twice := repeated(refs): + return [f"bom-ref values repeat: {', '.join(twice)}"] return [] @@ -174,13 +184,19 @@ def _reference_problems(document: Document) -> list[str]: components: Final[list[Component]] = list(walk_components([root, *document["components"], *tools])) if nameless := sorted(component["name"] for component in components if "bom-ref" not in component): return [f"components lack a bom-ref: {nameless}"] + formulas: Final[list[str]] = [ + ref + for formula in document.get("formulation", []) + for ref in (formula["bom-ref"], *(workflow["bom-ref"] for workflow in formula.get("workflows", []))) + ] + if problems := reference_problems( + [*(component["bom-ref"] for component in components), *formulas], document["dependencies"] + ): + return problems problems: Final[list[str]] = [] dependencies: Final[dict[str, set[str]]] = { entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"] } - refs: Final[set[str]] = {component["bom-ref"] for component in components} - if unknown := (set(dependencies) | set().union(*dependencies.values())) - refs: - problems.append(f"dependencies reference bom-refs that do not exist: {sorted(unknown)}") direct: Final[set[str]] = {component["bom-ref"] for component in document["components"]} if dependencies.get(root["bom-ref"]) != direct: problems.append( @@ -199,8 +215,7 @@ def _reference_problems(document: Document) -> list[str]: __all__ = [ - "MAX_TEXT", - "Json", "is_document", + "load_json", "validate_document", ] diff --git a/src/python_sbom/_validate/_licenses.py b/src/python_sbom/_validate/_licenses.py index 5f16461..af56aac 100644 --- a/src/python_sbom/_validate/_licenses.py +++ b/src/python_sbom/_validate/_licenses.py @@ -11,7 +11,7 @@ get_license_index, ) -from python_sbom._model._document import walk_components +from python_sbom._model._graph import shipped_libraries if TYPE_CHECKING: from license_expression import LicenseExpression @@ -28,27 +28,16 @@ ) -def license_problems(components: list[Component], allowed: frozenset[str]) -> list[str]: +def license_problems(contents: list[Component], allowed: frozenset[str]) -> list[str]: # SPDX matches license ids without regard to case folded: Final[frozenset[str]] = frozenset(identifier.casefold() for identifier in allowed) return [ f"{component.get('bom-ref', component['name'])}: {problem}" - for component in _shipped(components) + for component in shipped_libraries(contents) if (problem := _judge(component, allowed, folded)) ] -def _shipped(components: list[Component]) -> list[Component]: - # only what the artifact carries bytes for is judged: a declared requirement resolves at install time, and a library - # a bundled wheel vendors ships under that wheel's terms - return [ - component - for component in walk_components(components) - if component["type"] == "library" - and ("hashes" in component or any(child["type"] == "file" for child in component.get("components", []))) - ] - - def _judge(component: Component, allowed: frozenset[str], folded: frozenset[str]) -> str | None: licenses: Final[list[License]] = component.get("licenses", []) terms: Final[list[str]] = [ diff --git a/src/python_sbom/_validate/_limits.py b/src/python_sbom/_validate/_limits.py index 602e551..4df8753 100644 --- a/src/python_sbom/_validate/_limits.py +++ b/src/python_sbom/_validate/_limits.py @@ -1,10 +1,15 @@ from __future__ import annotations -from typing import Final +from typing import Final, TypeAlias + +# a JSON value, as json.loads hands it over +Json: TypeAlias = "str | int | float | bool | list[Json] | dict[str, Json] | None" # every check takes time in proportion to the count, so the cap bounds the work an untrusted document can ask for, far # above the few thousand components a wheel with its build tools' files lists MAX_COMPONENTS: Final[int] = 100_000 +# an exception's message, or a schema error's, can quote the hostile value that caused it, which runs to megabytes +MAX_TEXT: Final[int] = 200 def byte_limit(max_components: int) -> int: @@ -16,5 +21,7 @@ def byte_limit(max_components: int) -> int: __all__ = [ "MAX_COMPONENTS", + "MAX_TEXT", + "Json", "byte_limit", ] diff --git a/src/python_sbom/_validate/_schemas.py b/src/python_sbom/_validate/_schemas.py index 9174ce4..f74ff75 100644 --- a/src/python_sbom/_validate/_schemas.py +++ b/src/python_sbom/_validate/_schemas.py @@ -15,7 +15,7 @@ from spdx_tools.spdx.validation.package_validator import validate_package from spdx_tools.spdx.validation.spdx_id_validators import validate_spdx_id -from ._document import MAX_TEXT +from ._limits import MAX_TEXT if TYPE_CHECKING: from importlib.resources.abc import Traversable @@ -25,7 +25,7 @@ from python_sbom._formats._spdx import Spdx - from ._document import Json + from ._limits import Json def schema_error(document: Json) -> str | None: diff --git a/src/python_sbom/_wheel/__init__.py b/src/python_sbom/_wheel/__init__.py index 094cffe..24df4da 100644 --- a/src/python_sbom/_wheel/__init__.py +++ b/src/python_sbom/_wheel/__init__.py @@ -1 +1 @@ -"""Wheel archives: their RECORD, their metadata files, and writing an SBOM into a built one.""" +"""Zip artifacts: one checked handle per build, verifying the wheels they bundle, and writing an SBOM in.""" diff --git a/src/python_sbom/_wheel/_bundled.py b/src/python_sbom/_wheel/_bundled.py index 357cf0e..06d005b 100644 --- a/src/python_sbom/_wheel/_bundled.py +++ b/src/python_sbom/_wheel/_bundled.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib from collections import Counter from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Final @@ -13,8 +12,8 @@ from python_sbom._model._inputs import BundledWheel, RecordRow from python_sbom._model._metadata import CoreMetadata, parse_metadata -from ._archive import open_archive, read_member, read_wheel_metadata from ._record import record_rows +from ._zip import open_archive, read_member, read_wheel_metadata, sha256_hex if TYPE_CHECKING: import zipfile @@ -22,8 +21,6 @@ from typing import IO -# a bundled wheel is hashed in chunks of this size -_CHUNK: Final[int] = 1024 * 1024 # wheels verified at once, each perhaps in memory whole _WORKERS: Final[int] = 4 @@ -41,14 +38,12 @@ def read(path: str) -> BundledWheel: def _read_bundled_wheel(stream: IO[bytes], path: str) -> BundledWheel: where: Final[str] = f"bundled wheel {path}" - digest: Final = hashlib.sha256() - while chunk := stream.read(_CHUNK): - digest.update(chunk) + sha256: Final[str] = sha256_hex(stream) stream.seek(0) with open_archive(stream, where) as archive: # installer takes the project name from the archive's file name, which an archive read from memory lacks archive.filename = path - members: Final[list[str]] = sorted(name for name in archive.namelist() if not name.endswith("/")) + members: Final[list[str]] = sorted(info.filename for info in archive.infolist() if not info.is_dir()) wheel, metadata = read_wheel_metadata(archive, where) dist_info: Final[str] = wheel.dist_info_dir _check_named(metadata, where, f"{dist_info}/METADATA") @@ -63,7 +58,7 @@ def _read_bundled_wheel(stream: IO[bytes], path: str) -> BundledWheel: ] return BundledWheel( path=path, - sha256=digest.hexdigest(), + sha256=sha256, metadata_name=f"{dist_info}/METADATA", metadata=metadata, files=files, diff --git a/src/python_sbom/_wheel/_reader.py b/src/python_sbom/_wheel/_reader.py new file mode 100644 index 0000000..4891971 --- /dev/null +++ b/src/python_sbom/_wheel/_reader.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from io import BytesIO +from typing import TYPE_CHECKING, Final + +from ._bundled import read_bundled_wheels +from ._zip import open_archive, read_member, sha256_hex + +if TYPE_CHECKING: + import zipfile + from collections.abc import Generator + from pathlib import Path, PurePosixPath + + from python_sbom._model._inputs import BundledWheel + +# a bundled wheel is read whole, to hash it and open it as a zip, so its size is capped as metadata's is +_WHEEL_LIMIT: Final[int] = 512 * 1024 * 1024 + + +@contextmanager +def open_artifact(path: Path) -> Generator[ArtifactReader, None, None]: + # one open zip serves every read of a build, the provider's included, so the archive is checked and indexed once + with open_archive(path, str(path)) as archive: + yield ArtifactReader(path=path, archive=archive, members=frozenset(archive.namelist())) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ArtifactReader: + path: Path + archive: zipfile.ZipFile + members: frozenset[str] + + def digest(self, name: str) -> str: + with self.archive.open(name) as member: + return sha256_hex(member) + + def provider_read(self, path: PurePosixPath) -> bytes: + # a provider asking for a member the artifact lacks gets the KeyError a mapping raises, which it may expect + if (name := str(path)) not in self.members: + raise KeyError(name) + return self.read(name) + + def read(self, name: str) -> bytes: + return read_member(self.archive, name, str(self.path)) + + def bundled_wheels(self, names: list[str]) -> list[BundledWheel]: + # zipfile serializes the seeks of concurrent member reads on its own lock, so the verifying threads share this + # handle and inflate side by side + return read_bundled_wheels( + names, lambda name: BytesIO(read_member(self.archive, name, str(self.path), _WHEEL_LIMIT)) + ) + + +__all__ = [ + "ArtifactReader", + "open_artifact", +] diff --git a/src/python_sbom/_wheel/_record.py b/src/python_sbom/_wheel/_record.py index db6e31d..25d9260 100644 --- a/src/python_sbom/_wheel/_record.py +++ b/src/python_sbom/_wheel/_record.py @@ -30,12 +30,12 @@ def record_rows(rows: Iterable[tuple[str, str, str]], where: str) -> list[Record for entry in starmap(RecordEntry.from_elements, rows): if entry.hash_ is None: continue - algorithm, digest = decode_digest(str(entry.hash_), f"{where}: RECORD row for {entry.path}") + algorithm, digest = _decode_digest(str(entry.hash_), f"{where}: RECORD row for {entry.path}") found.append(RecordRow(path=entry.path, algorithm=algorithm, digest=digest, size=str(entry.size))) return found -def decode_digest(digest: str, where: str) -> tuple[str, str]: +def _decode_digest(digest: str, where: str) -> tuple[str, str]: # installer checks RECORD digests as urlsafe base64 under any hashlib name, while CycloneDX takes hex under its own # names for a few algorithms name, _, encoded = digest.partition("=") @@ -51,6 +51,5 @@ def decode_digest(digest: str, where: str) -> tuple[str, str]: __all__ = [ - "decode_digest", "record_rows", ] diff --git a/src/python_sbom/_wheel/_archive.py b/src/python_sbom/_wheel/_write.py similarity index 51% rename from src/python_sbom/_wheel/_archive.py rename to src/python_sbom/_wheel/_write.py index 583ee07..fca97bf 100644 --- a/src/python_sbom/_wheel/_archive.py +++ b/src/python_sbom/_wheel/_write.py @@ -1,41 +1,31 @@ from __future__ import annotations import csv -import hashlib import os import shutil import stat import tempfile import time import zipfile -import zlib -from collections import Counter -from contextlib import contextmanager from io import BytesIO, StringIO from pathlib import Path -from typing import IO, TYPE_CHECKING, Final +from typing import Final import zipremove from installer.records import Hash, RecordEntry -from installer.sources import WheelFile from installer.utils import copyfileobj_with_hashing from python_sbom._errors import SbomError -from python_sbom._model._metadata import CoreMetadata, parse_metadata -if TYPE_CHECKING: - from collections.abc import Generator +from ._zip import open_archive, wheel_source -# metadata files beyond this size are refused, so a hostile wheel cannot exhaust memory -_METADATA_LIMIT: Final[int] = 16 * 1024 * 1024 -_CHUNK: Final[int] = 1024 * 1024 # 1980-01-01, the earliest time a zip entry holds _ZIP_EPOCH: Final[int] = 315532800 def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: with open_archive(wheel, str(wheel)) as archive: - _, dist_info = _wheel_source(archive, str(wheel)) + _, dist_info = wheel_source(archive, str(wheel)) record: Final[str] = f"{dist_info}/RECORD" if missing := sorted({f"{dist_info}/METADATA", record} - set(archive.namelist())): msg = f"{wheel} has no {', '.join(missing)}" @@ -68,37 +58,6 @@ def write_sbom(wheel: Path, name: str, content: bytes, epoch: int) -> str: return sbom -@contextmanager -def open_archive(file: Path | IO[bytes], where: str) -> Generator[zipfile.ZipFile, None, None]: - try: - with zipfile.ZipFile(file) as archive: - # installers extract the first of two same-named members and zipfile reads the last, and a file system that - # ignores case merges names that differ only by case, so such names would hide what gets installed; - # installer's WheelFile accepts names that differ only by case - if clashes := sorted( - name for name, count in Counter(name.casefold() for name in archive.namelist()).items() if count > 1 - ): - msg = f"{where} holds more than one member named {', '.join(clashes)}, ignoring case" - raise SbomError(msg) - yield archive - # zipfile raises these as late as a member's last read, inside installer's checks too, which let them through - except (zipfile.BadZipFile, zlib.error) as error: - msg = f"{where} is not a valid zip archive: {error}" - raise SbomError(msg) from error - - -def _wheel_source(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, str]: - # installer refuses a wheel whose file name does not parse, or that holds other than one top-level .dist-info, since - # a second one is a decoy that could decide whose metadata counts - try: - wheel = WheelFile(archive) - dist_info = wheel.dist_info_dir - except ValueError as error: - msg = f"{where} is not a wheel: {error}" - raise SbomError(msg) from error - return wheel, dist_info - - def _entry(name: str, epoch: int) -> zipfile.ZipInfo: # zip times start at 1980; Unix attributes whichever system writes the entry, so the bytes match across systems entry: Final[zipfile.ZipInfo] = zipfile.ZipInfo(name, time.gmtime(max(epoch, _ZIP_EPOCH))[:6]) @@ -108,15 +67,13 @@ def _entry(name: str, epoch: int) -> zipfile.ZipInfo: def _rewritten_record(record: bytes, name: str, sbom: str, content: bytes) -> bytes: - lines: Final[list[str]] = [ - line for line in record.decode("utf-8").splitlines(keepends=True) if not line.startswith(f"{sbom},") - ] + lines: Final[list[str]] = [line for line in record.decode("utf-8").splitlines(keepends=True) if _path(line) != sbom] newline: Final[str] = "\r\n" if lines and lines[0].endswith("\r\n") else "\n" # a last row without its line end would take the SBOM's row onto its own line if lines and not lines[-1].endswith(("\n", "\r")): lines[-1] += newline # the row of RECORD itself stays last - at: Final[int] = next((index for index, line in enumerate(lines) if line.startswith(f"{name},")), len(lines)) + at: Final[int] = next((index for index, line in enumerate(lines) if _path(line) == name), len(lines)) # installer's helper hashes as it copies; the copy is thrown away digest, size = copyfileobj_with_hashing(BytesIO(content), BytesIO(), "sha256") row: Final[StringIO] = StringIO() @@ -125,44 +82,9 @@ def _rewritten_record(record: bytes, name: str, sbom: str, content: bytes) -> by return "".join(lines).encode() -def read_wheel_metadata(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, CoreMetadata]: - wheel, dist_info = _wheel_source(archive, where) - # read capped, as installer's read_dist_info decompresses the member whole - return wheel, parse_metadata(read_member(archive, f"{dist_info}/METADATA", where).decode("utf-8")) - - -def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _METADATA_LIMIT) -> bytes: - try: - info = archive.getinfo(name) - except KeyError as error: - msg = f"{where} has no {name}" - raise SbomError(msg) from error - # counted as it decompresses, since the size the archive states may lie and one read decompresses it all at once - content: Final[bytearray] = bytearray() - with archive.open(info) as member: - while chunk := member.read(_CHUNK): - content.extend(chunk) - if len(content) > limit: - msg = f"{where}: {name} is over the {limit} byte limit" - raise SbomError(msg) - return bytes(content) - - -def bundled_wheels_by_digest(wheel: Path) -> dict[str, list[str]]: - found: Final[dict[str, list[str]]] = {} - with open_archive(wheel, str(wheel)) as archive: - for name in sorted(archive.namelist()): - if name.endswith(".whl"): - found.setdefault(member_digest(archive, name, "sha256"), []).append(name) - return found - - -def member_digest(archive: zipfile.ZipFile, name: str | zipfile.ZipInfo, algorithm: str) -> str: - digest: Final = hashlib.new(algorithm) - with archive.open(name) as member: - while chunk := member.read(_CHUNK): - digest.update(chunk) - return digest.hexdigest() +def _path(line: str) -> str: + # RECORD quotes a path that holds a comma, a quote or a line end, so the raw start of a line misses its row + return next(csv.reader([line]), [""])[0] def append_member(archive: Path, name: str, content: bytes, epoch: int) -> None: @@ -177,10 +99,5 @@ def append_member(archive: Path, name: str, content: bytes, epoch: int) -> None: __all__ = [ "append_member", - "bundled_wheels_by_digest", - "member_digest", - "open_archive", - "read_member", - "read_wheel_metadata", "write_sbom", ] diff --git a/src/python_sbom/_wheel/_zip.py b/src/python_sbom/_wheel/_zip.py new file mode 100644 index 0000000..322f0f0 --- /dev/null +++ b/src/python_sbom/_wheel/_zip.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import hashlib +import zipfile +import zlib +from collections import Counter +from contextlib import contextmanager +from typing import IO, TYPE_CHECKING, Final + +from installer.sources import WheelFile + +from python_sbom._errors import SbomError +from python_sbom._model._metadata import CoreMetadata, parse_metadata + +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + +# metadata files beyond this size are refused, so a hostile wheel cannot exhaust memory +_METADATA_LIMIT: Final[int] = 16 * 1024 * 1024 +# members inflate a chunk at a time, so a size the archive states falsely is caught before it is all in memory +CHUNK: Final[int] = 1024 * 1024 + + +@contextmanager +def open_archive(file: Path | IO[bytes], where: str) -> Generator[zipfile.ZipFile, None, None]: + try: + with zipfile.ZipFile(file) as archive: + # installers extract the first of two same-named members and zipfile reads the last, and a file system that + # ignores case merges names that differ only by case, so such names would hide what gets installed; + # installer's WheelFile accepts names that differ only by case + if clashes := sorted( + name for name, count in Counter(name.casefold() for name in archive.namelist()).items() if count > 1 + ): + msg = f"{where} holds more than one member named {', '.join(clashes)}, ignoring case" + raise SbomError(msg) + yield archive + # zipfile raises these as late as a member's last read, inside installer's checks too, which let them through + except (zipfile.BadZipFile, zlib.error) as error: + msg = f"{where} is not a valid zip archive: {error}" + raise SbomError(msg) from error + + +def read_wheel_metadata(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, CoreMetadata]: + wheel, dist_info = wheel_source(archive, where) + # read capped, as installer's read_dist_info decompresses the member whole + return wheel, parse_metadata(read_member(archive, f"{dist_info}/METADATA", where).decode("utf-8")) + + +def wheel_source(archive: zipfile.ZipFile, where: str) -> tuple[WheelFile, str]: + # installer refuses a wheel whose file name does not parse, or that holds other than one top-level .dist-info, since + # a second one is a decoy that could decide whose metadata counts + try: + wheel = WheelFile(archive) + dist_info = wheel.dist_info_dir + except ValueError as error: + msg = f"{where} is not a wheel: {error}" + raise SbomError(msg) from error + return wheel, dist_info + + +def read_member(archive: zipfile.ZipFile, name: str, where: str, limit: int = _METADATA_LIMIT) -> bytes: + try: + info = archive.getinfo(name) + except KeyError as error: + msg = f"{where} has no {name}" + raise SbomError(msg) from error + # counted as it decompresses, since the size the archive states may lie and one read decompresses it all at once + content: Final[bytearray] = bytearray() + with archive.open(info) as member: + while chunk := member.read(CHUNK): + content.extend(chunk) + if len(content) > limit: + msg = f"{where}: {name} is over the {limit} byte limit" + raise SbomError(msg) + return bytes(content) + + +def sha256_hex(stream: IO[bytes]) -> str: + # hashlib.file_digest wants readinto, which a zip member stream typed as IO[bytes] does not promise + digest: Final = hashlib.sha256() + while chunk := stream.read(CHUNK): + digest.update(chunk) + return digest.hexdigest() + + +__all__ = [ + "CHUNK", + "open_archive", + "read_member", + "read_wheel_metadata", + "sha256_hex", + "wheel_source", +] diff --git a/src/python_sbom/hatch/_hook.py b/src/python_sbom/hatch/_hook.py index 1655c05..455412e 100644 --- a/src/python_sbom/hatch/_hook.py +++ b/src/python_sbom/hatch/_hook.py @@ -99,10 +99,11 @@ def finalize( return # what a wheel needs loads only when one is built, so an sdist build stays light from python_sbom._collect._facts import collect_facts # ruff: ignore[import-outside-top-level] # see above - from python_sbom._collect._source import wheel_request # ruff: ignore[import-outside-top-level] # see above from python_sbom._generate._embed import embed_into # ruff: ignore[import-outside-top-level] # see above root = Path(self.root) + # content_provider() may be a subclass's override, so its result goes through resolve_provider again, which + # returns a given instance as is, for the provider's source files to be listed among the build tools content, provider_scripts = resolve_provider(self.content_provider(), self._settings, root) # a custom hook script shapes the document, as a provider defined in the project does, so the SBOM lists both # among the build tools @@ -115,8 +116,9 @@ def finalize( lambda _root: self._revision, default_epoch=get_reproducible_timestamp() if self.build_config.reproducible else None, ) - wheel = Path(artifact_path) - embed_into(wheel, wheel_request(wheel, root, self._settings, scripts), content, facts) + embed_into( + Path(artifact_path), root, settings=self._settings, content=content, build_scripts=scripts, facts=facts + ) def content_provider(self) -> ContentProvider: """ diff --git a/tests/_cli/test_archive.py b/tests/_cli/test_archive.py index ce2eb12..23e3f97 100644 --- a/tests/_cli/test_archive.py +++ b/tests/_cli/test_archive.py @@ -10,6 +10,7 @@ from python_sbom._cli._main import main if TYPE_CHECKING: + from collections.abc import Callable from pathlib import Path @@ -73,3 +74,61 @@ def test_archive_embed_leaves_no_temporary_files(project: Path, zipapp: Path, te [], ["app.pyz", "app.pyz.cdx.json", "project", "temp"], ) + + +def direct(archive: Path) -> Path: + return archive + + +def hardlink(archive: Path) -> Path: + (alias := archive.with_name("alias.pyz")).hardlink_to(archive) + return alias + + +def symlink(archive: Path) -> Path: + alias = archive.with_name("alias.pyz") + try: + alias.symlink_to(archive) + except OSError: # pragma: no cover # only without the privilege to create symbolic links + pytest.skip("symbolic links need privileges on this platform") + return alias + + +@pytest.mark.parametrize( + "alias", + [ + pytest.param(direct, id="direct"), + pytest.param(hardlink, id="hardlink"), + pytest.param(symlink, id="symlink"), + ], +) +@pytest.mark.parametrize( + "flags", + [pytest.param([], id="sbom"), pytest.param(["--embed"], id="embed")], +) +def test_archive_refuses_output_over_archive( + project: Path, + zipapp: Path, + alias: Callable[[Path], Path], + flags: list[str], + capsys: pytest.CaptureFixture[str], +) -> None: + output = alias(zipapp) + before = zipapp.read_bytes() + arguments = [ + "archive", + str(zipapp), + "--root", + str(project), + "--build-env", + str(project), + "--output", + str(output), + *flags, + ] + with pytest.raises(SystemExit): + main(arguments) + assert (zipapp.read_bytes(), capsys.readouterr().err) == ( + before, + f"python-sbom: {output} is the archive {zipapp} itself, refusing to overwrite it with the SBOM\n", + ) diff --git a/tests/_cli/test_spdx.py b/tests/_cli/test_spdx.py index ecfa172..fde3b1b 100644 --- a/tests/_cli/test_spdx.py +++ b/tests/_cli/test_spdx.py @@ -18,7 +18,8 @@ def test_spdx(tmp_path: Path, describe: Describe) -> None: (source := tmp_path / "example.cdx.json").write_bytes(dumps(describe())) main(["spdx", str(source), str(target := tmp_path / "example.spdx.json")]) - assert json.loads(target.read_text(encoding="utf-8"))["spdxVersion"] == "SPDX-2.3" + text = target.read_bytes() + assert (text.endswith(b"}\n"), b"\r\n" in text, json.loads(text)["spdxVersion"]) == (True, False, "SPDX-2.3") def test_spdx_refuses_non_document(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: @@ -28,9 +29,21 @@ def test_spdx_refuses_non_document(tmp_path: Path, capsys: pytest.CaptureFixture assert capsys.readouterr().err == f"python-sbom: {source} is not a CycloneDX document python-sbom can render\n" -def test_spdx_refuses_malformed_document(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - document = {"metadata": {"component": {"bom-ref": "app", "name": "app"}}, "components": [{"name": "x"}]} - (source := tmp_path / "broken.json").write_text(json.dumps(document), encoding="utf-8") +@pytest.mark.parametrize( + "document", + [ + pytest.param( + json.dumps({"metadata": {"component": {"bom-ref": "app", "name": "app"}}, "components": [{"name": "x"}]}), + id="component-missing-fields", + ), + pytest.param( + json.dumps({"metadata": {"component": {"bom-ref": "app", "name": "app"}, "tools": 1}, "components": []}), + id="tools-not-object", + ), + ], +) +def test_spdx_refuses_malformed_document(document: str, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + (source := tmp_path / "broken.json").write_text(document, encoding="utf-8") with pytest.raises(SystemExit): main(["spdx", str(source), str(tmp_path / "out.json")]) assert capsys.readouterr().err.startswith( diff --git a/tests/_collect/test_archive.py b/tests/_collect/test_archive.py index 24ef924..ff8aab8 100644 --- a/tests/_collect/test_archive.py +++ b/tests/_collect/test_archive.py @@ -2,34 +2,15 @@ import hashlib import zipfile -from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING import pytest -from python_sbom._collect._archive import member_reader, read_archive -from python_sbom._errors import SbomError +from python_sbom._collect._archive import read_archive +from python_sbom._wheel._reader import open_artifact - -def test_member_reader_reads_a_member(tmp_path: Path) -> None: - with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: - archive.writestr("embed/table.py", "x = 1\n") - assert member_reader(wheel)(PurePosixPath("embed/table.py")) == b"x = 1\n" - - -def test_member_reader_misses(tmp_path: Path) -> None: - with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: - archive.writestr("other.py", "") - with pytest.raises(KeyError, match=r"missing\.py"): - member_reader(wheel)(PurePosixPath("missing.py")) - - -def test_member_reader_checks_the_archive(tmp_path: Path) -> None: - # the same checks as every other read, so names equal ignoring case are refused here too - with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: - archive.writestr("table.py", "") - archive.writestr("TABLE.py", "") - with pytest.raises(SbomError, match=r"holds more than one member named table\.py, ignoring case"): - member_reader(wheel)(PurePosixPath("table.py")) +if TYPE_CHECKING: + from pathlib import Path @pytest.mark.parametrize( @@ -45,5 +26,6 @@ def test_archive_member_names_never_reach_the_host(tmp_path: Path, name: str) -> with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w") as handle: handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\n") handle.writestr(zipfile.ZipInfo(name), b"inside the archive") - row = read_archive(archive).files[name] + with open_artifact(archive) as opened: + row = read_archive(opened).files[name] assert (row.digest, row.size) == (hashlib.sha256(b"inside the archive").hexdigest(), "18") diff --git a/tests/_collect/test_content.py b/tests/_collect/test_content.py index 8180605..71b06f5 100644 --- a/tests/_collect/test_content.py +++ b/tests/_collect/test_content.py @@ -1,6 +1,7 @@ from __future__ import annotations import dataclasses +import sys from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING @@ -117,3 +118,28 @@ def test_load_refuses(tmp_path: Path, module: str, spec: str, message: str, sett def test_provider_without_source_is_no_tool(tmp_path: Path, settings: Settings) -> None: provider = type("Generated", (ContentProvider,), {"__module__": "not_a_loaded_module"})() assert resolve_provider(provider, settings, tmp_path)[1] == () + + +def test_provider_of_another_project_is_not_reused(tmp_path: Path, settings: Settings) -> None: + # two projects whose providers share a module name, built in one process as hatch builds them + name = f"provider_{tmp_path.name}" + for project, value in (("first", "1"), ("second", "2")): + (tmp_path / project).mkdir() + (tmp_path / project / f"{name}.py").write_text( + f"from python_sbom import ContentProvider\n\n\nclass Content(ContentProvider):\n value = {value}\n", + encoding="utf-8", + ) + try: + resolve_provider(f"{name}:Content", settings, tmp_path / "first") + with pytest.raises( + SbomError, match=rf"this process already imported .*first.*{name}\.py, a module of the same" + ): + resolve_provider(f"{name}:Content", settings, tmp_path / "second") + finally: + sys.modules.pop(name, None) + + +def test_provider_of_the_same_project_loads_again(tmp_path: Path, module: str, settings: Settings) -> None: + first, _ = resolve_provider(f"{module}:Content", settings, tmp_path) + second, _ = resolve_provider(f"{module}:Content", settings, tmp_path) + assert type(first) is type(second) diff --git a/tests/_collect/test_source.py b/tests/_collect/test_source.py index 19ce26a..5968cad 100644 --- a/tests/_collect/test_source.py +++ b/tests/_collect/test_source.py @@ -3,25 +3,21 @@ import dataclasses import hashlib import re -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Final +from pathlib import PurePosixPath +from typing import TYPE_CHECKING import pytest -from python_sbom._collect._source import read_source_tree, wheel_request +from python_sbom._collect._source import copyright_notice, hash_build_scripts, selected_wheels from python_sbom._errors import SbomError -from python_sbom._model._inputs import BuildScript, SbomRequest -from python_sbom._model._metadata import parse_metadata +from python_sbom._model._inputs import BuildScript if TYPE_CHECKING: from collections.abc import Mapping - - from conftest import WriteWheel + from pathlib import Path from python_sbom._model._inputs import Settings -_PIP: Final[dict[str, bytes]] = {"pip/__init__.py": b"", "pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n"} - @pytest.mark.parametrize( ("files", "expected"), @@ -35,29 +31,18 @@ ), ], ) -def test_copyright(tmp_path: Path, settings: Settings, files: Mapping[str, str], expected: str | None) -> None: +def test_copyright(tmp_path: Path, files: Mapping[str, str], expected: str | None) -> None: for name, text in files.items(): (tmp_path / name).write_text(text, encoding="utf-8") - source = read_source_tree(request(tmp_path, settings, "License-File: LICENSE\nLicense-File: NOTICE\n"), inside={}) - assert source.copyright == expected - - -def request(root: Path, settings: Settings, metadata: str = "", scripts: tuple[PurePosixPath, ...] = ()) -> SbomRequest: - return SbomRequest( - root=root, - metadata=parse_metadata(f"Name: example\nVersion: 1.0\n{metadata}"), - wheel_name="example-1.0-py3-none-any.whl", - settings=settings, - build_scripts=scripts, - ) + assert copyright_notice(tmp_path, ["LICENSE", "NOTICE"]) == expected -def test_bundled_wheels_sorted_once(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: +def test_selected_wheels_sorted_once(tmp_path: Path, settings: Settings) -> None: for directory in ("b", "a"): - write_wheel(tmp_path / "src" / directory / "pip-1.0-py3-none-any.whl", _PIP) + (tmp_path / "src" / directory).mkdir(parents=True) + (tmp_path / "src" / directory / "pip-1.0-py3-none-any.whl").write_bytes(b"") settings = dataclasses.replace(settings, bundled_wheels=("src/*/*.whl", "src/a/*.whl")) - paths = [wheel.path for wheel in read_source_tree(request(tmp_path, settings), inside={}).bundled] - assert paths == ["src/a/pip-1.0-py3-none-any.whl", "src/b/pip-1.0-py3-none-any.whl"] + assert selected_wheels(tmp_path, settings) == ["src/a/pip-1.0-py3-none-any.whl", "src/b/pip-1.0-py3-none-any.whl"] @pytest.mark.parametrize( @@ -68,58 +53,28 @@ def test_bundled_wheels_sorted_once(tmp_path: Path, settings: Settings, write_wh pytest.param("ROOT/outside/*.whl", id="absolute"), ], ) -def test_bundled_wheel_pattern_outside_project( - tmp_path: Path, settings: Settings, write_wheel: WriteWheel, pattern: str -) -> None: - write_wheel(tmp_path / "outside" / "pip-1.0-py3-none-any.whl", _PIP) +def test_bundled_wheel_pattern_outside_project(tmp_path: Path, settings: Settings, pattern: str) -> None: + (tmp_path / "outside").mkdir() + (tmp_path / "outside" / "pip-1.0-py3-none-any.whl").write_bytes(b"") (project := tmp_path / "project" / "src").mkdir(parents=True) pattern = pattern.replace("ROOT", tmp_path.as_posix()) settings = dataclasses.replace(settings, bundled_wheels=(pattern,)) with pytest.raises(SbomError, match=f"`bundled-wheels` pattern {re.escape(repr(pattern))} must be relative"): - read_source_tree(request(project.parent, settings), inside={}) + selected_wheels(project.parent, settings) -def test_build_scripts_hashed(tmp_path: Path, settings: Settings) -> None: +def test_build_scripts_hashed_once(tmp_path: Path) -> None: (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") script = PurePosixPath("hatch_build.py") - source = read_source_tree(request(tmp_path, settings, scripts=(script,)), inside={}) - assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] + assert hash_build_scripts(tmp_path, [script, script]) == [ + BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest()) + ] -def test_build_script_outside_project(tmp_path: Path, settings: Settings) -> None: +def test_build_script_outside_project(tmp_path: Path) -> None: # a custom hook path may be absolute, which on Windows carries a drive the join with the root must keep (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") script = PurePosixPath((tmp_path / "hatch_build.py").as_posix()) - source = read_source_tree(request(tmp_path / "project", settings, scripts=(script,)), inside={}) - assert source.build_scripts == [BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest())] - - -def test_wheel_request(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: - wheel = write_wheel(tmp_path / "native-1.0-py3-none-any.whl", {"native-1.0.dist-info/METADATA": b"Name: native\n"}) - request = wheel_request(wheel, tmp_path, settings, (PurePosixPath("build.py"),)) - assert (request.root, request.metadata.name, request.wheel_name, request.settings, request.build_scripts) == ( - tmp_path, - "native", - "native-1.0-py3-none-any.whl", - settings, - (PurePosixPath("build.py"),), - ) - - -@pytest.mark.parametrize( - ("name", "message"), - [ - pytest.param("native.whl", "is not a wheel: Not a valid wheel filename", id="file-name"), - pytest.param( - "native-1.0-py3-none-any.whl", - r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", - id="no-dist-info", - ), - ], -) -def test_wheel_request_needs_a_wheel( - tmp_path: Path, settings: Settings, write_wheel: WriteWheel, name: str, message: str -) -> None: - archive = write_wheel(tmp_path / name, {"native/__init__.py": b""}, record=False) - with pytest.raises(SbomError, match=message): - wheel_request(archive, Path(), settings, ()) + assert hash_build_scripts(tmp_path / "project", [script]) == [ + BuildScript(path=script, sha256=hashlib.sha256(b"hook = 1\n").hexdigest()) + ] diff --git a/tests/_collect/test_wheel.py b/tests/_collect/test_wheel.py new file mode 100644 index 0000000..cd3dd19 --- /dev/null +++ b/tests/_collect/test_wheel.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import dataclasses +import hashlib +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Final + +import pytest + +from python_sbom._collect._wheel import read_wheel +from python_sbom._errors import SbomError +from python_sbom._model._inputs import BuildScript, WheelTree +from python_sbom._wheel._reader import open_artifact + +if TYPE_CHECKING: + from collections.abc import Mapping + + from conftest import WriteWheel + + from python_sbom._model._inputs import Settings + +_PIP: Final[dict[str, bytes]] = {"pip/__init__.py": b"", "pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\n"} +_NATIVE: Final[dict[str, bytes]] = { + "native-1.0.dist-info/METADATA": b"Name: native\nVersion: 1.0\nLicense-File: LICENSE\n" +} + + +def test_read_wheel(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: + (tmp_path / "LICENSE").write_text("MIT\n\nCopyright (c) 2026 Example\n", encoding="utf-8") + (tmp_path / "hatch_build.py").write_bytes(b"hook = 1\n") + tree = tree_of( + write_wheel(tmp_path / "native-1.0-py3-none-any.whl", _NATIVE), + tmp_path, + settings, + PurePosixPath("hatch_build.py"), + ) + assert (tree.name, tree.metadata.name, tree.copyright, tree.bundled, tree.build_scripts) == ( + "native-1.0-py3-none-any.whl", + "native", + "Copyright (c) 2026 Example", + [], + [BuildScript(path=PurePosixPath("hatch_build.py"), sha256=hashlib.sha256(b"hook = 1\n").hexdigest())], + ) + + +def tree_of(wheel: Path, root: Path, settings: Settings, *scripts: PurePosixPath) -> WheelTree: + with open_artifact(wheel) as artifact: + return read_wheel(artifact, root, settings, scripts) + + +def test_bundled_wheels_read_from_the_wheel(tmp_path: Path, settings: Settings, write_wheel: WriteWheel) -> None: + # identical copies, selected twice over, take the wheel's copies in turn, each keeping an identity of its own + for directory in ("b", "a"): + content = write_wheel(tmp_path / "src" / directory / "pip-1.0-py3-none-any.whl", _PIP).read_bytes() + wheel = built( + write_wheel, tmp_path, {"pkg/b/pip-1.0-py3-none-any.whl": content, "pkg/a/pip-1.0-py3-none-any.whl": content} + ) + settings = dataclasses.replace(settings, bundled_wheels=("src/*/*.whl", "src/a/*.whl")) + assert [bundled.path for bundled in tree_of(wheel, tmp_path, settings).bundled] == [ + "pkg/a/pip-1.0-py3-none-any.whl", + "pkg/b/pip-1.0-py3-none-any.whl", + ] + + +def built(write_wheel: WriteWheel, root: Path, bundled: Mapping[str, bytes]) -> Path: + return write_wheel(root / "dist" / "native-1.0-py3-none-any.whl", {**_NATIVE, **bundled}) + + +@pytest.mark.parametrize( + "shipped", + [ + pytest.param({"pkg/pip-1.0-py3-none-any.whl": b"the bytes of the build"}, id="changed-since-the-build"), + pytest.param({}, id="left-out-of-the-build"), + ], +) +def test_bundled_wheel_not_shipped_is_refused( + tmp_path: Path, settings: Settings, write_wheel: WriteWheel, shipped: dict[str, bytes] +) -> None: + write_wheel(tmp_path / "src" / "pip-1.0-py3-none-any.whl", _PIP) + settings = dataclasses.replace(settings, bundled_wheels=("src/*.whl",)) + message = r"bundled wheel src/pip-1\.0-py3-none-any\.whl is not in native-1\.0-py3-none-any\.whl byte for byte" + with pytest.raises(SbomError, match=message): + tree_of(built(write_wheel, tmp_path, shipped), tmp_path, settings) + + +def test_more_bundled_copies_than_shipped_is_refused( + tmp_path: Path, settings: Settings, write_wheel: WriteWheel +) -> None: + for directory in ("a", "b"): + content = write_wheel(tmp_path / "src" / directory / "pip-1.0-py3-none-any.whl", _PIP).read_bytes() + settings = dataclasses.replace(settings, bundled_wheels=("src/*/*.whl",)) + with pytest.raises(SbomError, match=r"bundled wheel src/b/pip-1\.0-py3-none-any\.whl is not in native"): + tree_of(built(write_wheel, tmp_path, {"pkg/pip-1.0-py3-none-any.whl": content}), tmp_path, settings) + + +@pytest.mark.parametrize( + ("name", "message"), + [ + pytest.param("native.whl", "is not a wheel: Not a valid wheel filename", id="file-name"), + pytest.param( + "native-1.0-py3-none-any.whl", + r"is not a wheel: Wheel doesn't contain exactly one \.dist-info directory", + id="no-dist-info", + ), + ], +) +def test_read_wheel_needs_a_wheel( + tmp_path: Path, settings: Settings, write_wheel: WriteWheel, name: str, message: str +) -> None: + archive = write_wheel(tmp_path / name, {"native/__init__.py": b""}, record=False) + with pytest.raises(SbomError, match=message): + tree_of(archive, Path(), settings) diff --git a/tests/_formats/test_spdx.py b/tests/_formats/test_spdx.py index fd5909f..31bd2e7 100644 --- a/tests/_formats/test_spdx.py +++ b/tests/_formats/test_spdx.py @@ -9,7 +9,7 @@ import pytest from python_sbom._formats._json import dumps -from python_sbom._formats._spdx import render_spdx, to_spdx +from python_sbom._formats._spdx import render_spdx from python_sbom._validate._schemas import spdx_problems from python_sbom.version import __version__ @@ -109,6 +109,19 @@ def test_spdx_ids_are_valid_and_tell_similar_refs_apart(describe: Describe) -> N assert len(set(ids)) == len(ids) +def test_spdx_ids_of_refs_differing_in_dropped_characters_stay_distinct(describe: Describe) -> None: + document = describe() + document["components"] = [ + {"type": "library", "bom-ref": ref, "name": "pip", "version": "1.0"} + for ref in [ + "bundled:foo---__-__-__---____--/pip-1.0-py3-none-any.whl", + "bundled:foo---___-_-_--___---_-/pip-1.0-py3-none-any.whl", + ] + ] + ids = [package["SPDXID"] for package in render_spdx(document)["packages"] if package["name"] == "pip"] + assert len(set(ids)) == 2 + + def test_repository_names_the_namespace(describe: Describe, settings: Settings) -> None: document = describe(settings=dataclasses.replace(settings, repository="https://github.com/example/example")) namespace = render_spdx(document)["documentNamespace"] @@ -215,11 +228,6 @@ def test_package_purpose(describe: Describe, kind: str, purpose: str) -> None: assert package["primaryPackagePurpose"] == purpose -def test_to_spdx_text(describe: Describe) -> None: - text = to_spdx(describe()) - assert (text.endswith("}\n"), json.loads(text)["spdxVersion"]) == (True, "SPDX-2.3") - - def test_other_tools_document_renders_valid_spdx() -> None: assert spdx_problems(render_spdx(json.loads(_FOREIGN))) == [] diff --git a/tests/_generate/test_archive.py b/tests/_generate/test_archive.py index 3a50b77..960528e 100644 --- a/tests/_generate/test_archive.py +++ b/tests/_generate/test_archive.py @@ -226,9 +226,7 @@ def test_archive_without_project_distribution(tmp_path: Path) -> None: with zipfile.ZipFile(buffer, "w") as handle: handle.writestr("__main__.py", "") (archive := tmp_path / "bare.pyz").write_bytes(buffer.getvalue()) - with pytest.raises( - SbomError, match=r"bare\.pyz holds 0 top-level \.dist-info/METADATA, not the one naming the project" - ): + with pytest.raises(SbomError, match=r"bare\.pyz holds no top-level \.dist-info/METADATA naming the project"): describe_archive( tmp_path, archive, facts=collect_facts(tmp_path, revision_reader(None)), provider=None, build_scripts=() ) @@ -272,3 +270,30 @@ def test_archive_refuses_distributions_sharing_a_directory(project: Path, tmp_pa describe_archive( project, archive, facts=collect_facts(project, revision_reader(None)), provider=None, build_scripts=() ) + + +@pytest.mark.parametrize("placement", [pytest.param("", id="top-level"), pytest.param("app/", id="nested")]) +def test_archive_bundled_wheel_judged_wherever_it_sits( + project: Path, tmp_path: Path, write_wheel: WriteWheel, placement: str +) -> None: + # the license policy and the vendored edges cover a wheel at the archive's top level as they do one nested deeper + wheel = write_wheel( + tmp_path / "pip-1.0-py3-none-any.whl", + { + "pip-1.0.dist-info/METADATA": b"Name: pip\nVersion: 1.0\nLicense-Expression: GPL-3.0-only\n", + "pip/_vendor/vendor.txt": b"urllib3==1.26.4\n", + }, + ) + with zipfile.ZipFile(archive := tmp_path / "app.pyz", "w") as handle: + handle.writestr("app-1.0.dist-info/METADATA", "Name: app\nVersion: 1.0\nLicense-Expression: MIT\n") + handle.writestr(f"{placement}{wheel.name}", wheel.read_bytes()) + document = describe_archive( + project, archive, facts=collect_facts(project, revision_reader(None)), provider=None, build_scripts=() + ) + (sbom := tmp_path / "app.pyz.cdx.json").write_text(json.dumps(document), encoding="utf-8") + ref = f"bundled:{placement}{wheel.name}" + vendoring = [entry["ref"] for entry in document["dependencies"] if entry["ref"].startswith("bundled:")] + assert (validate(sbom, frozenset({"MIT"})), vendoring) == ( + [f"{ref}: license GPL-3.0-only is outside the allowed licenses ['MIT']"], + [ref], + ) diff --git a/tests/_generate/test_embed.py b/tests/_generate/test_embed.py index 324bdc3..46b2d4b 100644 --- a/tests/_generate/test_embed.py +++ b/tests/_generate/test_embed.py @@ -7,19 +7,15 @@ from typing import TYPE_CHECKING, Final from python_sbom import BundledWheelInfo, ContentProvider, ProviderContext -from python_sbom._formats._json import dumps -from python_sbom._generate._embed import build_sbom, embed_sbom -from python_sbom._model._inputs import SbomRequest, SourceTree -from python_sbom._model._metadata import parse_metadata +from python_sbom._generate._embed import embed_sbom if TYPE_CHECKING: from pathlib import PurePosixPath import pytest - from conftest import ContextFor, Install, WriteWheel + from conftest import Install, WriteWheel from pytest_mock import MockerFixture - from python_sbom._model._inputs import BuildFacts, Settings _PINNED: Final[Path] = Path(__file__).parent / "pinned" @@ -151,18 +147,3 @@ def test_embed_output_is_pinned( def _sha256(content: bytes) -> str: return hashlib.sha256(content).hexdigest() - - -def test_sbom_file_name_and_content(settings: Settings, facts: BuildFacts, context_for: ContextFor) -> None: - request = SbomRequest( - root=Path(), - metadata=parse_metadata("Name: My.Project\nVersion: 1.0\n"), - wheel_name="my_project-1.0-py3-none-any.whl", - settings=settings, - build_scripts=(), - ) - context = context_for(Path(), Path()) - sbom = build_sbom( - request, SourceTree(copyright=None, bundled=[], build_scripts=[]), facts, ContentProvider(), context - ) - assert (sbom.file_name, sbom.content) == ("my-project.cdx.json", dumps(sbom.document)) diff --git a/tests/_model/test_bom.py b/tests/_model/test_bom.py index e21b087..2b0b632 100644 --- a/tests/_model/test_bom.py +++ b/tests/_model/test_bom.py @@ -9,7 +9,7 @@ import pytest from python_sbom import BundledWheelInfo, ContentProvider, Contributions, ProviderContext -from python_sbom._model._inputs import BuildScript, BundledWheel, Revision, SourceTree +from python_sbom._model._inputs import BuildScript, BundledWheel, Revision from python_sbom._model._metadata import parse_metadata if TYPE_CHECKING: @@ -135,7 +135,7 @@ def test_root_component( "License-File: LICENSE\nProject-URL: Source, https://github.com/example/example/\n", settings=dataclasses.replace(settings, supplier=supplier), facts=dataclasses.replace(facts, revision=Revision(commit="c0ffee", dirty=dirty)), - source=SourceTree(copyright="Copyright (c) 2026 Example", bundled=[], build_scripts=[]), + copyright_line="Copyright (c) 2026 Example", ) repository = "https://github.com/example/example" assert document["metadata"]["component"] == { @@ -262,7 +262,7 @@ def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> Bundl properties=[{"name": f"{context.identity.property_prefix}:{self._name}", "value": str(wheel)}] ) - document = describe(source=SourceTree(copyright=None, bundled=[wheel], build_scripts=[]), content=Seeded("seeded")) + document = describe(bundled=[wheel], content=Seeded("seeded")) assert document["components"][0]["properties"][1] == { "name": "example:seeded", "value": "src/pip-1.0-py3-none-any.whl", @@ -285,15 +285,10 @@ def bundled_wheel(self, context: ProviderContext, wheel: PurePosixPath) -> Bundl def test_build_script_is_a_linked_tool( describe: Describe, settings: Settings, facts: BuildFacts, revision: Revision | None, tree: str ) -> None: - source = SourceTree( - copyright=None, - bundled=[], - build_scripts=[BuildScript(path=PurePosixPath("hatch_build.py"), sha256="ef" * 32)], - ) document = describe( settings=dataclasses.replace(settings, repository="https://github.com/example/example"), facts=dataclasses.replace(facts, revision=revision), - source=source, + build_scripts=[BuildScript(path=PurePosixPath("hatch_build.py"), sha256="ef" * 32)], ) assert document["metadata"]["tools"]["components"][0] == { "type": "application", @@ -394,7 +389,7 @@ def test_bundled_wheel_supplier_is_the_projects_choice( ) document = describe( settings=dataclasses.replace(settings, supplier=_ORG), - source=SourceTree(copyright=None, bundled=[wheel], build_scripts=[]), + bundled=[wheel], content=content, ) assert document["components"][0].get("supplier") == expected diff --git a/tests/_model/test_components.py b/tests/_model/test_components.py index 16b8e88..c09ed62 100644 --- a/tests/_model/test_components.py +++ b/tests/_model/test_components.py @@ -5,6 +5,7 @@ import pytest +from python_sbom import BundledWheelInfo from python_sbom._model._components import ( bundled_wheel_component, component_from_metadata, @@ -201,7 +202,7 @@ def test_bundled_wheel_component() -> None: ) supplier: OrganizationalEntity = {"name": "PyPA", "url": ["https://www.pypa.io"]} properties: list[Property] = [{"name": "example:seeded-for-python", "value": "3.15"}] - assert bundled_wheel_component(wheel, "example", properties, supplier) == { + assert bundled_wheel_component(wheel, "example", BundledWheelInfo(properties=properties, supplier=supplier)) == { "type": "library", "bom-ref": f"bundled:{_WHEEL}", "name": "pip", @@ -256,7 +257,7 @@ def test_bundled_wheel_vendored_metadata_wins_over_manifest() -> None: (dist_info, parse_metadata("Name: urllib3\nVersion: 1.26.4\nLicense-Expression: MIT\n")), ], ) - component = bundled_wheel_component(wheel, "example", [], None) + component = bundled_wheel_component(wheel, "example", BundledWheelInfo()) assert component["components"] == [ { "type": "library", @@ -290,7 +291,7 @@ def component(path: str) -> Component: files=[], vendored=[], ) - return bundled_wheel_component(wheel, "example", [], None) + return bundled_wheel_component(wheel, "example", BundledWheelInfo()) refs = {component(f"src/{directory}/pip-1.0-py3-none-any.whl")["bom-ref"] for directory in ("a", "b")} assert refs == {"bundled:src/a/pip-1.0-py3-none-any.whl", "bundled:src/b/pip-1.0-py3-none-any.whl"} @@ -305,7 +306,7 @@ def test_bundled_wheel_supplier_never_comes_from_the_wheel() -> None: files=[], vendored=[], ) - component = bundled_wheel_component(wheel, "example", [], None) + component = bundled_wheel_component(wheel, "example", BundledWheelInfo()) assert "supplier" not in component @@ -340,7 +341,7 @@ def test_vendored_spellings_of_one_package_merge() -> None: ), ], ) - component = bundled_wheel_component(wheel, "example", [], None) + component = bundled_wheel_component(wheel, "example", BundledWheelInfo()) assert [child["name"] for child in component.get("components", [])] == ["jaraco_text"] diff --git a/tests/_validate/test_document.py b/tests/_validate/test_document.py index fb0f153..55e1354 100644 --- a/tests/_validate/test_document.py +++ b/tests/_validate/test_document.py @@ -20,7 +20,7 @@ from python_sbom import Component, License from python_sbom._model._inputs import BuildFacts from python_sbom._model._types import Document - from python_sbom._validate._document import Json + from python_sbom._validate._limits import Json # what auditwheel writes: CycloneDX 1.4 conventions, a legacy tools array, no timestamp _FOREIGN: Final[bytes] = json.dumps({ @@ -155,7 +155,7 @@ def test_serial_is_required(describe: Describe) -> None: def test_duplicate_refs(describe: Describe) -> None: library: Component = {"type": "library", "bom-ref": "pkg:pypi/example@1.0", "name": "copy"} assert validate_document(with_components(describe(), library), None) == [ - "bom-ref values are not unique: ['pkg:pypi/example@1.0']" + "bom-ref values repeat: pkg:pypi/example@1.0" ] @@ -168,9 +168,7 @@ def with_components(document: Document, *components: Component) -> bytes: def test_duplicate_refs_in_other_tools_documents() -> None: foreign = json.loads(_FOREIGN) foreign["components"].append({"type": "library", "bom-ref": "pkg:pypi/plain@1.0", "name": "copy"}) - assert validate_document(json.dumps(foreign).encode(), None) == [ - "bom-ref values are not unique: ['pkg:pypi/plain@1.0']" - ] + assert validate_document(json.dumps(foreign).encode(), None) == ["bom-ref values repeat: pkg:pypi/plain@1.0"] def test_component_without_bom_ref(describe: Describe) -> None: @@ -216,9 +214,7 @@ def test_schema_valid_variations_pass(describe: Describe, mutate: Callable[[dict def test_unknown_dependency_refs(describe: Describe) -> None: document = describe() document["dependencies"].append({"ref": "ghost", "dependsOn": ["phantom"]}) - assert validate_document(dumps(document), None) == [ - "dependencies reference bom-refs that do not exist: ['ghost', 'phantom']" - ] + assert validate_document(dumps(document), None) == ["dependencies name bom-refs no component has: ghost, phantom"] def test_root_must_depend_on_components(describe: Describe) -> None: @@ -317,3 +313,13 @@ def test_unexpected_error_is_reported(describe: Describe, mocker: MockerFixture) ) def test_is_document(value: Json, expected: bool) -> None: assert is_document(value) is expected + + +def test_repeated_dependency_entry_cannot_hide_an_invalid_one(describe: Describe) -> None: + # read as a mapping, the valid second entry for the root would replace the invalid first one, which the SPDX + # rendering still turns into an edge + document = describe("Requires-Dist: click\n") + document["dependencies"].insert(0, {"ref": document["metadata"]["component"]["bom-ref"], "dependsOn": []}) + assert validate_document(dumps(document), None) == [ + f"dependencies list more than one entry for: {document['metadata']['component']['bom-ref']}" + ] diff --git a/tests/_validate/test_schemas.py b/tests/_validate/test_schemas.py index 25b9919..02820e7 100644 --- a/tests/_validate/test_schemas.py +++ b/tests/_validate/test_schemas.py @@ -22,7 +22,7 @@ from python_sbom import Component from python_sbom._formats._spdx import Spdx - from python_sbom._validate._document import Json + from python_sbom._validate._limits import Json @pytest.mark.parametrize( diff --git a/tests/_wheel/test_reader.py b/tests/_wheel/test_reader.py new file mode 100644 index 0000000..c8d5bf4 --- /dev/null +++ b/tests/_wheel/test_reader.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import zipfile +from contextlib import ExitStack +from pathlib import Path, PurePosixPath + +import pytest + +from python_sbom._errors import SbomError +from python_sbom._wheel._reader import open_artifact + + +def test_provider_reads_a_member(tmp_path: Path) -> None: + with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: + archive.writestr("embed/table.py", "x = 1\n") + with open_artifact(wheel) as opened: + assert opened.provider_read(PurePosixPath("embed/table.py")) == b"x = 1\n" + + +def test_provider_read_misses(tmp_path: Path) -> None: + with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: + archive.writestr("other.py", "") + with open_artifact(wheel) as opened, pytest.raises(KeyError, match=r"missing\.py"): + opened.provider_read(PurePosixPath("missing.py")) + + +def test_artifact_checked_on_open(tmp_path: Path) -> None: + # the same checks as every other read, so names equal ignoring case are refused before any member is read + with zipfile.ZipFile(wheel := tmp_path / "a.whl", "w") as archive: + archive.writestr("table.py", "") + archive.writestr("TABLE.py", "") + with ( + pytest.raises(SbomError, match=r"holds more than one member named table\.py, ignoring case"), + ExitStack() as stack, + ): + stack.enter_context(open_artifact(wheel)) diff --git a/tests/_wheel/test_record.py b/tests/_wheel/test_record.py index 0e32df9..9fb6fd5 100644 --- a/tests/_wheel/test_record.py +++ b/tests/_wheel/test_record.py @@ -5,33 +5,41 @@ import pytest from python_sbom._errors import SbomError -from python_sbom._wheel._record import decode_digest +from python_sbom._model._inputs import RecordRow +from python_sbom._wheel._record import record_rows @pytest.mark.parametrize( - ("digest", "expected"), + ("digest", "algorithm", "expected"), [ pytest.param( "sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU", - ("SHA-256", hashlib.sha256(b"").hexdigest()), + "SHA-256", + hashlib.sha256(b"").hexdigest(), id="sha256", ), pytest.param( - "md5=1B2M2Y8AsgTpgAmY7PhCfg", ("MD5", hashlib.md5(b"", usedforsecurity=False).hexdigest()), id="md5" + "md5=1B2M2Y8AsgTpgAmY7PhCfg", "MD5", hashlib.md5(b"", usedforsecurity=False).hexdigest(), id="md5" ), ], ) -def test_decode_digest(digest: str, expected: tuple[str, str]) -> None: - assert decode_digest(digest, "w") == expected +def test_record_rows_decode_digests(digest: str, algorithm: str, expected: str) -> None: + assert record_rows([("a.py", digest, "0")], "w") == [ + RecordRow(path="a.py", algorithm=algorithm, digest=expected, size="0") + ] + + +def test_record_rows_skip_unhashed() -> None: + assert record_rows([("w.dist-info/RECORD", "", "")], "w") == [] @pytest.mark.parametrize( ("digest", "message"), [ - pytest.param("blake2b=AAAA", "w: unknown hash algorithm 'blake2b'", id="algorithm"), - pytest.param("sha256=abcde", "w: digest 'abcde' is not urlsafe base64", id="base64"), + pytest.param("blake2b=AAAA", "w: RECORD row for a.py: unknown hash algorithm 'blake2b'", id="algorithm"), + pytest.param("sha256=abcde", "w: RECORD row for a.py: digest 'abcde' is not urlsafe base64", id="base64"), ], ) -def test_decode_digest_rejects(digest: str, message: str) -> None: +def test_record_rows_reject_digests(digest: str, message: str) -> None: with pytest.raises(SbomError, match=message): - decode_digest(digest, "w") + record_rows([("a.py", digest, "0")], "w") diff --git a/tests/_wheel/test_archive.py b/tests/_wheel/test_write.py similarity index 80% rename from tests/_wheel/test_archive.py rename to tests/_wheel/test_write.py index ad55254..1f555f4 100644 --- a/tests/_wheel/test_archive.py +++ b/tests/_wheel/test_write.py @@ -3,22 +3,17 @@ import stat import struct import sys -import tracemalloc import zipfile import zlib -from contextlib import ExitStack -from io import BytesIO from pathlib import Path from typing import TYPE_CHECKING, Final import pytest from python_sbom._errors import SbomError -from python_sbom._wheel._archive import append_member, open_archive, read_member, write_sbom +from python_sbom._wheel._write import append_member, write_sbom if TYPE_CHECKING: - from collections.abc import Iterator - from conftest import WriteWheel from pytest_mock import MockerFixture @@ -192,67 +187,6 @@ def test_embed_refuses_corrupt_archive(tmp_path: Path) -> None: write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) -@pytest.mark.parametrize( - "names", - [ - pytest.param(("a.py", "A.py"), id="case"), - pytest.param(("pkg/a.py", "PKG/a.py"), id="directory-case"), - ], -) -def test_open_archive_refuses_names_equal_ignoring_case(names: tuple[str, str]) -> None: - message = f"w holds more than one member named {names[0].lower()}, ignoring case" - with pytest.raises(SbomError, match=message), ExitStack() as stack: - stack.enter_context(open_archive(zip_of(*((name, b"") for name in names)), "w")) - - -def zip_of(*members: tuple[str, bytes]) -> BytesIO: - buffer = BytesIO() - with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: - for name, content in members: - archive.writestr(name, content) - return buffer - - -def test_open_archive_refuses_corrupt_member() -> None: - data = bytearray(zip_of(("a", b"hello" * 1000)).getvalue()) - data[40:50] = b"\xff" * 10 - with pytest.raises(SbomError, match="w is not a valid zip"), open_archive(BytesIO(bytes(data)), "w") as archive: - archive.read("a") - - -def test_read_member() -> None: - with open_archive(zip_of(("a", b"12345")), "w") as archive: - assert read_member(archive, "a", "w", limit=5) == b"12345" - - -def test_read_member_over_limit() -> None: - with open_archive(zip_of(("a", b"12345")), "w") as archive, pytest.raises(SbomError, match="w: a is over the 4"): - read_member(archive, "a", "w", limit=4) - - -def test_read_member_missing() -> None: - with open_archive(zip_of(("a", b"")), "w") as archive, pytest.raises(SbomError, match="w has no b"): - read_member(archive, "b", "w") - - -@pytest.mark.usefixtures("traced") -def test_read_member_refuses_understated_size() -> None: - # the archive claims 10 bytes for 32 MiB; reading the member whole would decompress all of it before the CRC check - data = bytearray(zip_of(("a", b"A" * 32 * 1024 * 1024)).getvalue()) - data[22:26] = data[(central := data.rfind(b"PK\x01\x02")) + 24 : central + 28] = struct.pack(" Iterator[None]: - tracemalloc.start() - yield - tracemalloc.stop() - - def test_wheel_with_two_dist_info_directories(tmp_path: Path, write_wheel: WriteWheel) -> None: wheel = write_wheel( tmp_path / "native-1.0-py3-none-any.whl", @@ -282,6 +216,46 @@ def test_embed_ends_an_unterminated_last_row(tmp_path: Path, write_wheel: WriteW ] +@pytest.mark.parametrize( + ("record", "expected"), + [ + pytest.param( + b'native-1.0.dist-info/METADATA,,\n"native-1.0.dist-info/sboms/native.cdx.json",sha256=stale,1\n' + b"native-1.0.dist-info/RECORD,,\n", + f"native-1.0.dist-info/METADATA,,\n{_SBOM_ROW}\nnative-1.0.dist-info/RECORD,,\n".encode(), + id="quoted-sbom-row", + ), + pytest.param( + b'native-1.0.dist-info/METADATA,,\n"native-1.0.dist-info/RECORD",,\n', + f'native-1.0.dist-info/METADATA,,\n{_SBOM_ROW}\n"native-1.0.dist-info/RECORD",,\n'.encode(), + id="quoted-record-row", + ), + pytest.param( + b"native-1.0.dist-info/METADATA,,\r\n" + b'"native-1.0.dist-info/sboms/native.cdx.json",sha256=stale,1\r\n' + b"native-1.0.dist-info/RECORD,,\r\n", + f"native-1.0.dist-info/METADATA,,\r\n{_SBOM_ROW}\r\nnative-1.0.dist-info/RECORD,,\r\n".encode(), + id="crlf", + ), + pytest.param( + b"native-1.0.dist-info/METADATA,,\nnative-1.0.dist-info/RECORD,,", + f"native-1.0.dist-info/METADATA,,\n{_SBOM_ROW}\nnative-1.0.dist-info/RECORD,,\n".encode(), + id="no-final-newline", + ), + ], +) +def test_embed_rewrites_record_by_parsed_path( + tmp_path: Path, write_wheel: WriteWheel, record: bytes, expected: bytes +) -> None: + wheel = write_wheel( + tmp_path / "native-1.0-py3-none-any.whl", + {"native-1.0.dist-info/METADATA": b"Name: native\n", "native-1.0.dist-info/RECORD": record}, + record=False, + ) + write_sbom(wheel, "native.cdx.json", b"{}\n", 1_700_000_000) + assert zipfile.ZipFile(wheel).read("native-1.0.dist-info/RECORD") == expected + + def test_append_member_deflates_at_level_nine(tmp_path: Path) -> None: # the archive stores its members, as virtualenv's zipapp does, yet the SBOM goes in deflated, and the members keep # their bytes diff --git a/tests/_wheel/test_zip.py b/tests/_wheel/test_zip.py new file mode 100644 index 0000000..297dcc2 --- /dev/null +++ b/tests/_wheel/test_zip.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import io +import struct +import tracemalloc +import zipfile +from contextlib import ExitStack +from io import BytesIO +from typing import TYPE_CHECKING + +import pytest + +from python_sbom._errors import SbomError +from python_sbom._wheel._zip import open_archive, read_member, sha256_hex + +if TYPE_CHECKING: + from collections.abc import Iterator + + +@pytest.mark.parametrize( + "names", + [ + pytest.param(("a.py", "A.py"), id="case"), + pytest.param(("pkg/a.py", "PKG/a.py"), id="directory-case"), + ], +) +def test_open_archive_refuses_names_equal_ignoring_case(names: tuple[str, str]) -> None: + message = f"w holds more than one member named {names[0].lower()}, ignoring case" + with pytest.raises(SbomError, match=message), ExitStack() as stack: + stack.enter_context(open_archive(zip_of(*((name, b"") for name in names)), "w")) + + +def zip_of(*members: tuple[str, bytes]) -> BytesIO: + buffer = BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in members: + archive.writestr(name, content) + return buffer + + +def test_open_archive_refuses_corrupt_member() -> None: + data = bytearray(zip_of(("a", b"hello" * 1000)).getvalue()) + data[40:50] = b"\xff" * 10 + with pytest.raises(SbomError, match="w is not a valid zip"), open_archive(BytesIO(bytes(data)), "w") as archive: + archive.read("a") + + +def test_read_member() -> None: + with open_archive(zip_of(("a", b"12345")), "w") as archive: + assert read_member(archive, "a", "w", limit=5) == b"12345" + + +def test_read_member_over_limit() -> None: + with open_archive(zip_of(("a", b"12345")), "w") as archive, pytest.raises(SbomError, match="w: a is over the 4"): + read_member(archive, "a", "w", limit=4) + + +def test_read_member_missing() -> None: + with open_archive(zip_of(("a", b"")), "w") as archive, pytest.raises(SbomError, match="w has no b"): + read_member(archive, "b", "w") + + +@pytest.mark.usefixtures("traced") +def test_read_member_refuses_understated_size() -> None: + # the archive claims 10 bytes for 32 MiB; reading the member whole would decompress all of it before the CRC check + data = bytearray(zip_of(("a", b"A" * 32 * 1024 * 1024)).getvalue()) + data[22:26] = data[(central := data.rfind(b"PK\x01\x02")) + 24 : central + 28] = struct.pack(" Iterator[None]: + tracemalloc.start() + yield + tracemalloc.stop() + + +def test_sha256_hex() -> None: + assert sha256_hex(io.BytesIO(b"abc")) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" diff --git a/tests/conftest.py b/tests/conftest.py index 7f14b7c..47c74b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,20 +5,22 @@ import shutil import subprocess import zipfile -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Protocol import pytest from python_sbom import ContentProvider, ProviderContext -from python_sbom._collect._archive import member_reader -from python_sbom._generate._embed import build_sbom +from python_sbom._model._bom import assemble +from python_sbom._model._document import checked_contributions from python_sbom._model._identity import project_identity -from python_sbom._model._inputs import BuildFacts, Interpreter, SbomRequest, Settings, SourceTree +from python_sbom._model._inputs import BuildFacts, BuildScript, BundledWheel, Interpreter, Settings, WheelTree from python_sbom._model._metadata import parse_metadata +from python_sbom._model._wheel import wheel_artifact +from python_sbom._wheel._reader import open_artifact if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Callable, Mapping from python_sbom._model._types import Document @@ -44,13 +46,15 @@ def __call__(self, root: Path, artifact: Path) -> ProviderContext: ... class Describe(Protocol): - def __call__( + def __call__( # ruff: ignore[too-many-arguments] # each a part of the wheel a test varies self, metadata: str = "", *, settings: Settings = ..., facts: BuildFacts = ..., - source: SourceTree | None = None, + copyright_line: str | None = None, + bundled: list[BundledWheel] | None = None, + build_scripts: list[BuildScript] | None = None, content: ContentProvider | None = None, ) -> Document: ... @@ -140,34 +144,36 @@ def settings() -> Settings: @pytest.fixture def describe(settings: Settings, facts: BuildFacts) -> Describe: - def describe( + # the wheel pipeline from the read wheel on, as embed_into runs it, without a wheel on disk + def describe( # ruff: ignore[too-many-arguments] # each a part of the wheel a test varies metadata: str = "", *, settings: Settings = settings, facts: BuildFacts = facts, - source: SourceTree | None = None, + copyright_line: str | None = None, + bundled: list[BundledWheel] | None = None, + build_scripts: list[BuildScript] | None = None, content: ContentProvider | None = None, ) -> Document: - request = SbomRequest( - root=Path(), + tree = WheelTree( + name="example-1.0-py3-none-any.whl", metadata=parse_metadata(f"Name: Example\nVersion: 1.0\n{metadata}"), - wheel_name="example-1.0-py3-none-any.whl", - settings=settings, - build_scripts=(), + copyright=copyright_line, + bundled=bundled or [], + build_scripts=build_scripts or [], ) - context = ProviderContext( - root=Path(), + project = project_identity(tree.metadata, settings) + content = content or ContentProvider() + context = ProviderContext(root=Path(), settings=settings, identity=project, read=artifact_read(Path(tree.name))) + built = wheel_artifact( + tree, + project=project, settings=settings, - identity=project_identity(request.metadata, settings), - read=member_reader(Path("example-1.0-py3-none-any.whl")), + bundled_info={ + wheel.path: content.bundled_wheel(context, PurePosixPath(wheel.path)) for wheel in tree.bundled + }, ) - return build_sbom( - request, - source or SourceTree(copyright=None, bundled=[], build_scripts=[]), - facts, - content or ContentProvider(), - context, - ).document + return assemble(built, project, settings, facts, checked_contributions(content.contributions(context), content)) return describe @@ -176,6 +182,15 @@ def describe( def context_for(settings: Settings) -> ContextFor: def context(root: Path, artifact: Path) -> ProviderContext: identity = project_identity(parse_metadata("Name: Example\nVersion: 1.0\n"), settings) - return ProviderContext(root=root, settings=settings, identity=identity, read=member_reader(artifact)) + return ProviderContext(root=root, settings=settings, identity=identity, read=artifact_read(artifact)) return context + + +def artifact_read(artifact: Path) -> Callable[[PurePosixPath], bytes]: + # opens the artifact for each read, as a test context outlives no build that would hold it open + def read(path: PurePosixPath) -> bytes: + with open_artifact(artifact) as opened: + return opened.provider_read(path) + + return read From 0d988572670f286a128070e6a6a8c624729b8b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 7 Nov 2026 18:23:35 -0800 Subject: [PATCH 115/115] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20ci:=20pin=20github?= =?UTF-8?q?-workflows=206.7.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/check.yaml | 4 ++-- .github/workflows/codeql.yaml | 2 +- .github/workflows/news.yaml | 4 ++-- .github/workflows/release.yaml | 6 +++--- .github/workflows/scorecard.yaml | 2 +- .github/workflows/upgrade-release-tools.yaml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index e0d659e..3506978 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -16,7 +16,7 @@ jobs: test: name: ๐Ÿงช test if: github.event_name != 'schedule' || github.repository == 'tox-dev/python-sbom' - uses: tox-dev/github-workflows/.github/workflows/check.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/.github/workflows/check.yaml@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 permissions: contents: read with: @@ -57,6 +57,6 @@ jobs: permissions: {} steps: - name: ๐Ÿšฆ Require all needed jobs to pass - uses: tox-dev/github-workflows/required-checks@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/required-checks@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 with: needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index cec793a..7ac1c0c 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -13,7 +13,7 @@ permissions: {} jobs: analyze: name: ๐Ÿ›ก๏ธ Analyze - uses: tox-dev/github-workflows/.github/workflows/codeql.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/.github/workflows/codeql.yaml@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 permissions: contents: read security-events: write diff --git a/.github/workflows/news.yaml b/.github/workflows/news.yaml index d5a8c28..f194282 100644 --- a/.github/workflows/news.yaml +++ b/.github/workflows/news.yaml @@ -22,9 +22,9 @@ jobs: python-version: "3.15" enable-cache: false - name: ๐Ÿ“ฆ Install locked release tools - uses: tox-dev/github-workflows/install-locked-tools@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/install-locked-tools@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 with: groups: release-news venv: ${{ runner.temp }}/release-tools - name: ๐Ÿ” Validate changelog fragments - uses: tox-dev/github-workflows/validate-news@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/validate-news@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c11f6c9..ab0d396 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,12 +10,12 @@ concurrency: jobs: build: name: ๐Ÿ—๏ธ Build - uses: tox-dev/github-workflows/.github/workflows/validate-python-release.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/.github/workflows/validate-python-release.yaml@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 permissions: contents: read notes: name: ๐Ÿ“ Render GitHub Release notes - uses: tox-dev/github-workflows/.github/workflows/release-notes.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/.github/workflows/release-notes.yaml@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 permissions: contents: read with: @@ -45,7 +45,7 @@ jobs: release: name: ๐Ÿš€ Publish GitHub Release needs: [build, publish, notes] - uses: tox-dev/github-workflows/.github/workflows/github-release.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/.github/workflows/github-release.yaml@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 permissions: contents: write with: diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index eafe9e6..aedc96d 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -23,4 +23,4 @@ jobs: pull-requests: read # read reviews for the Code-Review check security-events: write # upload the SARIF results to code scanning statuses: read # read commit statuses for the CI-Tests check - uses: tox-dev/github-workflows/.github/workflows/scorecard.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/.github/workflows/scorecard.yaml@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 diff --git a/.github/workflows/upgrade-release-tools.yaml b/.github/workflows/upgrade-release-tools.yaml index a4aabe6..5e650e0 100644 --- a/.github/workflows/upgrade-release-tools.yaml +++ b/.github/workflows/upgrade-release-tools.yaml @@ -10,7 +10,7 @@ concurrency: jobs: upgrade: name: ๐Ÿ”€ Upgrade release tools - uses: tox-dev/github-workflows/.github/workflows/upgrade-release-tools.yaml@e1361e2efa1e9dd9705bb9be719d923cd9154707 # 6.7.1 + uses: tox-dev/github-workflows/.github/workflows/upgrade-release-tools.yaml@0c4cffd63033d38196319524d101caae1eef925b # 6.7.2 permissions: contents: read secrets: