diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..9191891 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +# actionlint 1.7.12 doesn't know concurrency.queue, which GitHub documents and this workflow +# relies on. Drop this once it does: https://github.com/rhysd/actionlint/issues/657 +paths: + .github/workflows/release-tags.yml: + ignore: + - 'unexpected key "queue" for "concurrency" section' diff --git a/.github/actions/wporg-svn/README.md b/.github/actions/wporg-svn/README.md new file mode 100644 index 0000000..c40c24a --- /dev/null +++ b/.github/actions/wporg-svn/README.md @@ -0,0 +1,234 @@ +# WordPress.org SVN release actions + +Three composite actions that release a plugin to plugins.svn.wordpress.org from a built ZIP. They split a release into a prepared tag, a separate switch of the stable pointer, and trunk housekeeping afterwards: + +| Action | Inputs | What it does | +|---|---|---| +| `tag` | `plugin-slug`, `zip-url` | Creates `tags/` in one commit, as a copy of the current stable tag that matches the ZIP exactly. Only changed files are uploaded. trunk and the stable pointer don't move. | +| `set-stable` | `plugin-slug`, `version` | Copies `tags//readme.txt` over `trunk/readme.txt` in one server-side commit. WordPress.org reads the Stable Tag from that file, so this is the release. | +| `update-trunk` | `plugin-slug`, `version` | Replaces trunk with `tags/` in one atomic server-side commit. Housekeeping after the release. | + +All three also take `wporg-username` and `wporg-password`. The version always comes from the plugin's PHP header (`Version:`), never from the ZIP's file name. + +## Read this first + +- **This is not WordPress.org's documented flow.** [Using Subversion](https://developer.wordpress.org/plugins/wordpress-org/how-to-use-subversion/) updates trunk first and copies trunk to a tag. These actions prepare the tag first, on purpose, so that it can be checked before anything points at it. +- **A new tag is public the moment it exists.** It is not private staging. If [Release Confirmation](https://developer.wordpress.org/plugins/wordpress-org/release-confirmation-emails/) is on, WordPress.org may see the new tag and email about a pending release before `set-stable` runs. Never confirm a tag that `tag` did not verify. +- **`tag` takes one commit.** It prepares the new tag in a local working copy and commits it once, so `tags/` appears complete or not at all. If the outcome of that commit is unclear, the action says so loudly and never cleans up or reuses the tag. See [Recovery](#recovery). +- **Released tags never change.** `tag` refuses a version whose tag already exists, under any spelling (`1.2` and `1.2.0` are the same version). + +## Order + +```text +tag -> QA the new tag, approve -> set-stable -> update-trunk +``` + +- QA `tags/` at the `revision` the tag job reports, not just the ZIP you started from. Pass that revision as `expected-revision`, as the example does, and both later actions refuse a tag that changed after QA. +- `update-trunk` also replaces `trunk/readme.txt`, so it refuses to run until `set-stable` has pointed trunk's Stable Tag at the version. It can't release anything by itself. +- `set-stable` has no upgrade-only rule. Pointing it at an older tag rolls the release back. + +## Example + +A plugin repository's release workflow. The `concurrency` lock and the protected `environment` are the caller's job: a composite action can't enforce either. Create the `wordpress-org` environment in the repository settings, with required reviewers, before the first run: a workflow that names an environment that doesn't exist gets one created with no protection rules, and the release job then runs without anyone approving it. Required reviewers are a [plan-gated feature](https://docs.github.com/en/actions/reference/workflows-and-actions/deployments-and-environments): on GitHub Free, Pro and Team they are only available in public repositories, so a private plugin repository needs GitHub Enterprise for this gate. + +```yaml +# A plugin repository's release workflow. Replace my-plugin, and replace +# with a full commit SHA of stellarwp/plugin-toolbox. +name: Release to WordPress.org + +on: + workflow_dispatch: + inputs: + plugin-slug: + description: The plugin's wordpress.org slug + required: true + zip-url: + description: HTTPS URL of the built plugin ZIP (prefer a short-lived signed URL) + required: true + +# One release at a time per plugin, held for the whole sequence. This lock only +# covers workflows in this repository: release each plugin from one place. +concurrency: + group: wporg-${{ inputs.plugin-slug }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + tag: + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.tag.outputs.version }} + revision: ${{ steps.tag.outputs.revision }} + steps: + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - id: tag + uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/tag@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + zip-url: ${{ inputs.zip-url }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} + + release: + needs: tag + runs-on: ubuntu-24.04 + # A protected environment: its reviewers QA tags/ at the tag job's + # revision output before approving this job. + environment: wordpress-org + steps: + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/set-stable@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + version: ${{ needs.tag.outputs.version }} + expected-revision: ${{ needs.tag.outputs.revision }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} + - uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/update-trunk@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + version: ${{ needs.tag.outputs.version }} + expected-revision: ${{ needs.tag.outputs.revision }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} +``` + +[GitHub concurrency](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency) only covers one repository, isn't first-in-first-out, and is no lock on SVN. Release each plugin from one workflow in one repository. Don't add job-level locks with the same group while the workflow holds it. + +## Inputs and outputs + +| Input | Used by | Value | +|---|---|---| +| `plugin-slug` | all | The wordpress.org slug: lowercase letters and digits, single hyphens between them. | +| `zip-url` | `tag` | HTTPS URL of the ZIP. Pass a private URL from a secret, or mask it in an earlier step: see [Security](#security). | +| `version` | `set-stable`, `update-trunk` | The `version` output of `tag`. | +| `expected-revision` | `set-stable`, `update-trunk` | Optional: the `revision` output of `tag`. When set, the action refuses unless `tags/` last changed at exactly that revision, so what gets released is the tag that was QA'd. Leave it empty for a manual run or a rollback. | +| `wporg-username` | all | `${{ secrets.WPORG_USERNAME }}`: an account with commit access to the plugin. | +| `wporg-password` | all | `${{ secrets.WPORG_PASSWORD }}`: that account's [SVN password](https://make.wordpress.org/plugins/2024/09/04/upcoming-security-changes-for-plugin-and-theme-authors-on-wordpress-org/), not its login password. | + +Every action outputs `version` and `revision` once it has verified its result. A failed action outputs nothing. + +| Output | Meaning | +|---|---| +| `version` | The version released (`tag`: read from the plugin header). | +| `revision` | `tag`: the commit that created the new tag. `set-stable`, `update-trunk`: their commit. For a verified no-op, the revision that was checked; the step summary says it was a no-op. | +| `previous-stable` | `tag` only: the tag it copied from. | + +## Requirements + +An Ubuntu runner with bash, Subversion 1.10 or newer (`svn`, and `svnmucc` for `set-stable` and `update-trunk`), Python 3.8 or newer, and, for `tag`, curl. Ubuntu runner images may not include Subversion: install the `subversion` package as in the example. Each action checks its tools and stops with a clear error if one is missing or the runner isn't Linux. + +## What gets checked + +Before any write, every action pins one repository revision and reads everything at that revision. A failed read always stops the action: an authentication, TLS, connection or permission error is never taken to mean a path is absent. + +`tag`: + +- trunk's readme names a numeric stable tag S, and `tags/S` exists. A stable tag of `trunk` (a first release) is not supported. +- The ZIP passes every check in [ZIP limits](#zip-limits). The plugin header version V is numeric and newer than S, and the ZIP's readme Stable Tag is exactly V. +- `tags/V` doesn't exist under any spelling, and the copy of `tags/S` sets no property that changes file bytes (`svn:eol-style`, `svn:keywords`, `svn:special`, `svn:externals`). +- It copies `tags/S` at the pinned revision into an otherwise empty working copy as `tags/V`, syncs the ZIP byte for byte (additions, deletions, dotfiles, file/directory swaps), adds files even where ignore patterns would hide them, and re-validates the version, readme and full tree. An artifact identical to `tags/S` is refused. +- Just before the commit it checks again that the stable pointer, `tags/S` and the absence of `tags/V` haven't changed. If someone creates `tags/V` before the commit lands, the commit fails instead of overwriting it. +- After the commit it exports the committed tag and compares it with the ZIP again. + +`set-stable`: `tags/V/readme.txt` and `trunk/readme.txt` are regular files without byte-changing properties, and the tag readme's Stable Tag is exactly V. The write is pinned to the snapshot revision, so a competing change to `trunk/readme.txt` makes it fail instead of being overwritten. Identical bytes are a verified no-op. + +`update-trunk`: `trunk` and `tags/V` exist, and trunk's readme already names V as its Stable Tag. The remove and copy are one commit, copying `tags/V` as it was at the snapshot revision, and a change anywhere inside trunk since then makes the whole commit fail. The result is compared on the server, files and properties both. A trunk that already matches is a verified no-op. + +## ZIP limits + +| Limit | Value | +|---|---| +| Download | HTTPS only, redirects too (at most 5); 30 s to connect, 10 min per attempt, 3 retries | +| ZIP size | 256 MiB | +| Files and directories | 50,000, counting parent directories the ZIP doesn't list | +| Expanded size | 1 GiB | +| Expansion ratio | 100 to 1 | +| Path depth | 64 levels | +| Path length | 1,024 bytes | + +Every entry is checked before anything is extracted. Rejected: + +- absolute paths, `..`, backslashes, empty or `.` path segments, and names that only differ by case or Unicode form; +- control characters; +- names that aren't valid UTF-8. Names without the ZIP's UTF-8 flag are read as UTF-8, which is what Info-ZIP's `zip` writes on macOS, and on Linux depending on its locales; +- entries with two different names, where an Info-ZIP Unicode Path field disagrees with the header; +- symlinks and other special files; +- encrypted entries; +- `.svn` or `.git` anywhere. + +The ZIP is either the plugin itself or one wrapper directory of any name holding it. `__MACOSX/` and `.DS_Store` are skipped as packaging noise. The plugin root must hold `readme.txt` and exactly one root-level PHP file with a `Plugin Name` header. That file must have exactly one `Version` header in its first 8 KiB, the same window WordPress reads. + +New files are added without SVN auto-props, including any a repository inherits. Like any `svn add`, binary files still get `svn:mime-type: application/octet-stream`. Nothing else is set, `svn:executable` included. Files carried over from the previous tag keep their properties. Before committing, `tag` checks that nothing in the new tag sets a property that changes file bytes. + +## Limitations + +- **Not supported:** first releases, prerelease versions (`1.2-beta`), building the ZIP, confirming a release on WordPress.org, and Git tags or releases. +- **Races are narrowed, not prevented.** The pinned base revision protects only the paths each action writes. The rechecks shrink the windows. Neither replaces running one release at a time. +- **Success means the commit landed and was verified.** WordPress.org processing, ZIP building and Release Confirmation happen afterwards and aren't checked. +- **Some failure paths are untested against wordpress.org.** The actions have shipped real releases of The Events Calendar and Event Tickets. The automated tests write only to throwaway local repositories, so two paths have never run against the real server: out-of-date detection over HTTPS and a lost response to a commit that landed. Supervise your plugin's first release with this page at hand. + +## Recovery + +A failure prints an error naming the stage and the revisions known so far, and repeats it in the step summary. Don't rerun a write blindly: each action has already inspected the server after a failed write and says what it found. + +**Nothing was written.** Every check runs before the only write, so any failure before the commit, and `the commit failed ... tags/V does not exist`, mean nothing changed. Fix the cause and rerun. + +**`tag` can't vouch for `tags/V`.** One of these: + +- `... cannot confirm that it created it`: the commit reported an error, but `tags/V` now exists. Either the response got lost after the commit landed, or someone else created the tag. +- `The commit's outcome is unknown`: the job stopped during the commit, usually because it was cancelled. If `tags/V` doesn't exist, nothing was written; rerun. +- `tags/V was created at rN but not verified`: the commit landed, but comparing the committed tag with the ZIP failed or didn't finish. + +Then: + +1. Don't confirm it in Release Confirmation, and don't run `set-stable` for it. +2. Look at what happened (reads only): + + ```sh + svn log -v -l 5 https://plugins.svn.wordpress.org//tags/ + svn diff --summarize https://plugins.svn.wordpress.org//tags/ https://plugins.svn.wordpress.org//tags/ + ``` + +3. A maintainer decides what to do: release it once someone has compared it with the ZIP, or delete it and run `tag` again. Deleting is a write to the live plugin, so it needs that maintainer's explicit go-ahead: + + ```sh + svn rm -m "Remove unverified tag " https://plugins.svn.wordpress.org//tags/ + ``` + + If WordPress.org already lists the tag as a pending release, follow its Release Confirmation guidance for a release you won't ship. + +**`set-stable` or `update-trunk` failed.** + +- `... does not match ...; check who else is committing`: the write failed, and the target doesn't hold the release. Usually another commit got there first or the server refused the write; `svn log` shows which. Find out who else is releasing before you rerun. +- `... now matches ... Rerunning is safe`: the write most likely landed but its response was lost. A rerun verifies it and reports a no-op. + +## Security + +- The password reaches `svn` and `svnmucc` on stdin only. It isn't in any process's arguments or in its child processes' environment. Each action masks it, along with the ZIP URL. +- The actions never print the ZIP URL. They mask it too, but only from their own steps on: GitHub logs a step's `with:` inputs before the action runs. If the URL must stay private, pass it from a secret, or mask it with `::add-mask::` in an earlier step. Nothing sends WordPress.org credentials to the ZIP host, and curl ignores `.curlrc`. +- SVN runs non-interactively with a private, throwaway config directory and no credential cache. Certificate checks are never skipped. Temporary files are removed on success, failure and cancellation. +- Nothing from the ZIP runs: no PHP, scripts or hooks. +- The credentials are repository secrets, because the `tag` job writes to SVN without an approval gate. The protected environment in the example gates the release job's approval; it doesn't protect the secrets. Anyone who can push a workflow to the repository can read repository secrets, so limit who can push, and never let the secrets reach workflows that run pull request code from forks. Prefer short-lived signed ZIP URLs: `workflow_dispatch` inputs are visible to anyone who can see the run. + +## Tests + +Locally, with Subversion, Python 3 and openssl installed: + +```sh +python3 -m unittest discover -s tests/wporg-svn -v +shellcheck -x .github/actions/wporg-svn/*/*.sh .github/actions/wporg-svn/lib/svn.sh tests/wporg-svn/guard/bin/* +actionlint && actionlint .github/actions/wporg-svn/examples/release.yml +``` + +The tests run the real scripts against throwaway `file://` repositories and a local HTTPS server. A test-only guard (`tests/wporg-svn/guard`) sits first on `PATH` and refuses any SVN write whose target isn't `file://`. + +Read-only checks against real plugins are opt-in. They read from wordpress.org, and every write goes to a local repository: + +```sh +WPORG_LIVE=1 python3 -m unittest discover -s tests/wporg-svn -p test_live.py -v +``` diff --git a/.github/actions/wporg-svn/examples/release.yml b/.github/actions/wporg-svn/examples/release.yml new file mode 100644 index 0000000..76c7903 --- /dev/null +++ b/.github/actions/wporg-svn/examples/release.yml @@ -0,0 +1,63 @@ +# A plugin repository's release workflow. Replace my-plugin, and replace +# with a full commit SHA of stellarwp/plugin-toolbox. +name: Release to WordPress.org + +on: + workflow_dispatch: + inputs: + plugin-slug: + description: The plugin's wordpress.org slug + required: true + zip-url: + description: HTTPS URL of the built plugin ZIP (prefer a short-lived signed URL) + required: true + +# One release at a time per plugin, held for the whole sequence. This lock only +# covers workflows in this repository: release each plugin from one place. +concurrency: + group: wporg-${{ inputs.plugin-slug }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + tag: + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.tag.outputs.version }} + revision: ${{ steps.tag.outputs.revision }} + steps: + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - id: tag + uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/tag@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + zip-url: ${{ inputs.zip-url }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} + + release: + needs: tag + runs-on: ubuntu-24.04 + # A protected environment: its reviewers QA tags/ at the tag job's + # revision output before approving this job. + environment: wordpress-org + steps: + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/set-stable@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + version: ${{ needs.tag.outputs.version }} + expected-revision: ${{ needs.tag.outputs.revision }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} + - uses: stellarwp/plugin-toolbox/.github/actions/wporg-svn/update-trunk@ + with: + plugin-slug: ${{ inputs.plugin-slug }} + version: ${{ needs.tag.outputs.version }} + expected-revision: ${{ needs.tag.outputs.revision }} + wporg-username: ${{ secrets.WPORG_USERNAME }} + wporg-password: ${{ secrets.WPORG_PASSWORD }} diff --git a/.github/actions/wporg-svn/lib/archive.py b/.github/actions/wporg-svn/lib/archive.py new file mode 100644 index 0000000..82b0cd3 --- /dev/null +++ b/.github/actions/wporg-svn/lib/archive.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Inspect a plugin ZIP before extracting anything, then extract it safely. + +Usage: + archive.py extract ZIP DEST extract into the empty directory DEST and print + the plugin root (DEST or its one wrapper directory) + +Every entry is checked before the first byte is written. Errors go to stderr +without echoing entry names, and exit 1. +""" + +from __future__ import annotations + +import os +import re +import stat +import struct +import sys +import unicodedata +import zipfile +import zlib + +MAX_ARCHIVE_BYTES = 256 * 1024 * 1024 +MAX_ENTRIES = 50_000 +MAX_EXPANDED_BYTES = 1024 * 1024 * 1024 +MAX_RATIO = 100 +MAX_DEPTH = 64 +MAX_PATH_BYTES = 1024 + +# Packaging noise skipped (never extracted): macOS resource forks and Finder files. +NOISE_TOP = "__MACOSX" +NOISE_NAME = ".DS_Store" +VCS = {".svn", ".git"} + + +class ArchiveError(Exception): + pass + + +def _check_control(name: str) -> None: + if any(ord(c) < 0x20 or 0x7F <= ord(c) <= 0x9F for c in name): + raise ArchiveError("an entry name contains a control character") + + +def _unicode_paths(info: zipfile.ZipInfo, header: bytes): + """Yield the name in each of the entry's Info-ZIP Unicode Path fields (0x7075). + + Only version 1 fields whose CRC matches the header name count; any other is + stale, as unzip and zipfile treat it. + """ + extra, header_crc = info.extra, zlib.crc32(header) + while len(extra) >= 4: + kind, size = struct.unpack(" str: + """The entry's name as its author wrote it. + + zipfile reads a name without the UTF-8 flag as CP437, but Info-ZIP's zip + writes UTF-8 there without setting the flag: on macOS, and on Ubuntu without + en_US.UTF-8. Such names are taken as UTF-8; one that isn't valid UTF-8 is + refused rather than guessed. + """ + raw = info.orig_filename + _check_control(raw) + flagged = info.flag_bits & 0x800 + name = raw + if not flagged and not raw.isascii(): + try: + name = raw.encode("cp437").decode("utf-8") + except UnicodeError: + raise ArchiveError("an entry name is not UTF-8") from None + _check_control(name) + # unzip and zipfile 3.12+ take the name from the last valid Unicode Path + # field; older zipfile ignores them. Every one must agree, on every Python. + if any(other != name for other in _unicode_paths(info, raw.encode("utf-8" if flagged else "cp437"))): + raise ArchiveError("an entry has two different names") + # Otherwise filename differs from both only where zipfile changed the name. + if info.filename not in (raw, name): + raise ArchiveError("an entry has an unsafe path") + return name + + +def _check_name(info: zipfile.ZipInfo) -> list: + """Return the path components of a safe entry name, or raise.""" + name = _name(info) + if "\\" in name or name.startswith("/") or re.match(r"[A-Za-z]:", name): + raise ArchiveError("an entry has an unsafe path") + if len(name.encode()) > MAX_PATH_BYTES: + raise ArchiveError(f"an entry path is longer than {MAX_PATH_BYTES} bytes") + parts = (name[:-1] if name.endswith("/") else name).split("/") + if len(parts) > MAX_DEPTH: + raise ArchiveError(f"an entry path is deeper than {MAX_DEPTH} levels") + if any(part in ("", ".", "..") for part in parts): + raise ArchiveError("an entry has an unsafe path") + return parts + + +def _check_type(info: zipfile.ZipInfo) -> None: + if info.flag_bits & 0x1: + raise ArchiveError("the archive has encrypted entries") + kind = stat.S_IFMT(info.external_attr >> 16) + # Hosts that store Unix modes: 3 is Unix, 19 is OS X. + if info.create_system in (3, 19) and kind not in (0, stat.S_IFREG, stat.S_IFDIR): + raise ArchiveError("an entry is not a regular file or directory") + if kind == stat.S_IFDIR and not info.is_dir(): + raise ArchiveError("an entry is not a regular file or directory") + + +def _fold(parts) -> tuple: + return tuple(unicodedata.normalize("NFC", part).casefold() for part in parts) + + +def inspect(zf: zipfile.ZipFile, archive_bytes: int) -> list: + """Validate every entry; return (info, parts) for the entries to extract.""" + infos = zf.infolist() + if len(infos) > MAX_ENTRIES: + raise ArchiveError(f"the archive has more than {MAX_ENTRIES} entries") + + kept, raw_names, kinds, spellings, total = [], set(), {}, {}, 0 + for info in infos: + parts = _check_name(info) + _check_type(info) + if info.orig_filename in raw_names: + raise ArchiveError("the archive has a duplicate or ambiguous path") + raw_names.add(info.orig_filename) + if parts[0] == NOISE_TOP or parts[-1] == NOISE_NAME: + continue + if any(part.lower() in VCS for part in parts): + raise ArchiveError("the archive contains version control metadata") + + # Every prefix is a directory; the last component is the entry itself. + # Each distinct one becomes a file or directory on disk, listed or not. + folded = _fold(parts) + for depth in range(1, len(parts) + 1): + kind = "file" if depth == len(parts) and not info.is_dir() else "dir" + key, spelling = folded[:depth], "/".join(parts[:depth]) + if key not in kinds: + kinds[key], spellings[key] = kind, spelling + if len(kinds) > MAX_ENTRIES: + raise ArchiveError(f"the archive has more than {MAX_ENTRIES} files and directories") + elif spellings[key] != spelling: + raise ArchiveError("the archive has a duplicate or ambiguous path") + elif kind == "file" or kinds[key] == "file": + raise ArchiveError("the archive has a file/directory conflict") + + total += info.file_size + kept.append((info, parts)) + + if total > MAX_EXPANDED_BYTES: + raise ArchiveError(f"the archive expands past {MAX_EXPANDED_BYTES} bytes") + if total > MAX_RATIO * max(archive_bytes, 1): + raise ArchiveError(f"the archive's expansion ratio exceeds {MAX_RATIO}") + return kept + + +def plugin_root(kept: list) -> tuple: + """Path components of the plugin root: () when flat, (wrapper,) when wrapped.""" + top = {parts[0] for _, parts in kept} + top_files = {parts[0] for info, parts in kept if len(parts) == 1 and not info.is_dir()} + if "readme.txt" in top_files: + return () + if len(top) == 1 and not top_files: + wrapper = next(iter(top)) + if any(parts == [wrapper, "readme.txt"] and not info.is_dir() for info, parts in kept): + return (wrapper,) + raise ArchiveError("the plugin root has no readme.txt") + raise ArchiveError( + "cannot find one plugin root: readme.txt must be at the top level or inside one wrapper directory") + + +def extract(zip_path: str, dest: str) -> str: + """Extract zip_path into the empty directory dest; return the plugin root.""" + size = os.path.getsize(zip_path) + if size > MAX_ARCHIVE_BYTES: + raise ArchiveError(f"the archive is larger than {MAX_ARCHIVE_BYTES} bytes") + dest = os.path.realpath(dest) + try: + with zipfile.ZipFile(zip_path) as zf: + kept = inspect(zf, size) + root = plugin_root(kept) + written = 0 + for info, parts in kept: + target = os.path.join(dest, *parts) + if not os.path.realpath(target).startswith(dest + os.sep): + raise ArchiveError("an entry has an unsafe path") + if info.is_dir(): + os.makedirs(target, exist_ok=True) + continue + os.makedirs(os.path.dirname(target), exist_ok=True) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + with zf.open(info) as source, os.fdopen(os.open(target, flags, 0o644), "wb") as out: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + written += len(chunk) + if written > MAX_EXPANDED_BYTES: + raise ArchiveError(f"the archive expands past {MAX_EXPANDED_BYTES} bytes") + out.write(chunk) + except (zipfile.BadZipFile, zipfile.LargeZipFile, NotImplementedError, RuntimeError, EOFError, + zlib.error, ValueError) as error: + raise ArchiveError("the archive is malformed") from error + return os.path.join(dest, *root) + + +def main(argv) -> int: + if len(argv) != 3 or argv[0] != "extract": + print(__doc__, file=sys.stderr) + return 2 + try: + print(extract(argv[1], argv[2])) + except (ArchiveError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/actions/wporg-svn/lib/release.py b/.github/actions/wporg-svn/lib/release.py new file mode 100644 index 0000000..35e5810 --- /dev/null +++ b/.github/actions/wporg-svn/lib/release.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Release metadata: numeric versions, the readme Stable Tag, the plugin header. + +Usage: + release.py version V exit 0 when V is a numeric version + release.py compare A B print lt, eq or gt + release.py equivalents V print the stdin lines numerically equal to V + release.py stable-tag README print the readme's Stable Tag value + release.py plugin-version DIR print the Version header of DIR's plugin file + +Errors go to stderr without echoing file contents, and exit 1. +""" + +from __future__ import annotations + +import collections +import os +import re +import sys + +VERSION = re.compile(r"[0-9]+(?:\.[0-9]+)*") + +# WordPress reads plugin headers from the first 8 KiB only (get_file_data). +HEADER_BYTES = 8192 + +# Header names the wordpress.org readme parser recognises (class-parser.php). +README_HEADERS = { + "tested", "tested up to", "requires", "requires at least", "requires php", "tags", + "contributors", "donate link", "stable tag", "license", "license uri", +} + +# PCRE's \R, which the readme parser splits lines on. +NEWLINES_UTF8 = re.compile("\r\n|[\n\x0b\x0c\r\x85

]") +NEWLINES_BYTES = re.compile("\r\n|[\n\x0b\x0c\r\x85]") + + +class ReleaseError(Exception): + pass + + +def is_version(value: str) -> bool: + return VERSION.fullmatch(value) is not None + + +def _key(value: str) -> list: + if not is_version(value): + raise ReleaseError("not a numeric version") + # Compare digit strings by (length, text) after dropping leading zeros: + # arbitrary precision without int(), which caps digit count on 3.11+. + return [(len(part), part) for part in (p.lstrip("0") or "0" for p in value.split("."))] + + +def compare(left: str, right: str) -> int: + a, b = _key(left), _key(right) + width = max(len(a), len(b)) + a += [(1, "0")] * (width - len(a)) + b += [(1, "0")] * (width - len(b)) + return (a > b) - (a < b) + + +def equivalents(version: str, names) -> list: + return [name for name in names if is_version(name) and compare(name, version) == 0] + + +def _php_empty(value) -> bool: + return value in ("", "0", b"", b"0") + + +def _ascii_lower(value: str) -> str: + return "".join(c.lower() if "A" <= c <= "Z" else c for c in value) + + +def _readme_header(line: str, only_valid: bool = False): + if ":" not in line or line.startswith(("#", "=")): + return None + key, value = line.split(":", 1) + key = _ascii_lower(key.strip(" \t*-\r\n")) + if only_valid and key not in README_HEADERS: + return None + return key, value + + +def stable_tag(data: bytes) -> str: + """Return the readme's single Stable Tag value, whitespace-trimmed. + + The header section ends where the wordpress.org readme parser ends it, so + examples further down the readme never count. + """ + if data.startswith(b"\xef\xbb\xbf"): + data = data[3:] + if data.startswith((b"\xff\xfe", b"\xfe\xff")): + raise ReleaseError("readme.txt is UTF-16; only UTF-8 is supported") + try: + encoding, lines = "utf-8", NEWLINES_UTF8.split(data.decode("utf-8")) + except UnicodeDecodeError: + encoding, lines = "latin-1", NEWLINES_BYTES.split(data.decode("latin-1")) + stack = collections.deque(lines) + + def first_nonwhitespace(): + while stack: + line = stack.popleft() + if not _php_empty(line.strip(" \t\n\r\0\x0b")): + return line + return "" + + # Title line, unless the readme starts straight with a header field. + line = first_nonwhitespace() + name = line.strip("#= \t\0\x0b") + if _readme_header(line, only_valid=True): + stack.appendleft(line) + name = "" + if stack and stack[0].strip("=-") == "": + stack.popleft() + if _ascii_lower(name) == "plugin name": + line = first_nonwhitespace() + if len(line.encode(encoding)) >= 50 or _readme_header(line, only_valid=True): + stack.appendleft(line) + + found = [] + line = first_nonwhitespace() + last_blank = False + while True: + header = _readme_header(line) + if header is None: + if not _php_empty(line): + break # The short description: the header section is over. + last_blank = True + else: + key, value = header + if key == "stable tag": + found.append(value) + elif key not in README_HEADERS and last_blank: + break + last_blank = False + if not stack: + break + line = stack.popleft() + + if not found: + raise ReleaseError("readme.txt has no Stable Tag field in its header") + if len(found) > 1: + raise ReleaseError(f"readme.txt has {len(found)} Stable Tag fields in its header") + return found[0].strip(" \t") + + +def _plugin_headers(data: bytes, name: bytes) -> list: + """Every match of WordPress's header regex, cleaned like get_file_data.""" + pattern = re.compile(rb"^(?:[ \t]*<\?php)?[ \t/*#@]*" + re.escape(name) + rb":(.*)$", re.M | re.I) + values = [] + for match in pattern.finditer(data): + raw = match.group(1) + cleaned = re.sub(rb"\s*(?:\*/|\?>).*", b"", raw).strip(b" \t\n\r\0\x0b") + values.append(b"" if _php_empty(raw) else cleaned) + return values + + +def plugin_version(root: str) -> str: + """Return the Version header of the one root PHP file declaring a plugin.""" + plugins = [] + for entry in sorted(os.listdir(root)): + path = os.path.join(root, entry) + if not entry.endswith(".php") or os.path.islink(path) or not os.path.isfile(path): + continue + with open(path, "rb") as handle: + data = handle.read(HEADER_BYTES).replace(b"\r", b"\n") + names = _plugin_headers(data, b"Plugin Name") + if not any(not _php_empty(value) for value in names): + continue + if len(names) > 1: + raise ReleaseError(f"a root PHP file has {len(names)} Plugin Name headers") + plugins.append(data) + + if not plugins: + raise ReleaseError("no root PHP file has a Plugin Name header") + if len(plugins) > 1: + raise ReleaseError(f"{len(plugins)} root PHP files have a Plugin Name header") + + versions = _plugin_headers(plugins[0], b"Version") + if len(versions) > 1: + raise ReleaseError(f"the plugin file has {len(versions)} Version headers") + if not versions or _php_empty(versions[0]): + raise ReleaseError("the plugin file has no Version header") + version = versions[0].decode("latin-1") + if not is_version(version): + raise ReleaseError("the plugin Version header is not a numeric version") + return version + + +def main(argv) -> int: + command, args = (argv[0], argv[1:]) if argv else ("", []) + try: + if command == "version" and len(args) == 1: + return 0 if is_version(args[0]) else 1 + if command == "compare" and len(args) == 2: + print({-1: "lt", 0: "eq", 1: "gt"}[compare(*args)]) + elif command == "equivalents" and len(args) == 1: + for name in equivalents(args[0], sys.stdin.read().splitlines()): + print(name) + elif command == "stable-tag" and len(args) == 1: + with open(args[0], "rb") as handle: + print(stable_tag(handle.read())) + elif command == "plugin-version" and len(args) == 1: + print(plugin_version(args[0])) + else: + print(__doc__, file=sys.stderr) + return 2 + except (ReleaseError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/actions/wporg-svn/lib/svn.sh b/.github/actions/wporg-svn/lib/svn.sh new file mode 100644 index 0000000..de6dd8c --- /dev/null +++ b/.github/actions/wporg-svn/lib/svn.sh @@ -0,0 +1,238 @@ +# shellcheck shell=bash +# Shared by the wporg-svn actions: strict mode, input checks, a private SVN +# configuration, revision-pinned reads, authenticated writes and failure +# reports. Set ACTION_NAME, then source this file; it does nothing on its own. +# +# Inputs arrive as environment variables, mapped by each action.yml: +# WPORG_SLUG WPORG_VERSION WPORG_ZIP_URL WPORG_USERNAME WPORG_PASSWORD +# Test seam, not an action input: WPORG_SVN_TEST_ROOT replaces the production +# root, and only a file:// URL is accepted, so it can never send credentials +# anywhere. +# +# Helpers return values in REPLY rather than through $(...), so a failure +# inside them stops the whole action. + +set -euo pipefail + +# Keep the password out of every child's environment before any child starts: +# svn reads it from stdin only. +SVN_PASSWORD=${WPORG_PASSWORD:-} +export -n SVN_PASSWORD +unset WPORG_PASSWORD + +# English messages and ASCII regex ranges, but UTF-8: in plain C, svn on Linux +# cannot convert non-ASCII file names in a working copy. +export LC_ALL=C.UTF-8 +umask 077 + +LIB_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PRODUCTION_ROOT=https://plugins.svn.wordpress.org + +STAGE="checking inputs" +WORK= +BASE= +SLUG= +VERSION= +REVISIONS= +RECOVERY= + +# Only fixed text and validated values ever reach these messages. +fail() { + printf '::error title=%s::%s\n' "$ACTION_NAME" "$1" + exit 1 +} + +note_revision() { + REVISIONS="${REVISIONS:+$REVISIONS, }$1" +} + +summary() { + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '%s\n' "$@" >>"$GITHUB_STEP_SUMMARY" + fi +} + +output() { + if [ -n "${GITHUB_OUTPUT:-}" ]; then + printf '%s=%s\n' "$1" "$2" >>"$GITHUB_OUTPUT" + else + printf '%s=%s\n' "$1" "$2" + fi +} + +on_exit() { + local status=$? + trap - EXIT + if [ "$status" -ne 0 ]; then + local subject="${SLUG:-the plugin}${VERSION:+ $VERSION}" + printf '::error title=%s::%s\n' "$ACTION_NAME" \ + "Failed while $STAGE for $subject. Known revisions: ${REVISIONS:-none}.${RECOVERY:+ $RECOVERY}" + summary "### $ACTION_NAME failed" "" "- Plugin: $subject" "- Stage: $STAGE" \ + "- Known revisions: ${REVISIONS:-none}" ${RECOVERY:+"- Recovery: $RECOVERY"} + fi + if [ -n "$WORK" ]; then + rm -rf -- "$WORK" + fi + exit "$status" +} +trap on_exit EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# Usage: require_tools TOOL... (each script names everything it runs) +require_tools() { + if [ -n "${RUNNER_OS:-}" ] && [ "$RUNNER_OS" != Linux ]; then + fail "unsupported runner: use an Ubuntu runner" + fi + local tool + for tool in "$@"; do + command -v "$tool" >/dev/null 2>&1 || + fail "missing required tool: $tool (on Ubuntu: sudo apt-get install -y subversion)" + done + python3 -c 'import sys; sys.exit(sys.version_info < (3, 8))' || fail "Python 3.8 or newer is required" + case $(svn --version --quiet) in + 1.[0-9].*) fail "Subversion 1.10 or newer is required for --password-from-stdin" ;; + esac +} + +validate_slug() { + local pattern='^[a-z0-9]+(-[a-z0-9]+)*$' + [[ ${WPORG_SLUG:-} =~ $pattern ]] || + fail "plugin-slug must be lowercase letters and digits, with single hyphens between them" + SLUG=$WPORG_SLUG +} + +validate_version() { + python3 "$LIB_DIR/release.py" version "${WPORG_VERSION:-}" || + fail "version must be numeric dot-separated digits, like 1.2.3" + VERSION=$WPORG_VERSION +} + +# Optional: empty means "not given". +validate_expected_revision() { + local pattern='^[1-9][0-9]*$' + [ -z "${WPORG_EXPECTED_REVISION:-}" ] || [[ $WPORG_EXPECTED_REVISION =~ $pattern ]] || + fail "expected-revision must be a revision number, like the tag action's revision output" +} + +# With expected-revision given, stop unless tags/ last changed at +# exactly that revision, as of REV: what gets released is the tag that was QA'd. +check_expected_revision() { + [ -n "${WPORG_EXPECTED_REVISION:-}" ] || return 0 + last_changed "$BASE/tags/$VERSION" "$1" + [ "$REPLY" = "$WPORG_EXPECTED_REVISION" ] || + fail "tags/$VERSION last changed at r$REPLY, not at the expected r$WPORG_EXPECTED_REVISION; it is not the tag that was approved" +} + +validate_credentials() { + if [ -z "${WPORG_USERNAME:-}" ] || [ -z "$SVN_PASSWORD" ]; then + fail "wporg-username and wporg-password are required" + fi + case $WPORG_USERNAME$SVN_PASSWORD in + *$'\n'* | *$'\r'*) fail "wporg-username and wporg-password must be single lines" ;; + esac +} + +# Resolve the repository URL and create the private workspace. +prepare() { + local root=$PRODUCTION_ROOT tmp=${RUNNER_TEMP:-${TMPDIR:-/tmp}} + if [ -n "${WPORG_SVN_TEST_ROOT:-}" ]; then + case $WPORG_SVN_TEST_ROOT in + file:///*) root=${WPORG_SVN_TEST_ROOT%/} ;; + *) fail "WPORG_SVN_TEST_ROOT only accepts file:// URLs" ;; + esac + fi + BASE="$root/$SLUG" + WORK=$(mktemp -d "${tmp%/}/wporg-svn.XXXXXXXX") + mkdir "$WORK/svn-config" + printf '[auth]\npassword-stores =\n[miscellany]\nenable-auto-props = no\n' >"$WORK/svn-config/config" + # Bulk updates: one streamed response per checkout or export instead of one + # request per file, 2-5x faster from plugins.svn.wordpress.org. + printf '[global]\nstore-passwords = no\nstore-auth-creds = no\nhttp-bulk-updates = yes\n' >"$WORK/svn-config/servers" +} + +# A short, sanitized description of the last SVN error: a class and the codes. +svn_error() { + local codes class=unclassified + codes=$(grep -oE 'E[0-9]{6}' "$WORK/err" | sort -u | tr '\n' ' ') || true + case " $codes" in + *" E170001 "* | *" E215004 "*) class="authentication failed" ;; + *" E230001 "* | *" E120171 "*) class="TLS certificate verification failed" ;; + *" E160024 "* | *" E160028 "* | *" E170004 "* | *" E160020 "* | *" E155011 "*) + class="out of date: another commit changed the destination" ;; + *" E175013 "* | *" E000013 "* | *" E220001 "*) class="permission denied" ;; + *" E170013 "* | *" E175002 "* | *" E175012 "* | *" E670002 "* | *" E670008 "* | *" E120108 "*) + class="could not connect" ;; + *" E160013 "* | *" E170000 "* | *" E200009 "*) class="path not found" ;; + esac + REPLY="$class${codes:+ (${codes% })}" +} + +# Anonymous read into $WORK/out. A failed read is never taken to mean +# "absent": it stops the action. +svn_read() { + if ! svn --non-interactive --no-auth-cache --config-dir "$WORK/svn-config" "$@" >"$WORK/out" 2>"$WORK/err"; then + svn_error + fail "SVN read failed while $STAGE: $REPLY" + fi +} + +# REPLY = the repository's current (operative) revision. Pin every dependent +# read to it. +snapshot() { + local pattern='^[0-9]+$' + svn_read info --show-item revision -- "$BASE" + REPLY=$(tr -d '[:space:]' <"$WORK/out") + [[ $REPLY =~ $pattern ]] || fail "could not read the repository revision" +} + +# REPLY = dir, file or absent: NAME inside DIR_URL at REV. +svn_kind() { + svn_read list --xml -r "$3" -- "$1@$3" + REPLY=$(python3 "$LIB_DIR/svnxml.py" kind "$2" <"$WORK/out") +} + +# REPLY = every entry name inside DIR_URL at REV, one per line. +svn_names() { + svn_read list --xml -r "$2" -- "$1@$2" + REPLY=$(python3 "$LIB_DIR/svnxml.py" names <"$WORK/out") +} + +# REPLY = the last revision that changed URL, as of REV. +last_changed() { + svn_read info --show-item last-changed-revision -r "$2" -- "$1@$2" + REPLY=$(tr -d '[:space:]' <"$WORK/out") +} + +# Copy the file URL at REV to DEST, byte for byte. +svn_cat() { + svn_read cat -r "$2" -- "$1@$2" + mv -- "$WORK/out" "$3" +} + +# Stop when anything at URL (recursively) sets a content-transforming property. +check_props() { + svn_read proplist -R --xml -r "$2" -- "$1@$2" + python3 "$LIB_DIR/svnxml.py" check-props <"$WORK/out" || + fail "$3 has SVN properties that change file contents; an exact release cannot keep them" +} + +# Authenticated writes. The password travels on stdin, never in argv. On +# success REPLY is the new revision. On failure they return 1 and the outcome +# is unknown: the caller inspects, reports and stops. Writes are never retried. +mucc_write() { + REPLY= + printf '%s\n' "$SVN_PASSWORD" | svnmucc --non-interactive --no-auth-cache --config-dir "$WORK/svn-config" \ + --username "$WPORG_USERNAME" --password-from-stdin "$@" >"$WORK/out" 2>"$WORK/err" || return 1 + REPLY=$(sed -n 's/^r\([0-9][0-9]*\) committed.*/\1/p' "$WORK/out") + [ -n "$REPLY" ] +} + +commit_write() { + REPLY= + printf '%s\n' "$SVN_PASSWORD" | svn --non-interactive --no-auth-cache --config-dir "$WORK/svn-config" \ + --username "$WPORG_USERNAME" --password-from-stdin commit -m "$2" -- "$1" >"$WORK/out" 2>"$WORK/err" || + return 1 + REPLY=$(sed -n 's/^Committed revision \([0-9][0-9]*\)\.$/\1/p' "$WORK/out") + [ -n "$REPLY" ] +} diff --git a/.github/actions/wporg-svn/lib/svnxml.py b/.github/actions/wporg-svn/lib/svnxml.py new file mode 100644 index 0000000..d44e743 --- /dev/null +++ b/.github/actions/wporg-svn/lib/svnxml.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Answer questions about the XML that svn prints. The XML arrives on stdin. + +Usage: + svnxml.py kind NAME print dir, file or absent for NAME in `svn list --xml` + svnxml.py names print every entry name in `svn list --xml` + svnxml.py check-props exit 1 when `svn proplist -R --xml` shows a property + that rewrites file bytes or pulls content from elsewhere + svnxml.py diff-empty exit 0 when `svn diff --summarize --xml` lists nothing + +Unreadable XML exits 3, so it is never mistaken for an answer. +""" + +from __future__ import annotations + +import sys +import xml.etree.ElementTree as ET + +# Checked-out bytes differ from repository bytes under these, or content comes +# from another location. An exact artifact release cannot keep them. +TRANSFORMING = ("svn:eol-style", "svn:externals", "svn:keywords", "svn:special") + + +def main(argv) -> int: + command, args = (argv[0], argv[1:]) if argv else ("", []) + if (command, len(args)) not in (("kind", 1), ("names", 0), ("check-props", 0), ("diff-empty", 0)): + print(__doc__, file=sys.stderr) + return 2 + try: + doc = ET.parse(sys.stdin.buffer).getroot() + except ET.ParseError: + print("error: unreadable svn XML", file=sys.stderr) + return 3 + + if command == "kind": + kinds = [entry.get("kind") for entry in doc.iter("entry") if entry.findtext("name") == args[0]] + print(kinds[0] if kinds else "absent") + elif command == "names": + for entry in doc.iter("entry"): + print(entry.findtext("name")) + elif command == "check-props": + counts = {} + for prop in doc.iter("property"): + if prop.get("name") in TRANSFORMING: + counts[prop.get("name")] = counts.get(prop.get("name"), 0) + 1 + for name, count in sorted(counts.items()): + print(f"error: {count} path(s) set {name}", file=sys.stderr) + return 1 if counts else 0 + elif command == "diff-empty": + changed = len(list(doc.iter("path"))) + if changed: + print(f"{changed} path(s) differ", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/actions/wporg-svn/lib/tree.py b/.github/actions/wporg-svn/lib/tree.py new file mode 100644 index 0000000..c3653e4 --- /dev/null +++ b/.github/actions/wporg-svn/lib/tree.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Make a working copy match an artifact tree exactly, and compare trees. + +Usage: + tree.py sync SRC WC make WC's files match SRC; WC/.svn is untouched + tree.py reconcile WC CONFIG_DIR schedule the SVN adds, deletes and replacements + that make WC commit as it now is; print the + number of changed paths + tree.py compare A B exit 0 when A and B hold the same paths, kinds + and bytes (a top-level .svn is ignored) +""" + +from __future__ import annotations + +import os +import shutil +import stat +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET + +FIXABLE = {"unversioned", "ignored", "missing", "obstructed", "modified"} +COMMITTABLE = {"added", "deleted", "replaced", "modified"} +# Not "copied": tag reconciles a fresh copy, where every changed node is copied. +FLAGS = ("tree-conflicted", "switched", "wc-locked") + + +class TreeError(Exception): + pass + + +def _kind(path): + try: + mode = os.lstat(path).st_mode + except FileNotFoundError: + return None + if stat.S_ISDIR(mode): + return "dir" + return "file" if stat.S_ISREG(mode) else "other" + + +def entries(root) -> dict: + """{relative path: kind} under root, skipping a top-level .svn.""" + found = {} + for dirpath, dirnames, filenames in os.walk(root): + rel = os.path.relpath(dirpath, root) + if rel == ".": + dirnames[:] = [d for d in dirnames if d != ".svn"] + for name in dirnames + filenames: + path = os.path.normpath(os.path.join(rel, name)) + found[path] = _kind(os.path.join(root, path)) + return found + + +def _depth(path): + return path.count(os.sep) + + +def _same_bytes(a, b) -> bool: + if os.path.getsize(a) != os.path.getsize(b): + return False + with open(a, "rb") as left, open(b, "rb") as right: + while True: + chunk = left.read(1024 * 1024) + if chunk != right.read(1024 * 1024): + return False + if not chunk: + return True + + +def _remove(path): + if _kind(path) == "dir": + shutil.rmtree(path) + else: + os.unlink(path) + + +def sync(src, wc): + """Make wc's tree (outside .svn) equal src's: kinds, paths and bytes.""" + want, have = entries(src), entries(wc) + if "other" in want.values(): + raise TreeError("the artifact contains something other than files and directories") + for rel in sorted(have, key=_depth): + if want.get(rel) != have[rel] and os.path.lexists(os.path.join(wc, rel)): + _remove(os.path.join(wc, rel)) + for rel in sorted(want, key=_depth): + source, target = os.path.join(src, rel), os.path.join(wc, rel) + if want[rel] == "dir": + os.makedirs(target, exist_ok=True) + elif not os.path.lexists(target) or not _same_bytes(source, target): + if os.path.lexists(target): + os.unlink(target) + shutil.copyfile(source, target) + + +class _Svn: + def __init__(self, config): + self.base = ["svn", "--non-interactive", "--config-dir", config] + + def run(self, *args) -> bytes: + result = subprocess.run(self.base + list(args), capture_output=True) + if result.returncode: + raise TreeError(f"svn {args[0]} failed: " + result.stderr.decode("utf-8", "replace").strip()[-300:]) + return result.stdout + + def status(self, wc) -> list: + """(path, item, props, flags) for every entry svn reports. + + A copy scheduled for addition (svn copy URL WC) reports its root as + added; that is the copy itself, not a change to reconcile or count. + """ + doc = ET.fromstring(self.run("status", "--xml", "--no-ignore", "--", wc + "@")) + found = [] + for entry in doc.iter("entry"): + status = entry.find("wc-status") + if entry.get("path") == wc and status.get("item") == "added" and status.get("copied") == "true": + continue + flags = {flag for flag in FLAGS if status.get(flag) == "true"} + found.append((entry.get("path"), status.get("item"), status.get("props"), flags)) + return found + + def _each(self, args, paths): + """Run svn args once for all paths, listed in a --targets file.""" + if not paths: + return + with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".targets") as targets: + targets.write("".join(path + "@\n" for path in paths)) + targets.flush() + self.run(*args, "--targets", targets.name) + + def add(self, *paths): + self._each(["add", "--depth", "infinity", "--no-ignore", "--no-auto-props"], paths) + + def rm(self, *paths): + self._each(["rm", "--force"], paths) + + +def _check(found, allowed, when): + # svn add itself sets svn:mime-type on binary files, so added items may carry + # properties; the caller checks the working copy's properties separately. + bad = sorted({f"{item} (props {props}{', ' + ', '.join(sorted(flags)) if flags else ''})" + for _, item, props, flags in found if item not in allowed or props == "conflicted" or flags}) + if bad: + raise TreeError(f"unexpected working copy state {when}: " + "; ".join(bad)) + + +def _tops(paths) -> list: + """paths without those inside another of them: svn add and rm recurse.""" + tops = set() + for path in sorted(paths, key=_depth): + parent = os.path.dirname(path) + while parent not in tops and parent != os.path.dirname(parent): + parent = os.path.dirname(parent) + if parent not in tops: + tops.add(path) + return sorted(tops) + + +def reconcile(wc, config) -> int: + """Schedule SVN operations so wc commits as it is on disk; return the change count.""" + svn = _Svn(config) + _check(svn.status(wc), FIXABLE, "before reconciling") + + # A file became a directory or the reverse: delete the old node, add the new. + for path, item, _, _ in svn.status(wc): + if item == "obstructed": + aside = tempfile.mkdtemp(dir=os.path.dirname(wc)) + os.rename(path, os.path.join(aside, "item")) + svn.rm(path) + os.rename(os.path.join(aside, "item"), path) + os.rmdir(aside) + svn.add(path) + + svn.rm(*_tops(p for p, item, _, _ in svn.status(wc) if item == "missing")) + svn.add(*_tops(p for p, item, _, _ in svn.status(wc) if item in ("unversioned", "ignored"))) + + final = svn.status(wc) + _check(final, COMMITTABLE, "after reconciling") + return len(final) + + +def compare(a, b) -> list: + """Relative paths whose presence, kind or bytes differ between a and b.""" + left, right = entries(a), entries(b) + differ = [rel for rel in sorted(set(left) | set(right)) if left.get(rel) != right.get(rel)] + for rel in sorted(set(left) & set(right)): + if left[rel] == "file" == right[rel] and not _same_bytes(os.path.join(a, rel), os.path.join(b, rel)): + differ.append(rel) + return differ + + +def main(argv) -> int: + command, args = (argv[0], argv[1:]) if argv else ("", []) + try: + if command == "sync" and len(args) == 2: + sync(*args) + elif command == "reconcile" and len(args) == 2: + print(reconcile(*args)) + elif command == "compare" and len(args) == 2: + differ = compare(*args) + if differ: + print(f"error: {len(differ)} path(s) differ, for example:", file=sys.stderr) + for rel in differ[:10]: + print(f" differs: {rel}", file=sys.stderr) + return 1 + else: + print(__doc__, file=sys.stderr) + return 2 + except (TreeError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/actions/wporg-svn/set-stable/action.yml b/.github/actions/wporg-svn/set-stable/action.yml new file mode 100644 index 0000000..1391efb --- /dev/null +++ b/.github/actions/wporg-svn/set-stable/action.yml @@ -0,0 +1,46 @@ +name: "WordPress.org SVN: set stable" +description: >- + Release a prepared tag: copy tags//readme.txt over trunk/readme.txt in one server-side commit, + which points the Stable Tag at that version. Choosing an older tag rolls the release back. +inputs: + plugin-slug: + description: The plugin's wordpress.org slug. + required: true + version: + description: The tag to release, usually the tag action's version output. + required: true + expected-revision: + description: >- + Optional: the tag action's revision output. When set, refuse unless tags/ last changed at + exactly this revision, so the release is the tag that was QA'd. Leave it empty for a manual run or rollback. + required: false + wporg-username: + description: WordPress.org account with commit access to the plugin. + required: true + wporg-password: + description: That account's SVN password, not its login password. + required: true +outputs: + version: + description: The version now set as stable. + value: ${{ steps.release.outputs.version }} + revision: + description: The revision of the readme commit, or the observed revision for a verified no-op. + value: ${{ steps.release.outputs.revision }} +runs: + using: composite + steps: + - name: Mask the password + shell: bash + env: + PASSWORD: ${{ inputs.wporg-password }} + run: printf '%s\n' "$PASSWORD" | while IFS= read -r line; do [ -z "$line" ] || { line=${line//%/%25}; echo "::add-mask::${line//$'\r'/%0D}"; }; done + - id: release + shell: bash + env: + WPORG_SLUG: ${{ inputs.plugin-slug }} + WPORG_VERSION: ${{ inputs.version }} + WPORG_EXPECTED_REVISION: ${{ inputs.expected-revision }} + WPORG_USERNAME: ${{ inputs.wporg-username }} + WPORG_PASSWORD: ${{ inputs.wporg-password }} + run: bash "$GITHUB_ACTION_PATH/set-stable.sh" diff --git a/.github/actions/wporg-svn/set-stable/set-stable.sh b/.github/actions/wporg-svn/set-stable/set-stable.sh new file mode 100644 index 0000000..0eb8cbf --- /dev/null +++ b/.github/actions/wporg-svn/set-stable/set-stable.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Move the release pointer: put tags//readme.txt over trunk/readme.txt +# in one server-side commit, with no checkout. There is no upgrade-only rule: +# choosing an older tag rolls the release back. +ACTION_NAME="wporg-svn set-stable" +# shellcheck source=SCRIPTDIR/../lib/svn.sh +. "${BASH_SOURCE[0]%/*}/../lib/svn.sh" + +require_tools svn svnmucc python3 cmp +validate_slug +validate_version +validate_expected_revision +validate_credentials +prepare + +STAGE="reading tags/$VERSION and trunk" +snapshot +R=$REPLY +note_revision "snapshot r$R" +svn_kind "$BASE/tags" "$VERSION" "$R" +[ "$REPLY" = dir ] || fail "tags/$VERSION does not exist at r$R" +check_expected_revision "$R" +svn_kind "$BASE/tags/$VERSION" readme.txt "$R" +[ "$REPLY" = file ] || fail "tags/$VERSION/readme.txt is not a file at r$R" +svn_kind "$BASE/trunk" readme.txt "$R" +[ "$REPLY" = file ] || fail "trunk/readme.txt is not a file at r$R" +check_props "$BASE/tags/$VERSION/readme.txt" "$R" "tags/$VERSION/readme.txt" +check_props "$BASE/trunk/readme.txt" "$R" "trunk/readme.txt" +svn_cat "$BASE/tags/$VERSION/readme.txt" "$R" "$WORK/release-readme.txt" +svn_cat "$BASE/trunk/readme.txt" "$R" "$WORK/trunk-readme.txt" +STABLE=$(python3 "$LIB_DIR/release.py" stable-tag "$WORK/release-readme.txt") +[ "$STABLE" = "$VERSION" ] || fail "the Stable Tag in tags/$VERSION/readme.txt is not $VERSION" + +if cmp -s "$WORK/release-readme.txt" "$WORK/trunk-readme.txt"; then + output version "$VERSION" + output revision "$R" + summary "### $ACTION_NAME: verified no-op" "" \ + "trunk/readme.txt already matches tags/$VERSION/readme.txt at r$R. Nothing was committed." + exit 0 +fi + +STAGE="committing trunk/readme.txt" +if ! mucc_write -r "$R" -m "Release $VERSION: set the Stable Tag" \ + put "$WORK/release-readme.txt" "$BASE/trunk/readme.txt"; then + svn_error + ERROR=$REPLY + STAGE="inspecting trunk/readme.txt after a failed write" + snapshot + HEAD=$REPLY + svn_cat "$BASE/trunk/readme.txt" "$HEAD" "$WORK/after-readme.txt" + STAGE="committing trunk/readme.txt" + if cmp -s "$WORK/release-readme.txt" "$WORK/after-readme.txt"; then + fail "the write reported an error ($ERROR), but trunk/readme.txt now matches tags/$VERSION/readme.txt. Rerunning is safe: it verifies and reports a no-op." + fi + fail "the write failed ($ERROR) and trunk/readme.txt does not match tags/$VERSION/readme.txt at r$HEAD; check who else is committing (svn log) before rerunning." +fi +N=$REPLY +note_revision "commit r$N" + +STAGE="verifying trunk/readme.txt at r$N" +svn_cat "$BASE/trunk/readme.txt" "$N" "$WORK/committed-readme.txt" +cmp -s "$WORK/release-readme.txt" "$WORK/committed-readme.txt" || + fail "trunk/readme.txt at r$N does not match tags/$VERSION/readme.txt" + +output version "$VERSION" +output revision "$N" +summary "### $ACTION_NAME: $SLUG $VERSION" "" \ + "trunk/readme.txt now matches tags/$VERSION/readme.txt (r$N). WordPress.org processing happens separately." diff --git a/.github/actions/wporg-svn/tag/action.yml b/.github/actions/wporg-svn/tag/action.yml new file mode 100644 index 0000000..69459d3 --- /dev/null +++ b/.github/actions/wporg-svn/tag/action.yml @@ -0,0 +1,46 @@ +name: "WordPress.org SVN: tag" +description: >- + Prepare a release: create tags/ in one commit, as a copy of the current stable tag that matches + a plugin ZIP exactly. Does not touch trunk or move the stable pointer. +inputs: + plugin-slug: + description: The plugin's wordpress.org slug. + required: true + zip-url: + description: >- + HTTPS URL of the built plugin ZIP. The action never prints it and masks it, but GitHub logs step inputs + before the action runs: pass it from a secret, or mask it in an earlier step. + required: true + wporg-username: + description: WordPress.org account with commit access to the plugin. + required: true + wporg-password: + description: That account's SVN password, not its login password. + required: true +outputs: + version: + description: The version released, read from the plugin header. + value: ${{ steps.release.outputs.version }} + revision: + description: The revision that created tags/. + value: ${{ steps.release.outputs.revision }} + previous-stable: + description: The stable tag that tags/ was copied from. + value: ${{ steps.release.outputs.previous-stable }} +runs: + using: composite + steps: + - name: Mask the artifact URL and the password + shell: bash + env: + ZIP_URL: ${{ inputs.zip-url }} + PASSWORD: ${{ inputs.wporg-password }} + run: printf '%s\n' "$ZIP_URL" "$PASSWORD" | while IFS= read -r line; do [ -z "$line" ] || { line=${line//%/%25}; echo "::add-mask::${line//$'\r'/%0D}"; }; done + - id: release + shell: bash + env: + WPORG_SLUG: ${{ inputs.plugin-slug }} + WPORG_ZIP_URL: ${{ inputs.zip-url }} + WPORG_USERNAME: ${{ inputs.wporg-username }} + WPORG_PASSWORD: ${{ inputs.wporg-password }} + run: bash "$GITHUB_ACTION_PATH/tag.sh" diff --git a/.github/actions/wporg-svn/tag/tag.sh b/.github/actions/wporg-svn/tag/tag.sh new file mode 100644 index 0000000..855b827 --- /dev/null +++ b/.github/actions/wporg-svn/tag/tag.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Prepare a release without moving the stable pointer: copy the stable tag into +# an otherwise empty working copy as tags/, make it match the ZIP +# exactly and commit it. trunk and existing tags stay as they are. +# +# This is one commit: tags/ appears complete, or not at all. The +# commit records a copy of the stable tag, so only changed files are uploaded. +ACTION_NAME="wporg-svn tag" +# shellcheck source=SCRIPTDIR/../lib/svn.sh +. "${BASH_SOURCE[0]%/*}/../lib/svn.sh" + +# Keep in step with MAX_ARCHIVE_BYTES in lib/archive.py. +MAX_ZIP_BYTES=268435456 + +require_tools svn python3 curl +validate_slug +validate_credentials +[ -n "${WPORG_ZIP_URL:-}" ] || fail "zip-url is required" +case $WPORG_ZIP_URL in + https://*) ;; + *) fail "zip-url must be an https:// URL" ;; +esac +case $WPORG_ZIP_URL in + *[[:space:][:cntrl:]]*) fail "zip-url must not contain spaces or control characters" ;; +esac +prepare + +release_py() { + python3 "$LIB_DIR/release.py" "$@" +} + +# REPLY = the numeric Stable Tag in trunk/readme.txt at REV. +stable_at() { + svn_kind "$BASE/trunk" readme.txt "$1" + [ "$REPLY" = file ] || fail "trunk/readme.txt is not a file at r$1" + svn_cat "$BASE/trunk/readme.txt" "$1" "$WORK/trunk-readme.txt" + REPLY=$(release_py stable-tag "$WORK/trunk-readme.txt") + release_py version "$REPLY" || + fail "the trunk/readme.txt Stable Tag is not a numeric version; first releases are not supported" +} + +# Fail when tags/, or another spelling of the same version, exists at REV. +destination_absent() { + local found + svn_names "$BASE/tags" "$1" + found=$(printf '%s\n' "$REPLY" | release_py equivalents "$VERSION") + if [ -n "$found" ]; then + fail "tags/$VERSION already exists at r$1 (or under an equivalent spelling); released tags are never reused" + fi +} + +# Fetch zip-url over HTTPS only. Nothing about the URL is ever printed: signed +# URLs are secrets. No credentials are sent to the artifact host. +download() { + local code status=0 + # curl before 8.4 ignores --max-filesize when the server sends no length, so a + # file size limit (1 KiB blocks, this subshell only) stops it too: SIGXFSZ, 153. + code=$(ulimit -f $((MAX_ZIP_BYTES / 1024 + 1)) && + curl -q --silent --show-error --fail --location --proto '=https' --proto-redir '=https' \ + --max-redirs 5 --connect-timeout 30 --max-time 600 --retry 3 --retry-delay 5 --retry-max-time 900 \ + --max-filesize "$MAX_ZIP_BYTES" --write-out '%{http_code}' --output "$WORK/artifact.zip" \ + "$WPORG_ZIP_URL" 2>"$WORK/err") || status=$? + if [ "$status" -eq 63 ] || [ "$status" -eq 153 ]; then + fail "downloading zip-url failed: the ZIP is larger than $((MAX_ZIP_BYTES / 1048576)) MiB" + fi + if [ "$status" -ne 0 ]; then + if [[ $code =~ ^[1-9][0-9][0-9]$ ]]; then + fail "downloading zip-url failed (HTTP $code)" + fi + fail "downloading zip-url failed (curl exit $status)" + fi +} + +STAGE="reading the current release" +snapshot +R=$REPLY +note_revision "snapshot r$R" +stable_at "$R" +S=$REPLY +svn_kind "$BASE/tags" "$S" "$R" +[ "$REPLY" = dir ] || fail "the stable tag tags/$S does not exist at r$R" + +STAGE="downloading the artifact" +download + +STAGE="inspecting the artifact" +mkdir "$WORK/artifact" +ROOT=$(python3 "$LIB_DIR/archive.py" extract "$WORK/artifact.zip" "$WORK/artifact") +VERSION=$(release_py plugin-version "$ROOT") +[ "$(release_py compare "$VERSION" "$S")" = gt ] || fail "the plugin version $VERSION is not newer than the stable $S" +[ "$(release_py stable-tag "$ROOT/readme.txt")" = "$VERSION" ] || + fail "the Stable Tag in the artifact readme.txt is not $VERSION" + +STAGE="checking tags/$VERSION and tags/$S" +destination_absent "$R" +last_changed "$BASE/tags/$S" "$R" +SOURCE_CHANGED=$REPLY + +# Stop with MESSAGE when anything in the new tag sets a content-transforming property. +check_tag_props() { + svn_read proplist -R --xml -- "$TAG" + python3 "$LIB_DIR/svnxml.py" check-props <"$WORK/out" || fail "$1" +} + +STAGE="copying tags/$S into a working copy" +WC="$WORK/wc" +TAG="$WC/$VERSION" +svn_read checkout --quiet --depth empty -r "$R" -- "$BASE/tags@$R" "$WC" +svn_read copy --quiet --ignore-externals -r "$R" -- "$BASE/tags/$S@$R" "$TAG" +check_tag_props "tags/$S has SVN properties that change file contents; an exact release cannot keep them" + +STAGE="synchronising tags/$VERSION with the artifact" +python3 "$LIB_DIR/tree.py" sync "$ROOT" "$TAG" +CHANGES=$(python3 "$LIB_DIR/tree.py" reconcile "$TAG" "$WORK/svn-config") +check_tag_props "the working copy of tags/$VERSION has SVN properties that change file contents" + +STAGE="validating tags/$VERSION before the commit" +[ "$(release_py plugin-version "$TAG")" = "$VERSION" ] || fail "the working copy's plugin Version is not $VERSION" +[ "$(release_py stable-tag "$TAG/readme.txt")" = "$VERSION" ] || fail "the working copy's Stable Tag is not $VERSION" +python3 "$LIB_DIR/tree.py" compare "$ROOT" "$TAG" || fail "the working copy does not match the artifact" +[ "$CHANGES" -gt 0 ] || fail "the artifact is identical to tags/$S; there is nothing to release" + +STAGE="rechecking before the commit" +snapshot +note_revision "recheck r$REPLY" +RECHECK=$REPLY +stable_at "$RECHECK" +[ "$REPLY" = "$S" ] || fail "the trunk/readme.txt Stable Tag changed after r$R; another release is in progress" +last_changed "$BASE/tags/$S" "$RECHECK" +[ "$REPLY" = "$SOURCE_CHANGED" ] || fail "tags/$S changed after r$R" +destination_absent "$RECHECK" + +STAGE="committing tags/$VERSION" +RECOVERY="The commit's outcome is unknown: if tags/$VERSION exists, do not approve or release it until it is verified against the ZIP (wporg-svn README, Recovery)." +if ! commit_write "$TAG" "Release $VERSION: create tags/$VERSION from the artifact"; then + svn_error + ERROR=$REPLY + STAGE="inspecting tags/$VERSION after a failed commit" + snapshot + HEAD=$REPLY + svn_kind "$BASE/tags" "$VERSION" "$HEAD" + STAGE="committing tags/$VERSION" + if [ "$REPLY" = absent ]; then + RECOVERY= + fail "the commit failed ($ERROR) and tags/$VERSION does not exist at r$HEAD, so nothing was written. Rerun once the cause is fixed." + fi + RECOVERY="tags/$VERSION exists at r$HEAD, but this run cannot confirm that it created it. Do not approve or release it: inspect it first (wporg-svn README, Recovery)." + fail "the commit reported an error ($ERROR) and tags/$VERSION now exists" +fi +N=$REPLY +note_revision "commit r$N" +RECOVERY="tags/$VERSION was created at r$N but not verified against the artifact. Do not approve or release it until it is (wporg-svn README, Recovery)." + +STAGE="verifying tags/$VERSION at r$N" +svn_read export --quiet --ignore-externals -r "$N" -- "$BASE/tags/$VERSION@$N" "$WORK/verify" +python3 "$LIB_DIR/tree.py" compare "$ROOT" "$WORK/verify" || fail "tags/$VERSION at r$N does not match the artifact" + +RECOVERY= +output version "$VERSION" +output revision "$N" +output previous-stable "$S" +summary "### $ACTION_NAME: $SLUG $VERSION" "" \ + "- tags/$VERSION created at r$N as a copy of tags/$S@$R that matches the artifact, and verified." \ + "- The stable pointer is unchanged. QA tags/$VERSION@$N, then run set-stable and update-trunk." diff --git a/.github/actions/wporg-svn/update-trunk/action.yml b/.github/actions/wporg-svn/update-trunk/action.yml new file mode 100644 index 0000000..82bcb20 --- /dev/null +++ b/.github/actions/wporg-svn/update-trunk/action.yml @@ -0,0 +1,46 @@ +name: "WordPress.org SVN: update trunk" +description: >- + Post-release housekeeping: replace trunk with tags/ in one atomic server-side commit. This also + replaces trunk/readme.txt, so it refuses to run until set-stable has pointed the Stable Tag at the version. +inputs: + plugin-slug: + description: The plugin's wordpress.org slug. + required: true + version: + description: The released tag to copy into trunk. + required: true + expected-revision: + description: >- + Optional: the tag action's revision output. When set, refuse unless tags/ last changed at + exactly this revision, so the release is the tag that was QA'd. Leave it empty for a manual run or rollback. + required: false + wporg-username: + description: WordPress.org account with commit access to the plugin. + required: true + wporg-password: + description: That account's SVN password, not its login password. + required: true +outputs: + version: + description: The version trunk now matches. + value: ${{ steps.release.outputs.version }} + revision: + description: The revision that replaced trunk, or the observed revision for a verified no-op. + value: ${{ steps.release.outputs.revision }} +runs: + using: composite + steps: + - name: Mask the password + shell: bash + env: + PASSWORD: ${{ inputs.wporg-password }} + run: printf '%s\n' "$PASSWORD" | while IFS= read -r line; do [ -z "$line" ] || { line=${line//%/%25}; echo "::add-mask::${line//$'\r'/%0D}"; }; done + - id: release + shell: bash + env: + WPORG_SLUG: ${{ inputs.plugin-slug }} + WPORG_VERSION: ${{ inputs.version }} + WPORG_EXPECTED_REVISION: ${{ inputs.expected-revision }} + WPORG_USERNAME: ${{ inputs.wporg-username }} + WPORG_PASSWORD: ${{ inputs.wporg-password }} + run: bash "$GITHUB_ACTION_PATH/update-trunk.sh" diff --git a/.github/actions/wporg-svn/update-trunk/update-trunk.sh b/.github/actions/wporg-svn/update-trunk/update-trunk.sh new file mode 100644 index 0000000..463c0a7 --- /dev/null +++ b/.github/actions/wporg-svn/update-trunk/update-trunk.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Post-release housekeeping: replace trunk with tags/ in one atomic +# commit (rm + copy of the tag pinned at the snapshot revision). No checkout, +# export or local transfer. This also replaces trunk/readme.txt, so it refuses +# to run until set-stable has pointed trunk's Stable Tag at the version. +ACTION_NAME="wporg-svn update-trunk" +# shellcheck source=SCRIPTDIR/../lib/svn.sh +. "${BASH_SOURCE[0]%/*}/../lib/svn.sh" + +require_tools svn svnmucc python3 +validate_slug +validate_version +validate_expected_revision +validate_credentials +prepare + +STAGE="reading trunk and tags/$VERSION" +snapshot +R=$REPLY +note_revision "snapshot r$R" +svn_kind "$BASE" trunk "$R" +[ "$REPLY" = dir ] || fail "trunk does not exist at r$R" +svn_kind "$BASE/tags" "$VERSION" "$R" +[ "$REPLY" = dir ] || fail "tags/$VERSION does not exist at r$R" +check_expected_revision "$R" +svn_kind "$BASE/trunk" readme.txt "$R" +[ "$REPLY" = file ] || fail "trunk/readme.txt is not a file at r$R" +svn_cat "$BASE/trunk/readme.txt" "$R" "$WORK/trunk-readme.txt" +[ "$(python3 "$LIB_DIR/release.py" stable-tag "$WORK/trunk-readme.txt")" = "$VERSION" ] || + fail "the trunk/readme.txt Stable Tag is not $VERSION at r$R; run set-stable for $VERSION first" + +# trunk and the tag hold the same tree, files and properties, at REV? +same_tree() { + local status=0 + svn_read diff --summarize --xml --old="$BASE/tags/$VERSION@$R" --new="$BASE/trunk@$1" + python3 "$LIB_DIR/svnxml.py" diff-empty <"$WORK/out" 2>/dev/null || status=$? + [ "$status" -le 1 ] || fail "could not read svn's diff output" + return "$status" +} + +if same_tree "$R"; then + output version "$VERSION" + output revision "$R" + summary "### $ACTION_NAME: verified no-op" "" "trunk already matches tags/$VERSION at r$R. Nothing was committed." + exit 0 +fi + +STAGE="replacing trunk with tags/$VERSION@$R" +if ! mucc_write -r "$R" -m "Release $VERSION: sync trunk with tags/$VERSION" \ + rm "$BASE/trunk" cp "$R" "$BASE/tags/$VERSION" "$BASE/trunk"; then + svn_error + ERROR=$REPLY + STAGE="inspecting trunk after a failed write" + snapshot + HEAD=$REPLY + if same_tree "$HEAD"; then + STAGE="replacing trunk with tags/$VERSION@$R" + fail "the write reported an error ($ERROR), but trunk now matches tags/$VERSION. Rerunning is safe: it verifies and reports a no-op." + fi + STAGE="replacing trunk with tags/$VERSION@$R" + fail "the write failed ($ERROR) and trunk does not match tags/$VERSION at r$HEAD; check who else is committing (svn log) before rerunning." +fi +N=$REPLY +note_revision "commit r$N" + +STAGE="verifying trunk at r$N" +same_tree "$N" || fail "trunk at r$N does not match tags/$VERSION@$R" + +output version "$VERSION" +output revision "$N" +summary "### $ACTION_NAME: $SLUG $VERSION" "" "trunk is now a copy of tags/$VERSION@$R (r$N)." diff --git a/.github/workflows/wporg-svn.yml b/.github/workflows/wporg-svn.yml new file mode 100644 index 0000000..54346a2 --- /dev/null +++ b/.github/workflows/wporg-svn.yml @@ -0,0 +1,98 @@ +# Self-checks for the wporg-svn actions. Nothing here touches wordpress.org: +# every SVN write goes to a throwaway file:// repository, behind the test-only +# write guard (tests/wporg-svn/guard). +name: wporg-svn actions + +on: + pull_request: + paths: + - .github/actions/wporg-svn/** + - .github/workflows/wporg-svn.yml + - tests/wporg-svn/** + push: + branches: [main] + paths: + - .github/actions/wporg-svn/** + - .github/workflows/wporg-svn.yml + - tests/wporg-svn/** + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # Locally: shellcheck -x .github/actions/wporg-svn/*/*.sh .github/actions/wporg-svn/lib/svn.sh tests/wporg-svn/guard/bin/* + - name: shellcheck + run: shellcheck -x .github/actions/wporg-svn/*/*.sh .github/actions/wporg-svn/lib/svn.sh tests/wporg-svn/guard/bin/* + # Locally: actionlint && actionlint .github/actions/wporg-svn/examples/release.yml + - name: actionlint + run: | + curl -fsSL -o "$RUNNER_TEMP/actionlint.tar.gz" \ + https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz + echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $RUNNER_TEMP/actionlint.tar.gz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/actionlint.tar.gz" -C "$RUNNER_TEMP" actionlint + "$RUNNER_TEMP/actionlint" + "$RUNNER_TEMP/actionlint" .github/actions/wporg-svn/examples/release.yml + + test: + strategy: + fail-fast: false + matrix: + runner: [ubuntu-22.04, ubuntu-24.04] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + # Locally: python3 -m unittest discover -s tests/wporg-svn -v + - name: Unit and integration tests + run: python3 -m unittest discover -s tests/wporg-svn -v + + # Runs the three composite actions the way a caller would, against the local fixture. + end-to-end: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install Subversion + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends subversion + - name: Start the fixture + run: | + echo "$PWD/tests/wporg-svn/guard/bin" >> "$GITHUB_PATH" + python3 tests/wporg-svn/fixture.py up "$RUNNER_TEMP/fixture" & + for _ in $(seq 60); do [ -f "$RUNNER_TEMP/fixture/env" ] && break; sleep 1; done + cat "$RUNNER_TEMP/fixture/env" >> "$GITHUB_ENV" + - id: tag + uses: ./.github/actions/wporg-svn/tag + with: + plugin-slug: fixture-plugin + zip-url: ${{ env.FIXTURE_ZIP_URL }} + wporg-username: fixture-user + wporg-password: fixture-password + - uses: ./.github/actions/wporg-svn/set-stable + with: + plugin-slug: fixture-plugin + version: ${{ steps.tag.outputs.version }} + expected-revision: ${{ steps.tag.outputs.revision }} + wporg-username: fixture-user + wporg-password: fixture-password + - uses: ./.github/actions/wporg-svn/update-trunk + with: + plugin-slug: fixture-plugin + version: ${{ steps.tag.outputs.version }} + expected-revision: ${{ steps.tag.outputs.revision }} + wporg-username: fixture-user + wporg-password: fixture-password + - name: Check the end state + env: + VERSION: ${{ steps.tag.outputs.version }} + run: python3 tests/wporg-svn/fixture.py check "$RUNNER_TEMP/fixture" "$VERSION" diff --git a/.gitignore b/.gitignore index 04f3b70..f688282 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea/ +__pycache__/ node_modules/ # No lockfile is committed. Each install resolves the newest versions the ranges in package.json diff --git a/tests/wporg-svn/fixture.py b/tests/wporg-svn/fixture.py new file mode 100644 index 0000000..b9fed88 --- /dev/null +++ b/tests/wporg-svn/fixture.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""The end-to-end fixture for CI: a file:// plugin repository at release 1.0, +and a local HTTPS host serving the 1.1 ZIP. Nothing here talks to wordpress.org. + +Usage: + fixture.py up DIR create the fixture in DIR, write DIR/env (KEY=value + lines for $GITHUB_ENV), then serve until killed + fixture.py check DIR VERSION exit 0 when tags/VERSION and trunk both match the + artifact and trunk's Stable Tag is VERSION +""" + +import os +import sys +import time + +import support +from test_tag import NEXT, STABLE + + +def up(folder): + home = os.path.join(folder, "home") + os.makedirs(home) + repo = support.Repo(folder, support.base_env(home)) + repo.seed("1.0", STABLE) + https = support.Https() + https.routes["/release.zip"] = support.make_zip(NEXT, wrapper="fixture-plugin") + with open(os.path.join(folder, "env.tmp"), "w") as handle: + handle.write(f"WPORG_SVN_TEST_ROOT={repo.root}\n" + f"CURL_CA_BUNDLE={https.ca}\n" + f"FIXTURE_ZIP_URL={https.url}/release.zip?sig=fixture\n") + os.rename(os.path.join(folder, "env.tmp"), os.path.join(folder, "env")) + while True: + time.sleep(3600) + + +def check(folder, version): + repo = support.Repo(folder, support.base_env(os.path.join(folder, "home")), create=False) + expected = support.expected_tree(NEXT) + problems = [path for path in (f"tags/{version}", "trunk") if repo.tree(path) != expected] + if b"Stable tag: " + version.encode() not in repo.cat("trunk/readme.txt"): + problems.append("trunk/readme.txt Stable Tag") + print("end state ok" if not problems else "end state wrong: " + ", ".join(problems)) + return 1 if problems else 0 + + +if __name__ == "__main__": + if sys.argv[1:2] == ["up"] and len(sys.argv) == 3: + up(sys.argv[2]) + elif sys.argv[1:2] == ["check"] and len(sys.argv) == 4: + sys.exit(check(sys.argv[2], sys.argv[3])) + else: + sys.exit(__doc__) diff --git a/tests/wporg-svn/guard/bin/svn b/tests/wporg-svn/guard/bin/svn new file mode 100755 index 0000000..572dc90 --- /dev/null +++ b/tests/wporg-svn/guard/bin/svn @@ -0,0 +1,3 @@ +#!/bin/sh +# Test-only: route svn through the write guard. +exec python3 "$(dirname "$0")/../write_guard.py" svn "$@" diff --git a/tests/wporg-svn/guard/bin/svnmucc b/tests/wporg-svn/guard/bin/svnmucc new file mode 100755 index 0000000..e97b3ea --- /dev/null +++ b/tests/wporg-svn/guard/bin/svnmucc @@ -0,0 +1,3 @@ +#!/bin/sh +# Test-only: route svnmucc through the write guard. +exec python3 "$(dirname "$0")/../write_guard.py" svnmucc "$@" diff --git a/tests/wporg-svn/guard/bin/svnrdump b/tests/wporg-svn/guard/bin/svnrdump new file mode 100755 index 0000000..b61ca5f --- /dev/null +++ b/tests/wporg-svn/guard/bin/svnrdump @@ -0,0 +1,3 @@ +#!/bin/sh +# Test-only: route svnrdump through the write guard. +exec python3 "$(dirname "$0")/../write_guard.py" svnrdump "$@" diff --git a/tests/wporg-svn/guard/write_guard.py b/tests/wporg-svn/guard/write_guard.py new file mode 100644 index 0000000..3a6c230 --- /dev/null +++ b/tests/wporg-svn/guard/write_guard.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Refuse SVN writes to anything but file:// repositories. + +Test-only. The bin/ shims put this in front of the real svn, svnmucc and +svnrdump, so no test, fixture or QA run can write to a production repository, +even by mistake. +Reads pass straight through. The rule is deliberately conservative: anything +that looks like a write is checked, and a false refusal only fails loudly. + +Usage: write_guard.py svn|svnmucc|svnrdump ARGS... +With WRITE_GUARD_LOG set, every argv is appended to that file as a JSON line. +""" + +import json +import os +import re +import subprocess +import sys + +BIN = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bin") +URL = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*)://") + +# svn subcommands that commit, or that commit when given a URL. +SVN_WRITES = { + "commit", "ci", "import", "mkdir", "copy", "cp", "move", "mv", "rename", "ren", "delete", "del", + "remove", "rm", "propset", "pset", "ps", "propdel", "pdel", "pd", "propedit", "pedit", "pe", + "lock", "unlock", +} + + +def is_remote(value): + match = URL.match(value) + return bool(match) and match.group(1).lower() != "file" + + +def real(tool): + """The tool from the PATH entries after the guard's own (wrappers come before it).""" + guard = os.path.realpath(BIN) + path = [d for d in os.environ.get("PATH", "").split(os.pathsep) if d] + resolved = [os.path.realpath(d) for d in path] + if guard in resolved: + path = path[resolved.index(guard) + 1:] + for directory in path: + if os.path.realpath(directory) == guard: + continue + candidate = os.path.join(directory, tool) + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + sys.exit(f"write-guard: no real {tool} on PATH") + + +def repo_root(path): + result = subprocess.run([real("svn"), "info", "--non-interactive", "--show-item", "repos-root-url", "--", path + "@"], + capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else None + + +def refusal(tool, args, root_of=repo_root): + """Why this command must not run, or None when it is a read or a local write.""" + if tool == "svnmucc": + urls = [arg for arg in args if URL.match(arg)] + if not urls: + return "svnmucc without an explicit file:// URL" + if any(is_remote(url) for url in urls): + return "svnmucc to a non-file:// repository" + return None + + if tool == "svnrdump": + if "load" not in args: + return None + urls = [arg for arg in args if URL.match(arg)] + if not urls or any(is_remote(url) for url in urls): + return "svnrdump load into anything but a file:// repository" + return None + + if not SVN_WRITES.intersection(args) and "--revprop" not in args: + return None + if any(is_remote(arg) for arg in args): + return "svn write to a non-file:// repository" + paths = [arg for arg in args if not arg.startswith("-") and os.path.exists(arg.rsplit("@", 1)[0])] + for path in paths + ["."]: + root = root_of(path) + if root and is_remote(root): + return "svn write from a working copy of a non-file:// repository" + return None + + +def main(): + tool, args = sys.argv[1], sys.argv[2:] + if os.environ.get("WRITE_GUARD_LOG"): + with open(os.environ["WRITE_GUARD_LOG"], "a") as log: + log.write(json.dumps([tool] + args) + "\n") + reason = refusal(tool, args) + if reason: + print(f"write-guard: refused: {reason}", file=sys.stderr) + sys.exit(99) + os.execv(real(tool), [tool] + args) + + +if __name__ == "__main__": + main() diff --git a/tests/wporg-svn/live_reads.sh b/tests/wporg-svn/live_reads.sh new file mode 100755 index 0000000..8e785fc --- /dev/null +++ b/tests/wporg-svn/live_reads.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Test-only and read-only: run the actions' read helpers against a real +# wordpress.org plugin and print what they saw. It never calls a write helper, +# and the tests run it with the write guard first on PATH. +# Usage: live_reads.sh SLUG +ACTION_NAME="wporg-svn live reads" +# shellcheck source=SCRIPTDIR/../../.github/actions/wporg-svn/lib/svn.sh +. "${BASH_SOURCE[0]%/*}/../../.github/actions/wporg-svn/lib/svn.sh" + +[ -z "${WPORG_SVN_TEST_ROOT:-}" ] || fail "live reads go to wordpress.org: unset WPORG_SVN_TEST_ROOT" +WPORG_SLUG=${1:-} +validate_slug +prepare + +STAGE="reading $SLUG" +snapshot +R=$REPLY +echo "revision=$R" +svn_kind "$BASE/trunk" readme.txt "$R" +echo "trunk-readme=$REPLY" +svn_cat "$BASE/trunk/readme.txt" "$R" "$WORK/readme.txt" +S=$(python3 "$LIB_DIR/release.py" stable-tag "$WORK/readme.txt") +python3 "$LIB_DIR/release.py" version "$S" || fail "the Stable Tag is not numeric" +echo "stable=$S" +svn_kind "$BASE/tags" "$S" "$R" +echo "stable-kind=$REPLY" +svn_kind "$BASE/tags" 0.0.0.0.0.1 "$R" +echo "probe-kind=$REPLY" +svn_names "$BASE/tags" "$R" +echo "tags=$(printf '%s\n' "$REPLY" | python3 -c 'import sys; print(",".join(l.strip() for l in sys.stdin))')" +last_changed "$BASE/tags/$S" "$R" +echo "stable-last-changed=$REPLY" diff --git a/tests/wporg-svn/support.py b/tests/wporg-svn/support.py new file mode 100644 index 0000000..5a7fa9b --- /dev/null +++ b/tests/wporg-svn/support.py @@ -0,0 +1,412 @@ +"""Shared paths, fixtures and the action runner for the wporg-svn tests. + +Every SVN write here goes to a throwaway file:// repository, and the write +guard (guard/bin) sits first on PATH for every subprocess, hooks included. +""" + +import atexit +import http.server +import io +import json +import os +import pathlib +import re +import shutil +import ssl +import struct +import subprocess +import sys +import tempfile +import threading +import unittest +import xml.etree.ElementTree as ET +import zipfile +import zlib + +TESTS = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(os.path.dirname(TESTS)) +ACTIONS = os.path.join(REPO, ".github", "actions", "wporg-svn") +LIB = os.path.join(ACTIONS, "lib") +GUARD_BIN = os.path.join(TESTS, "guard", "bin") + +sys.path.insert(0, LIB) + +SLUG = "fixture-plugin" +USERNAME = "fixture-user" +# Sentinels: neither may ever appear in anything an action prints or writes. +PASSWORD = "sentinel-password-7c1e" +SIGNATURE = "sentinel-signature-93ab" + +README = """=== Fixture Plugin === +Contributors: fixture +Requires at least: 6.0 +Tested up to: 6.6 +Stable tag: {stable} +License: GPLv2 + +A fixture. + +== Changelog == + += {version} = +* Release {version}. +""" + +PHP = """ hold the same release; assets holds a banner.""" + self.put_tree("trunk-new", files or plugin(stable)) + self.mucc("rm", self.url("trunk"), "mv", self.url("trunk-new"), self.url("trunk")) + self.mucc("put", write(self.staging, "banner.png", b"\x89PNG banner"), self.url("assets/banner.png")) + return self.mucc("cp", "HEAD", self.url("trunk"), self.url(f"tags/{stable}")) + + def tree(self, path, rev="HEAD"): + dest = tempfile.mktemp(dir=self.staging) + self.svn("export", "-q", "--ignore-externals", "-r", str(rev), f"{self.url(path)}@{rev}", dest) + try: + return read_tree(dest) + finally: + shutil.rmtree(dest) + + def cat(self, path, rev="HEAD"): + return self.svn("cat", "-r", str(rev), f"{self.url(path)}@{rev}") + + def last_changed(self, path, rev="HEAD"): + return int(self.svn("info", "--show-item", "last-changed-revision", f"{self.url(path)}@{rev}").decode()) + + def exists(self, path, rev="HEAD"): + return self.svn("info", f"{self.url(path)}@{rev}", check=False) != b"" + + def props(self, path, rev="HEAD"): + """{relative path: {name: value}} for every node under path.""" + target = self.url(path) + doc = ET.fromstring(self.svn("proplist", "-R", "-v", "--xml", "-r", str(rev), f"{target}@{rev}")) + found = {} + for node in doc.iter("target"): + rel = node.get("path")[len(target):].lstrip("/") + found[rel] = {p.get("name"): p.text or "" for p in node.iter("property")} + return found + + def changed(self, rev): + """Sorted (action, path) pairs committed in rev.""" + doc = ET.fromstring(self.svn("log", "-v", "--xml", "-r", str(rev), self.root)) + return sorted((p.get("action"), p.text) for p in doc.iter("path")) + + def hook(self, name, body): + """Install a repository hook; hooks run with an empty environment.""" + exports = "".join(f"export {k}='{self.env[k]}'\n" for k in ("PATH", "HOME", "LC_ALL")) + path = os.path.join(self.dir, "hooks", name) + with open(path, "w") as handle: + handle.write("#!/bin/sh\n" + exports + body) + os.chmod(path, 0o755) + + +_CERT = {} + + +def certificate(): + """A self-signed certificate for 127.0.0.1, created once per test run.""" + if not _CERT: + folder = tempfile.mkdtemp() + atexit.register(shutil.rmtree, folder, True) + cert, key = os.path.join(folder, "cert.pem"), os.path.join(folder, "key.pem") + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", key, "-out", cert, + "-days", "2", "-subj", "/CN=127.0.0.1", "-addext", "subjectAltName=IP:127.0.0.1"], + check=True, capture_output=True) + _CERT.update(cert=cert, key=key) + return _CERT["cert"], _CERT["key"] + + +class Https: + """A local HTTPS server. routes: path -> bytes, or callable(handler).""" + + def __init__(self): + self.routes, self.requests = {}, [] + owner = self + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + owner.requests.append({"path": self.path, "headers": dict(self.headers)}) + route = owner.routes.get(self.path.split("?")[0]) + if route is None: + self.send_error(404) + elif callable(route): + route(self) + else: + self.send_response(200) + self.send_header("Content-Length", str(len(route))) + self.end_headers() + self.wfile.write(route) + + def log_message(self, *args): + pass + + cert, key = certificate() + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert, key) + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.server.socket = context.wrap_socket(self.server.socket, server_side=True) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"https://127.0.0.1:{self.server.server_address[1]}" + self.ca = cert + + def close(self): + self.server.shutdown() + self.server.server_close() + + +class Result: + def __init__(self, process, outputs, summary): + self.code = process.returncode + self.stdout, self.stderr = process.stdout, process.stderr + self.log = process.stdout + process.stderr + self.outputs, self.summary = outputs, summary + + +class ActionTest(unittest.TestCase): + """Runs the real action scripts against a throwaway repository.""" + + def setUp(self): + self.tmp = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.home = os.path.join(self.tmp, "home") + self.scratch = os.path.join(self.tmp, "scratch") # the actions' TMPDIR + os.makedirs(self.home) + os.makedirs(self.scratch) + self.env = base_env(self.home) + self.repo = Repo(self.tmp, self.env) + self._https = None + + @property + def https(self): + if self._https is None: + self._https = Https() + self.addCleanup(self._https.close) + return self._https + + def action_env(self, **env): + merged = dict(self.env, WPORG_SVN_TEST_ROOT=self.repo.root, WPORG_SLUG=SLUG, WPORG_USERNAME=USERNAME, + WPORG_PASSWORD=PASSWORD, TMPDIR=self.scratch, + GITHUB_OUTPUT=os.path.join(self.tmp, "github-output"), + GITHUB_STEP_SUMMARY=os.path.join(self.tmp, "github-summary"), + WRITE_GUARD_LOG=os.path.join(self.tmp, "svn-calls.jsonl")) + if self._https: + merged["CURL_CA_BUNDLE"] = self._https.ca + merged.update(env) + return {k: v for k, v in merged.items() if v is not None} + + def wrap(self, tool, body): + """Put a test-only wrapper for tool first on the actions' PATH. + + The body runs under sh; "$NEXT" is the guarded tool it may exec. + """ + folder = os.path.join(self.tmp, "wrappers") + os.makedirs(folder, exist_ok=True) + path = os.path.join(folder, tool) + nxt = os.path.join(GUARD_BIN, tool) if tool in ("svn", "svnmucc", "svnrdump") else shutil.which(tool) + with open(path, "w") as handle: + handle.write(f"#!/bin/sh\nNEXT='{nxt}'\n{body}\n") + os.chmod(path, 0o755) + self.env["PATH"] = folder + os.pathsep + self.env["PATH"].replace(folder + os.pathsep, "") + + def start(self, action, **env): + """Start an action without waiting; it leads its own process group.""" + for name in ("github-output", "github-summary"): + open(os.path.join(self.tmp, name), "w").close() + script = os.path.join(ACTIONS, action, f"{action}.sh") + return subprocess.Popen(["bash", script], env=self.action_env(**env), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, start_new_session=True) + + def finish(self, process, timeout=180): + stdout, stderr = process.communicate(timeout=timeout) + process.stdout, process.stderr = stdout, stderr + with open(os.path.join(self.tmp, "github-output")) as handle: + outputs = dict(line.rstrip("\n").split("=", 1) for line in handle if "=" in line) + with open(os.path.join(self.tmp, "github-summary")) as handle: + summary = handle.read() + result = Result(process, outputs, summary) + + # Invariants for every run, successful or not. + self.assertEqual(os.listdir(self.scratch), [], "the action left temporary files behind") + everything = result.log + summary + repr(outputs) + for secret in (PASSWORD, SIGNATURE): + self.assertNotIn(secret, everything, "a secret leaked into the action's output") + self.assertNotRegex(everything, r"(?m)^::(?!error|warning|notice|add-mask)", + "unexpected workflow command") + return result + + def svn_calls(self): + """Every svn/svnmucc argv the actions ran, in order.""" + path = os.path.join(self.tmp, "svn-calls.jsonl") + if not os.path.exists(path): + return [] + with open(path) as handle: + return [json.loads(line) for line in handle] + + def run_action(self, action, **env): + return self.finish(self.start(action, **env)) + + def assert_success(self, result): + self.assertEqual(result.code, 0, result.log) + + def assert_failure(self, result, pattern): + self.assertNotEqual(result.code, 0, result.log) + self.assertRegex(result.log, pattern) + self.assertEqual(result.outputs, {}, "a failed action must not expose outputs") diff --git a/tests/wporg-svn/test_actions.py b/tests/wporg-svn/test_actions.py new file mode 100644 index 0000000..3e219db --- /dev/null +++ b/tests/wporg-svn/test_actions.py @@ -0,0 +1,103 @@ +"""The action.yml files wire inputs, scripts and outputs together correctly. + +actionlint checks the syntax; this checks the contract with the scripts, which +no linter knows about. The files are read as text: they are ours and simple. +""" + +import os +import re +import subprocess +import unittest + +from support import ACTIONS + +CREDENTIALS = {"wporg-username": "WPORG_USERNAME", "wporg-password": "WPORG_PASSWORD"} +EXPECTED = {"expected-revision": "WPORG_EXPECTED_REVISION"} +OPTIONAL = set(EXPECTED) +ACTIONS_SPEC = { + "tag": ({"plugin-slug": "WPORG_SLUG", "zip-url": "WPORG_ZIP_URL", **CREDENTIALS}, + {"version", "revision", "previous-stable"}), + "set-stable": ({"plugin-slug": "WPORG_SLUG", "version": "WPORG_VERSION", **EXPECTED, **CREDENTIALS}, + {"version", "revision"}), + "update-trunk": ({"plugin-slug": "WPORG_SLUG", "version": "WPORG_VERSION", **EXPECTED, **CREDENTIALS}, + {"version", "revision"}), +} + + +def block(text, key): + """The lines under a top-level key, up to the next top-level key.""" + match = re.search(rf"(?ms)^{key}:\n(.*?)(?=^\S|\Z)", text) + return match.group(1) if match else "" + + +def named(section): + """{name: its indented body} for the two-space-indented keys of a section.""" + parts = re.split(r"(?m)^ ([a-z-]+):\n", section) + return dict(zip(parts[1::2], parts[2::2])) + + +class ActionMetadataTest(unittest.TestCase): + def read(self, *path): + with open(os.path.join(*path)) as handle: + return handle.read() + + def test_each_action_matches_its_script(self): + # The integration tests prove the scripts read exactly these env names. + for action, (inputs, outputs) in ACTIONS_SPEC.items(): + with self.subTest(action=action): + text = self.read(ACTIONS, action, "action.yml") + script = self.read(ACTIONS, action, f"{action}.sh") + declared = named(block(text, "inputs")) + self.assertEqual(set(declared), set(inputs)) + for name, body in declared.items(): + self.assertIn("required: false" if name in OPTIONAL else "required: true", body, name) + + self.assertRegex(text, r"(?m)^ using: composite$") + steps = block(text, "runs") + self.assertEqual(steps.count("shell: bash"), steps.count("run:"), "every step needs shell: bash") + for run in re.findall(r"run: (.*)", steps): + self.assertNotIn("${{", run, "inputs reach scripts through env, never the script text") + self.assertIn(f'run: bash "$GITHUB_ACTION_PATH/{action}.sh"', steps) + + main = steps[steps.index("id: release"):] + for name, variable in inputs.items(): + self.assertIn(f"{variable}: ${{{{ inputs.{name} }}}}", main) + + declared_outputs = named(block(text, "outputs")) + self.assertEqual(set(declared_outputs), outputs) + emitted = set(re.findall(r"^output ([a-z-]+) ", script, re.M)) + self.assertEqual(emitted, outputs) + for name, body in declared_outputs.items(): + self.assertIn(f"value: ${{{{ steps.release.outputs.{name} }}}}", body) + + def test_the_zip_url_and_password_are_masked_first(self): + for action in ACTIONS_SPEC: + with self.subTest(action=action): + steps = block(self.read(ACTIONS, action, "action.yml"), "runs") + self.assertLess(steps.index("::add-mask::"), steps.index("id: release")) + self.assertIn("${{ inputs.wporg-password }}", steps[:steps.index("id: release")]) + if action == "tag": + self.assertIn("${{ inputs.zip-url }}", steps[:steps.index("id: release")]) + + def test_mask_values_are_escaped_for_the_runner(self): + # The runner unescapes %25, %0D and %0A in ::add-mask:: data before it masks + # the value, so a secret containing "%25" would otherwise mask the wrong string. + env = {"PATH": os.environ["PATH"], "ZIP_URL": "https://h/a%25b?sig=x%2F", "PASSWORD": "p%41ss\r"} + for action in ACTIONS_SPEC: + with self.subTest(action=action): + steps = block(self.read(ACTIONS, action, "action.yml"), "runs") + script = re.search(r"run: (.*::add-mask::.*)", steps).group(1) + out = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True, + check=True).stdout + self.assertIn("::add-mask::p%2541ss%0D\n", out) + if action == "tag": + self.assertIn("::add-mask::https://h/a%2525b?sig=x%252F\n", out) + + def test_the_readme_example_is_the_linted_example_file(self): + readme = self.read(ACTIONS, "README.md") + example = self.read(ACTIONS, "examples", "release.yml") + self.assertIn("```yaml\n" + example + "```\n", readme, "README example drifted from examples/release.yml") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_archive.py b/tests/wporg-svn/test_archive.py new file mode 100644 index 0000000..7b54429 --- /dev/null +++ b/tests/wporg-svn/test_archive.py @@ -0,0 +1,346 @@ +"""ZIP inspection, safe extraction and plugin root resolution.""" + +import os +import shutil +import stat +import subprocess +import tempfile +import unittest +import zipfile +from unittest import mock + +import support # noqa: F401 (puts lib/ on sys.path) + +import archive + +PLUGIN = { + "readme.txt": "=== X ===\nStable tag: 1.1\n\nShort.\n", + "fixture.php": " bytes/str, or name -> ZipInfo-tweaking dict.""" + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as zf: + for name, value in entries.items(): + info = zipfile.ZipInfo(name) + info.compress_type = zipfile.ZIP_DEFLATED + data = value + if isinstance(value, dict): + data = value.get("data", b"") + if "mode" in value: + info.external_attr = value["mode"] << 16 + info.create_system = value.get("host", info.create_system) + if name.endswith("/"): + info.external_attr = info.external_attr or (stat.S_IFDIR | 0o755) << 16 + zf.writestr(info, data.encode() if isinstance(data, str) else data) + return path + + +def set_flag_bits(path, bits, on=True): + """Set, or clear, general purpose flag bits in every header.""" + with open(path, "r+b") as handle: + blob = bytearray(handle.read()) + for signature, offset in ((b"PK\x03\x04", 6), (b"PK\x01\x02", 8)): + start = blob.find(signature) + while start != -1: + at = start + offset + flags = int.from_bytes(blob[at:at + 2], "little") + blob[at:at + 2] = (flags | bits if on else flags & ~bits).to_bytes(2, "little") + start = blob.find(signature, start + 4) + handle.seek(0) + handle.write(blob) + + +class ArchiveTest(unittest.TestCase): + def setUp(self): + self.tmp = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp) + self.zip = os.path.join(self.tmp, "plugin.zip") + self.dest = os.path.join(self.tmp, "out") + os.mkdir(self.dest) + + def extract(self, entries): + build_zip(self.zip, entries) + return archive.extract(self.zip, self.dest) + + def assert_rejected(self, entries, message): + with self.assertRaisesRegex(archive.ArchiveError, message): + self.extract(entries) + self.assertEqual(os.listdir(self.dest), [], "nothing may be extracted from a rejected archive") + self.assertEqual(sorted(os.listdir(self.tmp)), ["out", "plugin.zip"], "nothing written outside") + + def read(self, *parts): + with open(os.path.join(*parts), "rb") as handle: + return handle.read() + + def test_flat_archive(self): + entries = dict(PLUGIN) + entries.update({".htaccess": "deny", "sub dir/a@b.txt": "x", "deep/er/empty.txt": "", "assets/": b""}) + root = self.extract(entries) + self.assertEqual(root, self.dest) + self.assertEqual(self.read(root, ".htaccess"), b"deny") + self.assertEqual(self.read(root, "sub dir", "a@b.txt"), b"x") + self.assertEqual(self.read(root, "deep", "er", "empty.txt"), b"") + self.assertTrue(os.path.isdir(os.path.join(root, "assets"))) + + def test_single_wrapper_directory_of_any_name(self): + root = self.extract({"whatever-name/" + k: v for k, v in PLUGIN.items()}) + self.assertEqual(root, os.path.join(self.dest, "whatever-name")) + self.assertEqual(self.read(root, "readme.txt"), PLUGIN["readme.txt"].encode()) + + def test_ignores_documented_packaging_noise(self): + entries = {"w/" + k: v for k, v in PLUGIN.items()} + entries.update({"__MACOSX/w/._readme.txt": "junk", "w/.DS_Store": "junk", ".DS_Store": "junk"}) + root = self.extract(entries) + self.assertEqual(root, os.path.join(self.dest, "w")) + self.assertEqual(sorted(os.listdir(self.dest)), ["w"]) + self.assertNotIn(".DS_Store", os.listdir(root)) + + def test_rejects_multiple_roots(self): + entries = {"a/" + k: v for k, v in PLUGIN.items()} + entries.update({"b/" + k: v for k, v in PLUGIN.items()}) + self.assert_rejected(entries, "plugin root") + + def test_rejects_deep_wrappers(self): + self.assert_rejected({"a/b/" + k: v for k, v in PLUGIN.items()}, "readme.txt") + + def test_rejects_unrelated_siblings_of_a_wrapper(self): + entries = {"w/" + k: v for k, v in PLUGIN.items()} + entries["LICENSE"] = "gpl" + self.assert_rejected(entries, "plugin root") + + def test_rejects_missing_readme(self): + self.assert_rejected({"fixture.php": PLUGIN["fixture.php"]}, "plugin root") + + def test_rejects_readme_directory(self): + self.assert_rejected({"readme.txt/x": "x", "fixture.php": "x"}, "plugin root") + + def test_rejects_traversal_and_absolute_paths(self): + for name in ("../evil.php", "a/../../evil.php", "/abs/evil.php", "C:/evil.php", "a\\..\\evil.php", + "a/./b.php", "a//b.php", "./b.php"): + with self.subTest(name=name): + entries = dict(PLUGIN) + entries[name] = "x" + self.assert_rejected(entries, "unsafe path") + + def test_rejects_control_characters(self): + for name in ("a\nb.txt", "a\x7fb.txt", "a\x1bb.txt", "a\x85b.txt"): + with self.subTest(name=repr(name)): + entries = dict(PLUGIN) + entries[name] = "x" + self.assert_rejected(entries, "control character") + + def test_rejects_nul_in_names(self): + # zipfile truncates names at NUL when writing, so patch the raw bytes. + entries = dict(PLUGIN) + entries["evil.php#.txt"] = "x" + build_zip(self.zip, entries) + with open(self.zip, "r+b") as handle: + blob = handle.read().replace(b"evil.php#.txt", b"evil.php\0.txt") + handle.seek(0) + handle.write(blob) + with self.assertRaisesRegex(archive.ArchiveError, "control character"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_rejects_duplicates_and_ambiguous_spellings(self): + pairs = [ + ("README.TXT", "readme.txt"), + ("Dir/a.txt", "dir/b.txt"), + ("caf\u00e9.txt", "cafe\u0301.txt"), + ("a", "a/b.txt"), + ] + for first, second in pairs: + with self.subTest(first=first, second=second): + entries = dict(PLUGIN) + entries[first] = "1" + entries[second] = "2" + self.assert_rejected(entries, "duplicate|conflict") + + def test_rejects_exact_duplicate_entries(self): + build_zip(self.zip, PLUGIN) + with zipfile.ZipFile(self.zip, "a") as zf, self.assertWarns(UserWarning): + zf.writestr("readme.txt", "again") + with self.assertRaisesRegex(archive.ArchiveError, "duplicate"): + archive.extract(self.zip, self.dest) + + def test_rejects_symlinks_and_special_files(self): + # Host 19 (OS X) stores Unix modes too; Go's archive/zip reads them that way. + for host in (3, 19): + for kind in (stat.S_IFLNK, stat.S_IFIFO, stat.S_IFCHR, stat.S_IFBLK, stat.S_IFSOCK): + with self.subTest(host=host, kind=kind): + entries = dict(PLUGIN) + entries["link"] = {"data": "/etc/passwd", "mode": kind | 0o777, "host": host} + self.assert_rejected(entries, "not a regular file") + + def test_rejects_vcs_metadata(self): + for name in (".svn/entries", "w/.git/config", ".git", "x/.SVN/wc.db"): + with self.subTest(name=name): + entries = dict(PLUGIN) + entries[name] = "x" + self.assert_rejected(entries, "version control") + + def test_keeps_vcs_lookalikes(self): + entries = dict(PLUGIN) + entries.update({".gitignore": "x", ".svnignore": "y", "git/x": "z"}) + root = self.extract(entries) + self.assertEqual(self.read(root, ".gitignore"), b"x") + + def test_rejects_encrypted_entries(self): + build_zip(self.zip, PLUGIN) + set_flag_bits(self.zip, 0x1) + with self.assertRaisesRegex(archive.ArchiveError, "encrypted"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_rejects_malformed_archives(self): + with open(self.zip, "wb") as handle: + handle.write(b"this is not a zip file at all") + with self.assertRaisesRegex(archive.ArchiveError, "malformed"): + archive.extract(self.zip, self.dest) + + def test_rejects_truncated_archives(self): + build_zip(self.zip, PLUGIN) + with open(self.zip, "r+b") as handle: + handle.truncate(os.path.getsize(self.zip) // 2) + with self.assertRaisesRegex(archive.ArchiveError, "malformed"): + archive.extract(self.zip, self.dest) + + def test_rejects_corrupt_content(self): + with zipfile.ZipFile(self.zip, "w", zipfile.ZIP_STORED) as zf: + for name, data in PLUGIN.items(): + zf.writestr(name, data) + with open(self.zip, "r+b") as handle: + blob = bytearray(handle.read()) + at = blob.find(b"Stable tag") + blob[at] ^= 0xFF + handle.seek(0) + handle.write(blob) + with self.assertRaisesRegex(archive.ArchiveError, "malformed"): + archive.extract(self.zip, self.dest) + + def test_unflagged_utf8_names_keep_their_spelling(self): + # What Info-ZIP's zip writes on macOS, and on Ubuntu without en_US.UTF-8. + build_zip(self.zip, {**PLUGIN, "caf\u00e9/na\u00efve.txt": "x", "\u65e5\u672c.txt": "y"}) + set_flag_bits(self.zip, 0x800, on=False) + with zipfile.ZipFile(self.zip) as zf: + self.assertFalse(any(info.flag_bits & 0x800 for info in zf.infolist())) + root = archive.extract(self.zip, self.dest) + self.assertEqual(self.read(root, "caf\u00e9", "na\u00efve.txt"), b"x") + self.assertEqual(self.read(root, "\u65e5\u672c.txt"), b"y") + + def test_utf8_names_from_info_zip_keep_their_spelling(self): + # Flagged or not: Ubuntu's zip sets the UTF-8 flag once en_US.UTF-8 is installed. + if not shutil.which("zip"): + self.skipTest("Info-ZIP zip is not installed") + source = os.path.join(self.tmp, "src") + for name, data in {**PLUGIN, "caf\u00e9/na\u00efve.txt": "x", "\u65e5\u672c.txt": "y"}.items(): + support.write(os.path.join(source, "plugin"), name, data) + subprocess.run(["zip", "-qr", self.zip, "plugin"], cwd=source, check=True) + root = archive.extract(self.zip, self.dest) + self.assertEqual(self.read(root, "caf\u00e9", "na\u00efve.txt"), b"x") + self.assertEqual(self.read(root, "\u65e5\u672c.txt"), b"y") + + def test_flagged_utf8_names_keep_their_spelling(self): + root = self.extract({**PLUGIN, "caf\u00e9.php": "x"}) + self.assertEqual(self.read(root, "caf\u00e9.php"), b"x") + + def unicode_path_zip(self, header_raw, *unicode_names, **field): + build_zip(self.zip, PLUGIN) + support.add_unicode_path_entry(self.zip, header_raw, *unicode_names, **field) + + def test_a_unicode_path_field_that_agrees_is_accepted(self): + self.unicode_path_zip("caf\u00e9.php".encode(), "caf\u00e9.php") + root = archive.extract(self.zip, self.dest) + self.assertEqual(self.read(root, "caf\u00e9.php"), b"z") + + def test_a_unicode_path_field_that_disagrees_is_rejected(self): + # On every Python: zipfile before 3.12 ignores the field. + for header in ("caf\u00e9.php", "plain.php"): + with self.subTest(header=header): + self.unicode_path_zip(header.encode(), "other.php") + with self.assertRaisesRegex(archive.ArchiveError, "two different names"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_every_unicode_path_field_must_agree(self): + # unzip and zipfile 3.12+ take the last valid field: an agreeing one must + # not hide a later one that disagrees, nor the reverse. + for names in (("caf\u00e9.php", "other.php"), ("other.php", "caf\u00e9.php")): + with self.subTest(names=names): + shutil.rmtree(self.dest) + os.mkdir(self.dest) + self.unicode_path_zip("caf\u00e9.php".encode(), *names) + with self.assertRaisesRegex(archive.ArchiveError, "two different names"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_repeated_agreeing_unicode_path_fields_are_accepted(self): + self.unicode_path_zip("caf\u00e9.php".encode(), "caf\u00e9.php", "caf\u00e9.php") + root = archive.extract(self.zip, self.dest) + self.assertEqual(self.read(root, "caf\u00e9.php"), b"z") + + def test_a_stale_unicode_path_field_is_ignored(self): + # Its CRC no longer matches the header name, which a later tool changed. + self.unicode_path_zip("caf\u00e9.php".encode(), "other.php", crc=0) + root = archive.extract(self.zip, self.dest) + self.assertEqual(sorted(os.listdir(root)), sorted(["caf\u00e9.php", *PLUGIN])) + self.assertEqual(self.read(root, "caf\u00e9.php"), b"z") + + def test_names_that_are_not_utf8_are_rejected(self): + # A legacy tool's CP437 name: its e-acute is byte 0x82, which is not UTF-8. + build_zip(self.zip, {**PLUGIN, "cafX.txt": "x"}) + support.patch_name(self.zip, "cafX.txt", b"caf\x82.txt") + with self.assertRaisesRegex(archive.ArchiveError, "not UTF-8"): + archive.extract(self.zip, self.dest) + self.assertEqual(os.listdir(self.dest), []) + + def test_path_depth_and_length_limits(self): + root = self.extract({**PLUGIN, "d/" * 63 + "f.txt": "x"}) + self.assertEqual(self.read(root, *["d"] * 63, "f.txt"), b"x") + for name in ("d/" * 64 + "f.txt", "d/" * 4000 + "f.txt", "/".join(["a" * 200] * 6)): + with self.subTest(levels=name.count("/") + 1, length=len(name)): + shutil.rmtree(self.dest) + os.mkdir(self.dest) + self.assert_rejected({**PLUGIN, name: "x"}, "deeper than 64 levels|longer than 1024 bytes") + + def test_implicit_parent_directories_count_toward_the_limit(self): + # Three entries, but nine files and directories once the parents exist. + entries = {**PLUGIN, "b/a/a/a/a/a/x.txt": ""} + with mock.patch.object(archive, "MAX_ENTRIES", 8): + self.assert_rejected(entries, "more than 8 files and directories") + with mock.patch.object(archive, "MAX_ENTRIES", 9): + self.extract(entries) + + def test_shared_parent_directories_count_once(self): + with mock.patch.object(archive, "MAX_ENTRIES", 5): + root = self.extract({**PLUGIN, "d/a.txt": "1", "d/b.txt": "2"}) + self.assertEqual(self.read(root, "d", "b.txt"), b"2") + + def test_entry_count_limit(self): + entries = dict(PLUGIN) + entries.update({f"f{i}.txt": "x" for i in range(5)}) + with mock.patch.object(archive, "MAX_ENTRIES", 6): + self.assert_rejected(entries, "entries") + + def test_expanded_size_limit(self): + entries = dict(PLUGIN) + entries["big.bin"] = os.urandom(4096) + with mock.patch.object(archive, "MAX_EXPANDED_BYTES", 4096): + self.assert_rejected(entries, "expands") + + def test_expansion_ratio_limit(self): + entries = dict(PLUGIN) + entries["zeros.bin"] = b"\0" * (1024 * 1024) + self.assert_rejected(entries, "ratio") + + def test_archive_size_limit(self): + build_zip(self.zip, PLUGIN) + with mock.patch.object(archive, "MAX_ARCHIVE_BYTES", 10): + with self.assertRaisesRegex(archive.ArchiveError, "larger"): + archive.extract(self.zip, self.dest) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_live.py b/tests/wporg-svn/test_live.py new file mode 100644 index 0000000..3c78eae --- /dev/null +++ b/tests/wporg-svn/test_live.py @@ -0,0 +1,159 @@ +"""Read-only checks against real wordpress.org plugins. Opt-in: WPORG_LIVE=1. + +Nothing here writes to wordpress.org. The read helpers never write, every write +in the mirror test goes to a local file:// repository, and the write guard is +first on PATH for all of it. + + WPORG_LIVE=1 python3 -m unittest discover -s tests/wporg-svn -p test_live.py -v + WPORG_LIVE_PLUGINS plugins for the read checks (comma-separated) + WPORG_LIVE_MIRROR plugins for the mirror release (comma-separated) +""" + +import os +import shutil +import subprocess +import tempfile +import time +import unittest + +from support import TESTS, ActionTest, Repo, base_env, read_tree # first: puts lib/ on sys.path + +import archive # noqa: E402 +import release # noqa: E402 + +LIVE = os.environ.get("WPORG_LIVE") == "1" +PLUGINS = os.environ.get("WPORG_LIVE_PLUGINS", "the-events-calendar,event-tickets,kadence-blocks,give").split(",") +MIRROR = os.environ.get("WPORG_LIVE_MIRROR", "image-widget,advanced-post-manager,the-events-calendar").split(",") +SVN_ROOT = "https://plugins.svn.wordpress.org" + + +def zip_url(slug, version): + return f"https://downloads.wordpress.org/plugin/{slug}.{version}.zip" + + +@unittest.skipUnless(LIVE, "set WPORG_LIVE=1 to read from wordpress.org") +class LiveReadTest(unittest.TestCase): + def setUp(self): + self.tmp = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.env = dict(base_env(self.tmp), TMPDIR=self.tmp) + + cache = {} + + def read(self, slug): + """Run the read harness once per plugin.""" + if slug not in self.cache: + started = time.time() + result = subprocess.run(["bash", os.path.join(TESTS, "live_reads.sh"), slug], env=self.env, + capture_output=True, text=True, timeout=1200) + seen = dict(line.split("=", 1) for line in result.stdout.splitlines() if "=" in line) + self.cache[slug] = (result, seen, time.time() - started) + return self.cache[slug][:2] + + def test_read_helpers_against_real_plugins(self): + for slug in PLUGINS: + with self.subTest(slug=slug): + result, seen = self.read(slug) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertTrue(seen["revision"].isdigit()) + self.assertEqual(seen["trunk-readme"], "file") + self.assertEqual(seen["stable-kind"], "dir") + self.assertEqual(seen["probe-kind"], "absent") + self.assertIn(seen["stable"], seen["tags"].split(",")) + print(f"\n {slug}: r{seen['revision']} stable {seen['stable']} " + f"({len(seen['tags'].split(','))} tags), read helpers took {self.cache[slug][2]:.0f}s") + + def test_a_missing_plugin_is_an_error_not_an_absence(self): + result, seen = self.read("zz-no-such-plugin-0000") + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("revision", seen) + self.assertRegex(result.stdout, r"SVN read failed while reading zz-no-such-plugin-0000: \w") + print("\n " + [line for line in result.stdout.splitlines() if "SVN read failed" in line][0]) + + def test_parsers_on_real_release_zips(self): + for slug in PLUGINS: + with self.subTest(slug=slug): + _, seen = self.read(slug) + stable, folder = seen["stable"], tempfile.mkdtemp(dir=self.tmp) + path = os.path.join(folder, "release.zip") + subprocess.run(["curl", "-q", "-fsSL", "--proto", "=https", "-o", path, zip_url(slug, stable)], + check=True, timeout=600) + os.mkdir(os.path.join(folder, "out")) + root = archive.extract(path, os.path.join(folder, "out")) + self.assertEqual(release.plugin_version(root), stable) + with open(os.path.join(root, "readme.txt"), "rb") as handle: + self.assertEqual(release.stable_tag(handle.read()), stable) + files = sum(len(names) for _, _, names in os.walk(root)) + print(f"\n {slug} {stable}: {os.path.getsize(path)} bytes zipped, {files} files, " + f"wrapper {os.path.basename(root)!r}") + + def test_the_guard_refuses_a_commit_from_a_real_working_copy(self): + wc = os.path.join(self.tmp, "wc") + subprocess.run(["svn", "checkout", "--non-interactive", "--depth", "empty", + f"{SVN_ROOT}/{PLUGINS[0]}/trunk", wc], env=self.env, check=True, capture_output=True) + # Even if the guard failed, an unmodified checkout has nothing to commit. + result = subprocess.run(["svn", "commit", "--non-interactive", "-m", "never", wc], env=self.env, + capture_output=True, text=True) + self.assertEqual(result.returncode, 99, result.stderr) + self.assertIn("write-guard: refused", result.stderr) + + +@unittest.skipUnless(LIVE, "set WPORG_LIVE=1 to read from wordpress.org") +class LiveMirrorReleaseTest(ActionTest): + """QA on real releases: real artifacts in, every write to a local repository.""" + + def real(self, *args): + return subprocess.run(["svn", "--non-interactive", *args], env=self.env, check=True, capture_output=True, + timeout=1800).stdout + + def test_real_releases_through_all_three_actions(self): + for slug in MIRROR: + with self.subTest(slug=slug): + self.mirror(slug) + + def mirror(self, slug): + readme = self.real("cat", f"{SVN_ROOT}/{slug}/trunk/readme.txt") + version = release.stable_tag(readme) + names = self.real("list", f"{SVN_ROOT}/{slug}/tags/").decode().replace("/", "").split() + older = [n for n in names if release.is_version(n) and release.compare(n, version) < 0] + previous = max(older, key=lambda n: [int(p) for p in n.split(".")]) + started = time.time() + + # Seed a local repository with the real previous release (reads from wordpress.org). + work = tempfile.mkdtemp(dir=self.tmp) + exported = os.path.join(work, "previous") + self.real("export", "-q", "--ignore-externals", f"{SVN_ROOT}/{slug}/tags/{previous}", exported) + with open(os.path.join(exported, "readme.txt"), "rb") as handle: + if release.stable_tag(handle.read()) != previous: + self.skipTest(f"{slug} {previous}: the tag's readme does not name its own version") + local = Repo(work, self.env, slug=slug) + local.svn("import", "-q", "--no-ignore", "--no-auto-props", "-m", "seed", exported, local.url("seed")) + local.mucc("rm", local.url("trunk"), "mv", local.url("seed"), local.url("trunk")) + local.mucc("cp", "HEAD", local.url("trunk"), local.url(f"tags/{previous}")) + + common = {"WPORG_SLUG": slug, "WPORG_SVN_TEST_ROOT": local.root} + tagged = self.run_action("tag", WPORG_ZIP_URL=zip_url(slug, version), **common) + self.assert_success(tagged) + self.assertEqual(tagged.outputs["version"], version) + self.assertEqual(tagged.outputs["previous-stable"], previous) + + # The local tag must match the real one byte for byte (another read). + real_tag = os.path.join(work, "real-tag") + self.real("export", "-q", "--ignore-externals", f"{SVN_ROOT}/{slug}/tags/{version}", real_tag) + local_tag = local.tree(f"tags/{version}") + expected = read_tree(real_tag) + differ = sorted(set(local_tag) ^ set(expected)) + sorted( + k for k in set(local_tag) & set(expected) if local_tag[k] != expected[k]) + self.assertEqual(differ, [], f"local tags/{version} differs from the real one") + + self.assert_success(self.run_action("set-stable", WPORG_VERSION=version, **common)) + self.assert_success(self.run_action("update-trunk", WPORG_VERSION=version, **common)) + self.assertEqual(local.tree("trunk"), local_tag) + self.assertEqual(release.stable_tag(local.cat("trunk/readme.txt")), version) + print(f"\n {slug}: {previous} -> {version}, {len(local_tag)} paths, " + f"{len(local.changed(int(tagged.outputs['revision'])))} changed by the tag commit, " + f"{time.time() - started:.0f}s") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_recovery.py b/tests/wporg-svn/test_recovery.py new file mode 100644 index 0000000..2ddd7c2 --- /dev/null +++ b/tests/wporg-svn/test_recovery.py @@ -0,0 +1,294 @@ +"""Races, failed commits, cancellation, ambiguous writes and secret handling. + +Most failures come from real repository hooks. A few come from PATH wrappers: +those simulate what a file:// repository cannot produce (a lost network +response, an authentication or TLS failure), and each says so. +""" + +import os +import signal +import time +import unittest + +from support import PASSWORD, SIGNATURE, ActionTest, expected_tree, make_zip, plugin +from test_tag import NEXT, STABLE + + +class Base(ActionTest): + def setUp(self): + super().setUp() + self.repo.seed("1.0", STABLE) + self.https.routes["/release.zip"] = make_zip(NEXT) + self.zip_url = f"{self.https.url}/release.zip?sig={SIGNATURE}" + + def tag(self, **env): + env.setdefault("WPORG_ZIP_URL", self.zip_url) + return self.run_action("tag", **env) + + def point_trunk_at(self, stable): + """What set-stable does: trunk's readme names stable.""" + self.repo.mucc("put", self.staged(f"readme-{stable}.txt", plugin(stable)["readme.txt"]), + self.repo.url("trunk/readme.txt")) + + def staged(self, name, text): + path = os.path.join(self.repo.staging, name) + with open(path, "w") as handle: + handle.write(text) + return path + + def on_commit(self, number, command): + """Run command from a start-commit hook, on the number-th commit only.""" + count = os.path.join(self.repo.staging, "commit-count") + self.repo.hook("start-commit", f""" +n=$(cat '{count}' 2>/dev/null || echo 0); n=$((n + 1)); echo $n > '{count}' +if [ "$n" = {number} ]; then {command}; fi +""") + + +class TagCommitTest(Base): + """The tag's only write: tags/ lands complete, or not at all.""" + + def cancel_during(self, hook, marker): + process = self.start("tag", WPORG_ZIP_URL=self.zip_url) + deadline = time.time() + 120 + while not os.path.exists(marker) and time.time() < deadline: + time.sleep(0.2) + self.assertTrue(os.path.exists(marker), f"the {hook} hook never ran") + os.killpg(process.pid, signal.SIGTERM) # what a runner does when the job is cancelled + result = self.finish(process, timeout=60) # also checks temp cleanup and secrets + self.assertEqual(result.code, 143, result.log) + self.assertEqual(result.outputs, {}) + return result + + def test_a_rejected_commit_writes_nothing_and_a_rerun_succeeds(self): + head = self.repo.head() + self.repo.hook("pre-commit", "echo 'rejected by policy' >&2; exit 1\n") + result = self.tag() + self.assert_failure(result, r"the commit failed \(.*\) and tags/1\.1 does not exist at r\d+, " + r"so nothing was written") + self.assertRegex(result.log, r"Known revisions: snapshot r\d+, recheck r\d+\.") + self.assertEqual(self.repo.head(), head) + self.repo.hook("pre-commit", "exit 0\n") + self.assert_success(self.tag()) + self.assertEqual(self.repo.tree("tags/1.1"), expected_tree(NEXT)) + + def test_cancellation_during_the_commit_leaves_no_tag(self): + marker = os.path.join(self.repo.staging, "committing") + self.repo.hook("pre-commit", f"touch '{marker}'; sleep 30\n") + result = self.cancel_during("pre-commit", marker) + self.assertRegex(result.log, r"Failed while committing tags/1\.1") + self.assertRegex(result.log, r"The commit's outcome is unknown: if tags/1\.1 exists") + self.assertFalse(self.repo.exists("tags/1.1")) + + def test_cancellation_while_the_commit_lands_still_warns(self): + marker = os.path.join(self.repo.staging, "committed") + self.repo.hook("post-commit", f"touch '{marker}'; sleep 30\n") # the commit is durable now + result = self.cancel_during("post-commit", marker) + self.assertTrue(self.repo.exists("tags/1.1")) + self.assertRegex(result.log, r"if tags/1\.1 exists, do not approve or release it") + + def test_a_failed_verification_reports_what_the_tag_now_holds(self): + # Simulated: a lost connection while exporting the committed tag. + self.wrap("svn", 'case " $* " in *" export "*) echo "svn: E175002: Connection reset" >&2; exit 1;; esac\n' + 'exec "$NEXT" "$@"') + result = self.tag() + self.assert_failure(result, r"Failed while verifying tags/1\.1") + self.assertRegex(result.log, r"tags/1\.1 was created at r\d+ but not verified.*Do not approve or release it") + + +class RaceBeforeTheCommitTest(Base): + def race_during_download(self, *actions): + payload = self.https.routes["/release.zip"] + + def handler(request): + self.repo.mucc(*actions, message="racer") + request.send_response(200) + request.send_header("Content-Length", str(len(payload))) + request.end_headers() + request.wfile.write(payload) + + self.https.routes["/release.zip"] = handler + + def assert_nothing_written(self, result, pattern, head): + self.assert_failure(result, pattern) + self.assertEqual(self.repo.head(), head + 1, "only the competing commit may land") + self.assertFalse(self.repo.exists("tags/1.1")) + + def test_stable_pointer_moved_during_the_download(self): + readme = self.staged("readme.txt", plugin("1.0", stable="0.9")["readme.txt"]) + self.repo.mucc("cp", "HEAD", self.repo.url("tags/1.0"), self.repo.url("tags/0.9")) + head = self.repo.head() + self.race_during_download("put", readme, self.repo.url("trunk/readme.txt")) + self.assert_nothing_written(self.tag(), r"Stable Tag changed after r\d+", head) + + def test_stable_tag_edited_during_the_download(self): + head = self.repo.head() + self.race_during_download("put", self.staged("x.txt", "edited"), self.repo.url("tags/1.0/keep.txt")) + self.assert_nothing_written(self.tag(), r"tags/1\.0 changed after r\d+", head) + + def test_destination_created_during_the_download(self): + head = self.repo.head() + self.race_during_download("mkdir", self.repo.url("tags/1.1")) + result = self.tag() + self.assert_failure(result, r"tags/1\.1 already exists at r\d+") + self.assertEqual(self.repo.head(), head + 1) + + def test_destination_created_at_write_time_is_never_overwritten(self): + self.on_commit(1, f"svnmucc --non-interactive -m racer mkdir '{self.repo.url('tags/1.1')}'") + result = self.tag() + self.assert_failure(result, r"the commit reported an error .* and tags/1\.1 now exists") + self.assertRegex(result.log, "cannot confirm that it created it. Do not approve or release it") + self.assertEqual(self.repo.tree("tags/1.1"), {}, "the other writer's tag stays as it was") + + def test_an_unrelated_new_tag_does_not_block_the_commit(self): + self.on_commit(1, f"svnmucc --non-interactive -m racer mkdir '{self.repo.url('tags/0.5')}'") + result = self.tag() + self.assert_success(result) + self.assertEqual(self.repo.tree("tags/1.1"), expected_tree(NEXT)) + + +class AmbiguousWriteTest(Base): + """A lost response after a successful write. file:// always answers, so a + wrapper runs the real (guarded) write and then reports a timeout.""" + + def lose_the_response(self): + self.wrap("svnmucc", '"$NEXT" "$@" >/dev/null 2>&1\necho "svnmucc: E175012: Connection timed out" >&2\nexit 1') + + def test_tag_commit_with_a_lost_response_is_not_trusted(self): + self.wrap("svn", 'case " $* " in *" commit "*) "$NEXT" "$@" >/dev/null 2>&1\n' + 'echo "svn: E175012: Connection timed out" >&2; exit 1;; esac\nexec "$NEXT" "$@"') + result = self.tag() + self.assert_failure(result, r"the commit reported an error \(could not connect \(E175012\)\) and tags/1\.1 now exists") + self.assertRegex(result.log, "cannot confirm that it created it") + self.assertFalse(any("export" in call for call in self.svn_calls()), "never continue past an unconfirmed write") + self.assertTrue(self.repo.exists("tags/1.1")) + + def test_set_stable_with_a_lost_response_says_a_rerun_is_safe(self): + self.repo.put_tree("tags/1.1", plugin("1.1")) + self.lose_the_response() + result = self.run_action("set-stable", WPORG_VERSION="1.1") + self.assert_failure(result, r"now matches tags/1\.1/readme\.txt\. Rerunning is safe") + self.env["PATH"] = self.env["PATH"].split(os.pathsep, 1)[1] + rerun = self.run_action("set-stable", WPORG_VERSION="1.1") + self.assert_success(rerun) + self.assertRegex(rerun.summary, "no-op") + + def test_update_trunk_with_a_lost_response_says_a_rerun_is_safe(self): + self.repo.put_tree("tags/1.1", plugin("1.1")) + self.point_trunk_at("1.1") + self.lose_the_response() + result = self.run_action("update-trunk", WPORG_VERSION="1.1") + self.assert_failure(result, r"trunk now matches tags/1\.1\. Rerunning is safe") + + +class ErrorClassificationTest(Base): + """Failed reads stop the action; they are never taken to mean "absent".""" + + def test_authentication_failure_on_the_commit(self): + # Simulated: file:// never asks for credentials. + self.wrap("svn", 'case " $* " in *" commit "*) echo "svn: E170001: Authorization failed" >&2; exit 1;; esac\n' + 'exec "$NEXT" "$@"') + head = self.repo.head() + result = self.tag() + self.assert_failure(result, r"the commit failed \(authentication failed \(E170001\)\) and tags/1\.1 does not exist") + self.assertEqual(self.repo.head(), head) + + def test_tls_failure_on_a_read_is_not_absence(self): + # Simulated: file:// has no TLS. + self.wrap("svn", 'case " $* " in *" list "*) echo "svn: E170013: Unable to connect" >&2; ' + 'echo "svn: E230001: Server SSL certificate verification failed" >&2; exit 1;; esac\n' + 'exec "$NEXT" "$@"') + self.repo.put_tree("tags/1.1", plugin("1.1")) + for action, env in (("set-stable", {"WPORG_VERSION": "1.1"}), ("tag", {"WPORG_ZIP_URL": self.zip_url})): + with self.subTest(action=action): + result = self.run_action(action, **env) + self.assert_failure(result, r"SVN read failed while .*TLS certificate verification failed \(E170013 E230001\)") + self.assertNotRegex(result.log, "does not exist") + + def test_connection_failure_on_a_read_is_not_absence(self): + # Simulated: file:// cannot drop a connection. + self.wrap("svn", 'case " $* " in *" list "*) echo "svn: E170013: Unable to connect" >&2; ' + 'echo "svn: E175002: Connection reset" >&2; exit 1;; esac\nexec "$NEXT" "$@"') + result = self.run_action("update-trunk", WPORG_VERSION="1.1") + self.assert_failure(result, r"could not connect \(E170013 E175002\)") + self.assertNotRegex(result.log, "does not exist") + + def test_connection_failure_while_copying_the_stable_tag_stops_the_tag(self): + # Simulated: file:// cannot drop a connection. + self.wrap("svn", 'case " $* " in *" copy "*) echo "svn: E170013: Unable to connect" >&2; ' + 'echo "svn: E175002: Connection reset" >&2; exit 1;; esac\nexec "$NEXT" "$@"') + head = self.repo.head() + result = self.tag() + self.assert_failure(result, r"SVN read failed while copying tags/1\.0 into a working copy: " + r"could not connect \(E170013 E175002\)") + self.assertEqual(self.repo.head(), head) + + def test_permission_failure_is_not_absence(self): + db = os.path.join(self.repo.dir, "db") + os.chmod(db, 0) + self.addCleanup(os.chmod, db, 0o755) + result = self.run_action("set-stable", WPORG_VERSION="1.1") + self.assert_failure(result, r"SVN read failed while reading .*permission denied") + self.assertNotRegex(result.log, "does not exist") + + +class HelperFailureTest(Base): + """A helper that crashes must stop the action, never read as "nothing found".""" + + def test_a_crash_while_looking_for_the_destination_stops_the_tag(self): + self.wrap("python3", 'case " $* " in *" equivalents "*) exit 1;; esac\nexec "$NEXT" "$@"') + head = self.repo.head() + self.assert_failure(self.tag(), r"Failed while checking tags/1\.1") + self.assertEqual(self.repo.head(), head) + + def test_unreadable_diff_output_stops_update_trunk(self): + self.repo.put_tree("tags/1.1", plugin("1.1")) + self.point_trunk_at("1.1") + self.wrap("svn", 'case " $* " in *" diff "*) echo "> \'{dump}\'\nexec "$NEXT" "$@"') + result = self.tag() + self.assert_success(result) + argv = repr(self.svn_calls()) + self.assertIn("--password-from-stdin", argv) + self.assertNotIn(PASSWORD, argv) + with open(dump) as handle: + inherited = PASSWORD in handle.read() # never dump the environment into test output + self.assertFalse(inherited, "child processes must not inherit the password") + + def test_signed_url_is_never_echoed_even_on_failure(self): + self.https.routes["/release.zip"] = make_zip({"readme.txt": "broken"}) + result = self.tag() # finish() asserts the signature appears nowhere + self.assertNotEqual(result.code, 0) + + +class RunnerTest(Base): + def test_unsupported_runner(self): + self.assert_failure(self.tag(RUNNER_OS="Windows"), "unsupported runner") + + def test_missing_tool(self): + path = self.env["PATH"] + for missing in ("svn", "curl"): + with self.subTest(missing=missing): + # A PATH holding every tool except the missing one. + farm = os.path.join(self.tmp, "farm-" + missing) + os.makedirs(farm) + for folder in path.split(os.pathsep): + for name in (os.listdir(folder) if os.path.isdir(folder) else []): + link, target = os.path.join(farm, name), os.path.join(folder, name) + if name != missing and not os.path.lexists(link) and os.access(target, os.X_OK): + os.symlink(target, link) + self.env["PATH"] = farm + self.assert_failure(self.tag(), "missing required tool: " + missing) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_release.py b/tests/wporg-svn/test_release.py new file mode 100644 index 0000000..4fb8045 --- /dev/null +++ b/tests/wporg-svn/test_release.py @@ -0,0 +1,214 @@ +"""Version, readme Stable Tag and plugin header parsing.""" + +import tempfile +import unittest + +from support import write + +import release + +README = """=== Fixture Plugin === +Contributors: someone +Tags: events +Requires at least: 6.0 +Tested up to: 6.6 +Stable tag: {stable} +License: GPLv2 + +A short description. + +== Changelog == + += 1.0 = +* Stable tag: 9.9 is only an example in the body. +""" + +PLUGIN = """/readme.txt over trunk/readme.txt, server-side only.""" + +import unittest + +from support import ActionTest, SLUG, plugin, write + + +class SetStableTest(ActionTest): + def setUp(self): + super().setUp() + self.repo.seed("1.0") + self.repo.put_tree("tags/1.1", plugin("1.1")) + + def set_stable(self, version="1.1", **env): + return self.run_action("set-stable", WPORG_VERSION=version, **env) + + def test_copies_the_tag_readme_into_trunk_and_nothing_else(self): + before = self.repo.head() + trunk_before = self.repo.tree("trunk") + result = self.set_stable() + self.assert_success(result) + + revision = int(result.outputs["revision"]) + self.assertEqual(result.outputs["version"], "1.1") + self.assertEqual(revision, before + 1) + self.assertEqual(self.repo.changed(revision), [("M", f"/{SLUG}/trunk/readme.txt")]) + self.assertEqual(self.repo.cat("trunk/readme.txt"), self.repo.cat("tags/1.1/readme.txt")) + trunk_after = self.repo.tree("trunk") + del trunk_before["readme.txt"], trunk_after["readme.txt"] + self.assertEqual(trunk_after, trunk_before) + + def test_preserves_the_exact_readme_bytes(self): + readme = b"=== X ===\r\nStable tag: 1.2\r\n\r\nShort.\r\n\xe2\x9c\x93 bytes \t \r\n" + self.repo.put_tree("tags/1.2", {**plugin("1.2"), "readme.txt": readme}) + self.assert_success(self.set_stable("1.2")) + self.assertEqual(self.repo.cat("trunk/readme.txt"), readme) + + def test_identical_readme_is_a_verified_no_op(self): + self.assert_success(self.set_stable()) + head = self.repo.head() + result = self.set_stable() + self.assert_success(result) + self.assertEqual(self.repo.head(), head, "a no-op must not commit") + self.assertEqual(result.outputs, {"version": "1.1", "revision": str(head)}) + self.assertRegex(result.summary, "(?i)no-op") + + def test_can_select_an_older_tag(self): + # Documented: set-stable has no upgrade-only policy, so this rolls back. + self.assert_success(self.set_stable("1.1")) + self.assert_success(self.set_stable("1.0")) + self.assertEqual(self.repo.cat("trunk/readme.txt"), self.repo.cat("tags/1.0/readme.txt")) + + def test_expected_revision_refuses_a_tag_changed_after_approval(self): + approved = self.repo.last_changed("tags/1.1") + self.repo.mucc("put", write(self.repo.staging, "late.php", "/dev/null +fi +""") + result = self.set_stable() + self.assert_failure(result, r"(?i)out of date|conflict") + self.assertEqual(self.repo.cat("trunk/readme.txt").decode(), open(racer).read()) + self.assertRegex(result.log, r"trunk/readme\.txt does not match tags/1\.1/readme\.txt at r\d+; " + r"check who else is committing \(svn log\) before rerunning") + self.assertNotRegex(result.log, "not changed by this run") + + + def test_edit_between_snapshot_and_write_is_never_overwritten(self): + # The -r baseline's own window: another commit lands after our + # reads and before our write starts. The wrapper only times that real + # commit; it fakes nothing about SVN. + racer = self.repo.staging + "/racer-readme.txt" + with open(racer, "w") as handle: + handle.write("=== X ===\nStable tag: 1.0\n\nEdited after the snapshot.\n") + marker = self.repo.staging + "/raced" + self.wrap("svnmucc", f""" +if [ ! -e '{marker}' ]; then + touch '{marker}' + "$NEXT" --non-interactive -m racer put '{racer}' '{self.repo.url("trunk/readme.txt")}' >/dev/null +fi +exec "$NEXT" "$@" +""") + result = self.set_stable() + self.assert_failure(result, r"out of date") + self.assertEqual(self.repo.cat("trunk/readme.txt").decode(), open(racer).read()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_tag.py b/tests/wporg-svn/test_tag.py new file mode 100644 index 0000000..cae4996 --- /dev/null +++ b/tests/wporg-svn/test_tag.py @@ -0,0 +1,289 @@ +"""tag: create tags/ in one commit, as a copy of the stable tag that matches the ZIP exactly.""" + +import os +import shutil +import subprocess +import tempfile +import unittest + +from support import ActionTest, SIGNATURE, SLUG, add_unicode_path_entry, as_bytes, expected_tree, make_zip, plugin, write + +STABLE = plugin("1.0", extra={ + "keep.txt": "unchanged", + "same-size.txt": "AAAA", + "src/a.php": " \'{seen}\'\nexec "$NEXT" "$@"') + self.assert_success(self.tag()) + with open(seen) as handle: + self.assertEqual(int(handle.read()), 256 * 1024 + 1, "in 1 KiB blocks") + + def test_downloads_past_the_size_limit_are_named(self): + # Simulated: serving 256 MiB is too slow for a test. 63 is curl's own + # --max-filesize exit; 153 is the SIGXFSZ kill from the file size limit. + for status in (63, 153): + with self.subTest(status=status): + self.wrap("curl", f"exit {status}") + self.rejected(r"downloading zip-url failed: the ZIP is larger than 256 MiB") + + def test_credentials_never_reach_the_artifact_host(self): + self.assert_success(self.tag()) + for request in self.https.requests: + headers = {k.lower(): v for k, v in request["headers"].items()} + self.assertNotIn("authorization", headers) + self.assertNotIn("fixture-user", repr(request)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_update_trunk.py b/tests/wporg-svn/test_update_trunk.py new file mode 100644 index 0000000..22fa5ee --- /dev/null +++ b/tests/wporg-svn/test_update_trunk.py @@ -0,0 +1,180 @@ +"""update-trunk: replace trunk with tags/ in one atomic, revision-pinned commit.""" + +import unittest + +from support import ActionTest, SLUG, plugin, write + + +class UpdateTrunkTest(ActionTest): + def setUp(self): + super().setUp() + self.repo.seed("1.0", plugin("1.0", extra={"dev-only.txt": "trunk work", "src/old.php": "/dev/null +fi +""") + result = self.update_trunk() + self.assert_failure(result, r"out of date") + self.assertEqual(self.repo.cat("trunk/src/old.php").decode(), open(racer).read()) + self.assertIn("dev-only.txt", self.repo.tree("trunk"), "trunk must be left intact") + self.assertEqual(self.repo.head(), self.snapshot + 1, "only the competing commit may land") + + def test_copies_the_snapshot_even_if_the_tag_changes_afterwards(self): + # The tag changes after our reads and before our write starts (a real + # commit, timed by the wrapper). The copy must still take the snapshot. + tag_at_snapshot = self.repo.tree("tags/1.1") + racer = self.repo.staging + "/edited.php" + with open(racer, "w") as handle: + handle.write("/dev/null +fi +exec "$NEXT" "$@" +""") + result = self.update_trunk() + self.assert_success(result) + self.assertEqual(self.repo.tree("trunk"), tag_at_snapshot) + self.assertNotEqual(self.repo.tree("trunk"), self.repo.tree("tags/1.1")) + + def test_rejected_transaction_leaves_trunk_intact(self): + trunk = self.repo.tree("trunk") + self.repo.hook("pre-commit", "echo 'rejected by policy' >&2\nexit 1\n") + result = self.update_trunk() + self.assert_failure(result, r"trunk does not match tags/1\.1 at r\d+; " + r"check who else is committing \(svn log\) before rerunning") + self.assertEqual(self.repo.head(), self.snapshot) + self.assertEqual(self.repo.tree("trunk"), trunk) + + + def test_edit_between_snapshot_and_write_aborts_the_replacement(self): + # As in set-stable: a real competing commit, timed by the wrapper to land + # after our reads and before our write starts. + racer = self.repo.staging + "/racer.php" + with open(racer, "w") as handle: + handle.write("/dev/null +fi +exec "$NEXT" "$@" +""") + result = self.update_trunk() + self.assert_failure(result, r"out of date") + self.assertEqual(self.repo.cat("trunk/src/old.php").decode(), open(racer).read()) + self.assertIn("dev-only.txt", self.repo.tree("trunk")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/wporg-svn/test_write_guard.py b/tests/wporg-svn/test_write_guard.py new file mode 100644 index 0000000..6c191fd --- /dev/null +++ b/tests/wporg-svn/test_write_guard.py @@ -0,0 +1,109 @@ +"""The test-only write guard: no test or QA run may write to a non-file:// repository.""" + +import os +import subprocess +import sys +import tempfile +import unittest + +import support + +sys.path.insert(0, os.path.join(support.TESTS, "guard")) + +import write_guard # noqa: E402 + +REMOTE = "https://plugins.svn.wordpress.org.invalid/fixture-plugin" # .invalid never resolves +LOCAL = "file:///tmp/repo/fixture-plugin" + + +def root_of(remote): + return lambda path: REMOTE if remote else LOCAL + + +class RefusalTest(unittest.TestCase): + def refused(self, tool, *args, remote_wc=False): + return write_guard.refusal(tool, list(args), root_of(remote_wc)) is not None + + def test_svnmucc_only_writes_to_file_urls(self): + self.assertTrue(self.refused("svnmucc", "-m", "x", "mkdir", REMOTE + "/tags/1.0")) + self.assertTrue(self.refused("svnmucc", "-U", REMOTE, "-m", "x", "rm", "trunk")) + self.assertTrue(self.refused("svnmucc", "-m", "x", "rm", "trunk")) + self.assertTrue(self.refused("svnmucc", "cp", "1", LOCAL + "/a", "svn://example.invalid/b")) + self.assertFalse(self.refused("svnmucc", "-m", "x", "cp", "3", LOCAL + "/tags/1", LOCAL + "/tags/2")) + + def test_svnrdump_only_loads_into_file_urls(self): + self.assertTrue(self.refused("svnrdump", "load", REMOTE)) + self.assertTrue(self.refused("svnrdump", "--non-interactive", "--config-dir", "/tmp/c", "load", "--", REMOTE)) + self.assertTrue(self.refused("svnrdump", "load")) + self.assertFalse(self.refused("svnrdump", "load", LOCAL)) + self.assertFalse(self.refused("svnrdump", "dump", "--quiet", "-r", "5", "--", REMOTE + "/tags/1.0")) + + def test_svn_url_writes_to_remote_repositories_are_refused(self): + for args in ( + ["mkdir", "-m", "x", REMOTE + "/tags/9"], + ["rm", "-m", "x", REMOTE + "/trunk"], + ["copy", "-m", "x", REMOTE + "/trunk", REMOTE + "/tags/9"], + ["import", "-m", "x", ".", REMOTE + "/trunk"], + ["propset", "--revprop", "-r", "5", "svn:log", "x", REMOTE], + ["--non-interactive", "--config-dir", "/tmp/c", "move", "-m", "x", REMOTE + "/a", REMOTE + "/b"], + ["mkdir", "-m", "x", "http://plugins.svn.wordpress.org.invalid/x"], + ): + with self.subTest(args=args): + self.assertTrue(self.refused("svn", *args)) + + def test_svn_working_copy_writes_check_the_repository_root(self): + for args in (["commit", "-m", "x"], ["ci", "-m", "x", "wc"], ["lock", "wc/f"], ["unlock", "wc/f"], + ["--config-dir", "/tmp/c", "commit", "-m", "x", "wc"], ["rm", "wc/f"], + ["propset", "p", "v", "wc/f"]): + with self.subTest(args=args): + self.assertTrue(self.refused("svn", *args, remote_wc=True)) + self.assertFalse(self.refused("svn", *args, remote_wc=False)) + + def test_reads_and_local_operations_pass(self): + for args in (["cat", REMOTE + "/trunk/readme.txt"], ["list", "--xml", REMOTE + "/tags"], + ["info", "--show-item", "revision", REMOTE], ["log", "-v", REMOTE], + ["export", REMOTE + "/tags/1.0", "/tmp/x"], ["checkout", REMOTE + "/trunk", "/tmp/wc"], + ["diff", "--summarize", REMOTE + "/a", REMOTE + "/b"], ["proplist", "-R", REMOTE], + ["add", "wc/new"], ["status", "--xml", "wc"], ["update", "wc"], ["--version", "--quiet"]): + with self.subTest(args=args): + self.assertFalse(self.refused("svn", *args, remote_wc=True)) + + +class ShimTest(unittest.TestCase): + """The shims on PATH really stop the command before it reaches svn.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, self.tmp) + self.env = support.base_env(self.tmp) + + def run_shim(self, *argv): + return subprocess.run(list(argv), env=self.env, stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=60) + + def test_remote_writes_never_reach_svn(self): + for argv in (["svnmucc", "--non-interactive", "-m", "x", "mkdir", REMOTE + "/tags/9"], + ["svn", "--non-interactive", "mkdir", "-m", "x", REMOTE + "/tags/9"], + ["svnrdump", "--non-interactive", "load", REMOTE]): + with self.subTest(argv=argv): + result = self.run_shim(*argv) + self.assertEqual(result.returncode, 99, result.stderr) + self.assertIn("write-guard: refused", result.stderr) + + def test_local_writes_and_reads_pass_through(self): + repo = os.path.join(self.tmp, "repo") + subprocess.run(["svnadmin", "create", repo], check=True) + url = "file://" + repo + self.assertEqual(self.run_shim("svnmucc", "-m", "x", "mkdir", url + "/a").returncode, 0) + self.assertEqual(self.run_shim("svn", "mkdir", "-m", "x", url + "/b").returncode, 0) + listing = self.run_shim("svn", "list", url) + self.assertEqual(listing.stdout.split(), ["a/", "b/"]) + + def test_shims_are_first_on_the_test_path(self): + for tool in ("svn", "svnmucc", "svnrdump"): + found = subprocess.run(["sh", "-c", f"command -v {tool}"], env=self.env, capture_output=True, text=True) + self.assertEqual(os.path.dirname(found.stdout.strip()), support.GUARD_BIN) + + +if __name__ == "__main__": + unittest.main()