techwritermat · Nov 12, 2026 · Nov 12, 2026 · Nov 12, 2026 · Nov 12, 2026
@@ -0,0 +1,182 @@
+name: Docs snippets
+
+# Live tests of the Python and asyncio docs code samples (docs-snippets/README.md). Separate from run-tests.yml on
+# purpose: this suite calls the real PubNub service and is never part of the unit and integration tests.
+#
+# No workflow-level paths filter: a required check behind a paths filter stays pending. The `changes` job decides
+# instead, and the `gate` job is the one to require in branch protection.
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+  schedule:
+    - cron: '41 4 * * *'
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  changes:
+    name: Relevant changes
+    runs-on:
+      group: organization/Default
+    outputs:
+      changed: ${{ steps.diff.outputs.changed }}
+      live: ${{ steps.diff.outputs.live }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Decide with git diff
+        id: diff
+        env:
+          EVENT: ${{ github.event_name }}
+          PR_BASE: ${{ github.event.pull_request.base.sha }}
+          PUSH_BEFORE: ${{ github.event.before }}
+          FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
+        run: |
+          changed=false
+          if [ "$EVENT" = "schedule" ] || [ "$EVENT" = "workflow_dispatch" ]; then
+            changed=true
+          else
+            base="$PR_BASE"
+            [ "$EVENT" = "push" ] && base="$PUSH_BEFORE"
+            if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ] || ! git cat-file -e "$base^{commit}" 2>/dev/null; then
+              changed=true
+            elif git diff --name-only "$base" HEAD | grep -qE '^(docs-snippets/|pubnub/|setup\.py$|requirements-dev\.txt$|\.github/workflows/docs-snippets\.yml$)'; then
+              changed=true
+            fi
+          fi
+          live=$changed
+          if [ "$FORK" = "true" ]; then
+            live=false
+            [ "$changed" = "true" ] && echo "live shards skipped: fork PR without secrets" >> "$GITHUB_STEP_SUMMARY"
+          fi
+          echo "changed=$changed" >> "$GITHUB_OUTPUT"
+          echo "live=$live" >> "$GITHUB_OUTPUT"
+          echo "relevant changes: $changed, live shards: $live"
+
+  dry:
+    name: Dry run and manifest (Python ${{ matrix.python }})
+    needs: [changes]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on:
+      group: organization/Default
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install dependencies
+        run: ./scripts/install.sh
+      - name: Dry run (no network) with compile, selftest, marker lint, manifest and reporter self-test
+        working-directory: docs-snippets
+        run: python -u -m harness.cli dry
+
+  live:
+    name: Live shards (open, pam)
+    needs: [changes, dry]
+    if: needs.changes.outputs.live == 'true'
+    runs-on:
+      group: organization/Default
+    # The shards run one after another in this one job, never in parallel. The pam shard opens its keyset with an
+    # application-level grant for a moment, and the janitor's revoke would break a shard running beside it.
+    timeout-minutes: 30
+    # One live job at a time across the repository, so two workflow runs never share the keysets. A newer waiting run
+    # replaces an older waiting one, a running one is never cancelled.
+    concurrency:
+      group: docs-snippets-live
+      cancel-in-progress: false
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      # No secret is set on the job or on this step, so installing dependencies never sees a key.
+      - name: Install dependencies
+        run: ./scripts/install.sh
+      # Each shard runs even when an earlier shard failed, but not after a cancel. Keys are set on the shard steps only.
+      - name: Shard open
+        id: open
+        if: '!cancelled()'
+        working-directory: docs-snippets
+        env:
+          SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_OPEN_SUBSCRIBE_KEY }}
+          PUBLISH_KEY: ${{ secrets.SDK_DOCS_OPEN_PUBLISH_KEY }}
+          DOCS_SNIPPETS_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}-open
+        run: python -u -m harness.cli run --shard open
+      - name: Shard pam
+        id: pam
+        if: '!cancelled()'
+        working-directory: docs-snippets
+        env:
+          SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_OPEN_SUBSCRIBE_KEY }}
+          PUBLISH_KEY: ${{ secrets.SDK_DOCS_OPEN_PUBLISH_KEY }}
+          PAM_SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_PAM_SUBSCRIBE_KEY }}
+          PAM_PUBLISH_KEY: ${{ secrets.SDK_DOCS_PAM_PUBLISH_KEY }}
+          PAM_SECRET_KEY: ${{ secrets.SDK_DOCS_PAM_SECRET_KEY }}
+          DOCS_SNIPPETS_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}-pam
+        run: python -u -m harness.cli run --shard pam
+      # After a failure or a cancel: revoke the application-level grants and remove leftovers.
+      - name: Revoke grants and clean up
+        if: cancelled() || failure()
+        working-directory: docs-snippets
+        env:
+          SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_OPEN_SUBSCRIBE_KEY }}
+          PUBLISH_KEY: ${{ secrets.SDK_DOCS_OPEN_PUBLISH_KEY }}
+          PAM_SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_PAM_SUBSCRIBE_KEY }}
+          PAM_PUBLISH_KEY: ${{ secrets.SDK_DOCS_PAM_PUBLISH_KEY }}
+          PAM_SECRET_KEY: ${{ secrets.SDK_DOCS_PAM_SECRET_KEY }}
+        run: python -u -m harness.cli janitor
+      - name: Show the result per run
+        if: always()
+        run: cat docs-snippets/report/*/summary.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true
+      - name: Upload the reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-snippets-reports
+          path: docs-snippets/report/
+          if-no-files-found: warn
+
+  gate:
+    name: Docs snippets
+    if: always()
+    needs: [changes, dry, live]
+    runs-on:
+      group: organization/Default
+    steps:
+      - name: Decide
+        env:
+          RESULTS: ${{ needs.changes.result }} ${{ needs.dry.result }} ${{ needs.live.result }}
+          CHANGED: ${{ needs.changes.outputs.changed }}
+        run: |
+          echo "changes, dry, live: $RESULTS (relevant changes: $CHANGED)"
+          for result in $RESULTS; do
+            if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then
+              echo "a needed job ended with $result"
+              exit 1
+            fi
+          done
+          echo "docs snippets gate passed"
@@ -48,9 +48,6 @@ jobs:
         run: |
           ./scripts/install.sh
           python scripts/run-tests.py
-      - name: Cancel workflow runs for commit on error
-        if: failure()
-        uses: ./.github/.release/actions/actions/utils/fast-jobs-failure
   acceptance-tests:
     name: Acceptance tests
     runs-on:
@@ -96,9 +93,6 @@ jobs:
           name: acceptance-test-reports
           path: ./reports
           retention-days: 7
-      - name: Cancel workflow runs for commit on error
-        if: failure()
-        uses: ./.github/.release/actions/actions/utils/fast-jobs-failure
   all-tests:
     name: Tests
     needs: [tests, acceptance-tests]

@@ -18,9 +18,6 @@ jobs:
         run: |
           pip3 install --user --ignore-installed -r requirements-dev.txt
           flake8 --exclude=scripts/,src/,.cache,.git,.idea,.tox,._trial_temp/,venv/ --ignore F811,E402
-      - name: Cancel workflow runs for commit on error
-        if: failure()
-        uses: ./.github/.release/actions/actions/utils/fast-jobs-failure
   pubnub-yml:
     name: "Validate .pubnub.yml"
     runs-on:
@@ -39,9 +36,6 @@ jobs:
         uses: ./.github/.release/actions/actions/validators/pubnub-yml
         with:
           token: ${{ secrets.GH_TOKEN }}
-      - name: Cancel workflow runs for commit on error
-        if: failure()
-        uses: ./.github/.release/actions/actions/utils/fast-jobs-failure
   all-validations:
     name: Validations
     needs: [pubnub-yml, lint]

@@ -83,3 +83,8 @@ PubNubTwisted.ipynb
 
 venv/
 reports/
+
+# Docs snippet suite
+docs-snippets/.run/
+docs-snippets/report/
+docs-snippets/.env.local
@@ -0,0 +1,6 @@
+# Copy to .env.local and fill in. Never commit .env.local.
+PUBLISH_KEY=
+SUBSCRIBE_KEY=
+PAM_PUBLISH_KEY=
+PAM_SUBSCRIBE_KEY=
+PAM_SECRET_KEY=
@@ -0,0 +1,45 @@
+# Authoring a page module
+
+Read `README.md` and `sync/publish.py` (the template) first. This page lists the rules for adding a page.
+
+## Files
+
+* `sync/<page>.py` and, where the asyncio docs have the page, `asyncio/<page>.py`. Page names follow the new docs names: `publish`, `subscribe`, `presence`, `message_persistence`, `message_actions`, `channel_groups`, `file_sharing`, `mobile_push`, `encryption`, `misc`, `configuration`, `access_manager`, `objects`.
+* One step per operation. A step lists the snippet ids it contains: `@step('id_a', 'id_b', role='open')`. A step has a timeout (default 120 s, `timeout=` to change).
+* Snippet ids are snake_case: `<operation>_<variant>`, for example `fetch_history_builder`, `fetch_history_named`. Builder pattern and Named Arguments are different code paths, so both get a snippet and the same step covers both when one result proves both.
+* Do not edit anything outside your own page modules. Needed harness changes go into your report.
+
+## What goes between the markers
+
+* Only what a reader should copy: the call, and the lines that use its result. No `main()`, no `PNConfiguration`, no `os.getenv(..., 'demo')`, no imports that the page's other code does not need. The variable `pubnub` (and `pubnub_a`, `pubnub_b` only when the docs example needs a second client) already exists. Imports that the sample needs go in `# snippet.hide` ... `# snippet.show` lines or, if a reader needs to see them, inside the region.
+* Start from the code in the current docs page (`docs/sdks/python/api-reference/*.md` and `docs/sdks/asyncio/api-reference/*.md` in `/Users/mateusz.wiktor/repos/documentation`, read only). Keep its idioms and literal names (`my_channel`, `ch1`, ...). Do not invent a new style, but fix code that is wrong. Report each docs claim the live service or the SDK source contradicts.
+* Literal channel, group, user and file names inside a region are rewritten by staging to carry the run suffix. The names registered globally are in `harness/names.py` (`REGISTERED`). A literal that is not registered is not rewritten, so for your own names add a module-level list `REGISTERED_NAMES = ['name', ...]` near the top of your module. Outside the markers write `names.with_suffix('my_channel')` to get the same value the region uses.
+* The extractor removes the common indentation. A `print()` of the result is fine. `pubnub.publish()` style calls stay on their own lines.
+* Asyncio samples use `await ... .future()` for builders (the asyncio SDK has no `.sync()`). `unsubscribe_all()` is not awaited.
+
+## What goes outside the markers
+
+* Client creation with `clients.sync(role, 'name')` or `aclients.asyncio(role, 'name')`, listeners (`harness.listeners.listen / alisten`), seeding and every assertion.
+* Assertions use `harness.assertions` (`ok`, `eq`, `contains`, `not_contains`, `timetoken`, `raises`). Reads of eventually consistent data use `eventually` (`aeventually` in asyncio). No fixed sleeps. Prove the effect through a second client, a listener or a read back, not through the response of the call alone. Do not repeat what `tests/` (cassettes) already checks.
+* Never weaken an assertion to make a step pass. A failure the service causes is a finding for the report.
+* Clean up what you create (channel groups, App Context objects, files, messages) in a `finally` or fixture, and register leftovers with `world.record(kind, name)` where the janitor knows the kind.
+* Check return types and attribute names in the SDK source under `/Users/mateusz.wiktor/repos/python/pubnub/` before asserting. Cite nothing from memory.
+
+## Blocks that cannot run
+
+* Signature blocks: keep them out of the suite for now and list them in the report (a drift test comes later).
+* Response blocks (JSON): assert the keys at runtime inside the step that makes the call, treat values such as timetokens as illustrative.
+* A sample that cannot run in CI (push delivery, a host with no network) is registered with `@compile_only(...)` or `@stub(...)` and a reason, in a function that is never collected (its name must not start with `test_`). Never use skip, skipif or xfail.
+
+## Checks to run
+
+From `docs-snippets/`, with `PY=/tmp/pyvenv-docs/bin/python`:
+
+```bash
+$PY -m harness.cli lint
+$PY -m harness.cli manifest            # regenerates coverage.json, the lead merges it, do not commit
+$PY -m flake8 . --ignore F811,E402 --max-line-length 120 --exclude .run
+$PY -u -m harness.cli run --shard open -k "<your module name>"     # or --shard pam
+```
+
+Live runs are shared with other people working at the same time, so keep each run small (your modules only). The run prints key status as set or missing and nothing else. Never print, log or open any env value or file.
@@ -0,0 +1,97 @@
+# Live docs snippet tests
+
+These tests run the code samples of the Python and asyncio SDK docs against the real PubNub service. They check that each sample does what its docs page says.
+
+## Why they exist
+
+A mocked test never runs the docs code and never talks to the service, so it cannot find a wrong docs sample. The existing tests in `tests/` check request shape against recorded cassettes. This suite checks what the service does.
+
+## Layout
+
+| Path | Purpose |
+| :-- | :-- |
+| `sync/` | One module per docs page for the sync `PubNub` client (threads and callbacks) |
+| `asyncio/` | The same pages for `PubNubAsyncio` |
+| `harness/` | Key loader, staging, marker lint, manifest, janitor, reporter, `eventually` |
+| `selftest/` | Dry checks of the harness itself |
+| `coverage.json` | Generated manifest, one entry per snippet id |
+| `.run/`, `report/` | Git-ignored output |
+
+The `asyncio/` folder has no `__init__.py` on purpose, so it can never shadow the standard library module.
+
+## How a step is written
+
+A step is a test function. The docs code sits between `# snippet.<id>` and `# snippet.end` inside the step body. Client setup and every assertion sit outside the markers. A marker lint enforces it.
+
+```python
+@step('signal_message', role='open')
+def test_signal(clients):
+    pubnub = clients.sync('open', 'signaler')
+    seen = listen(clients.sync('open', 'subscriber'), names.with_suffix('my_channel'))
+
+    # snippet.signal_message
+    envelope = pubnub.signal() \
+        .channel("my_channel") \
+        .message("typing_on") \
+        .sync()
+    # snippet.end
+
+    a.timetoken(envelope.result.timetoken)
+    eventually(lambda: a.eq([s.message for s in seen.signals], ['typing_on']))
+```
+
+Docs pages embed the region with `EmbeddedCode` and `snippets={['signal_message']}`. The extractor removes the common indentation.
+
+## What a run does
+
+| Stage | What happens |
+| :-- | :-- |
+| 1. Load keys | The loader reads the env names below and prints `set` or `missing` per role. It never falls back to demo keys |
+| 2. Stage | The suite is copied to `.run/<runId>/`. Registered names get the run suffix and every marker line is instrumented in place, so line numbers never move |
+| 3. Steps | Each step runs its snippet, then asserts the effect through the service |
+| 4. Honesty | A step fails when a snippet did not reach its `snippet.end` line or when no assertion ran after it started |
+| 5. Janitor | Revokes application-level Access Manager (v2/v3) grants and removes leftovers older than six hours |
+| 6. Report | `report/<shard>/run.json` and `summary.md`, with every key and token masked |
+
+## Run it locally
+
+You need Python 3.9 or later, the dev requirements (`pip install -r requirements-dev.txt`) and your own keysets. The suite imports `pubnub` from this checkout. Run from `docs-snippets/`.
+
+```bash
+cp .env.example .env.local        # fill in your keys, never commit this file
+python -u -m harness.cli env      # prints set or missing per role
+python -u -m harness.cli dry      # no network: compile, selftest, marker lint, manifest check
+python -u -m harness.cli run      # all shards
+python -u -m harness.cli run --shard open
+python -u -m harness.cli manifest [--check]
+python -u -m harness.cli janitor
+```
+
+## Keys
+
+| Role | Env names | Used for |
+| :-- | :-- | :-- |
+| `open` | `PUBLISH_KEY`, `SUBSCRIBE_KEY` | Publish, subscribe, presence, storage, message actions, App Context, files, push, channel groups, encryption, misc |
+| `pam` | `PAM_PUBLISH_KEY`, `PAM_SUBSCRIBE_KEY`, `PAM_SECRET_KEY` | Access Manager (v2/v3) samples and the 403 branches |
+
+The names differ from the `PN_KEY_*` names of the existing tests on purpose, so the suite never reads those keys by accident. Real env vars win over the env file (`DOCS_SNIPPETS_ENV_FILE`, default `docs-snippets/.env.local`).
+
+The Python SDK has no DataSync API, so there is no `datasync` role.
+
+In CI the workflow [`docs-snippets.yml`](../.github/workflows/docs-snippets.yml) maps the `SDK_DOCS_OPEN_*` and `SDK_DOCS_PAM_*` secrets onto these names on the shard steps only. Without a GitHub Environment that has required reviewers, anyone with push access can read these secrets through a pull request.
+
+## Rules
+
+| Rule | Why |
+| :-- | :-- |
+| No skips, no `xfail`, no `-k` in CI | Every run must run all valid tests. A missing key fails the run with its exact name |
+| Samples that cannot run are `compile_only` or `stub` with a reason in `coverage.json` | They are listed honestly, never skipped silently |
+| Never weaken an assertion | Waiting longer with `eventually` is fine. Loosening a check is not |
+| No fixed sleeps | Use `eventually(fn, timeout=15, interval=0.5)` for every read after a write |
+| Names carry the run suffix and the marker `dsn` | Parallel and repeated runs never collide, and the janitor can tell leftovers apart |
+| Assertions sit outside the markers | The region stays byte for byte what the docs show |
+| Keys and tokens never reach output | The reporter masks values, URL-encoded forms and key shapes, and a self-test proves it |
+
+## Mutation check
+
+`DOCS_SNIPPETS_MUTATE=<snippet id> python -u -m harness.cli run --shard open` makes the named step throw at its start marker. That step must fail. It is refused in CI.