From 70de9fd42303395f30c2d9f47fb111e5002d7ab3 Mon Sep 17 00:00:00 2001 From: Mateusz Wiktor Date: Thu, 12 Nov 2026 11:00:02 +0100 Subject: [PATCH 1/4] ci: remove the cross-workflow cancel step from Tests and Validations The "Cancel workflow runs for commit on error" step (fast-jobs-failure) cancels every workflow of the commit when one fast job fails, so an unrelated failure cancelled the first attempt of other workflows. The JavaScript SDK removed the same step in PR 539. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/run-tests.yml | 6 ------ .github/workflows/run-validations.yml | 6 ------ 2 files changed, 12 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 8a5641c8..dcea89f3 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -48,9 +48,6 @@ jobs: run: | ./scripts/install.sh python scripts/run-tests.py - - name: Cancel workflow runs for commit on error - if: failure() - uses: ./.github/.release/actions/actions/utils/fast-jobs-failure acceptance-tests: name: Acceptance tests runs-on: @@ -96,9 +93,6 @@ jobs: name: acceptance-test-reports path: ./reports retention-days: 7 - - name: Cancel workflow runs for commit on error - if: failure() - uses: ./.github/.release/actions/actions/utils/fast-jobs-failure all-tests: name: Tests needs: [tests, acceptance-tests] diff --git a/.github/workflows/run-validations.yml b/.github/workflows/run-validations.yml index 2f3b2b4e..06780238 100644 --- a/.github/workflows/run-validations.yml +++ b/.github/workflows/run-validations.yml @@ -18,9 +18,6 @@ jobs: run: | pip3 install --user --ignore-installed -r requirements-dev.txt flake8 --exclude=scripts/,src/,.cache,.git,.idea,.tox,._trial_temp/,venv/ --ignore F811,E402 - - name: Cancel workflow runs for commit on error - if: failure() - uses: ./.github/.release/actions/actions/utils/fast-jobs-failure pubnub-yml: name: "Validate .pubnub.yml" runs-on: @@ -39,9 +36,6 @@ jobs: uses: ./.github/.release/actions/actions/validators/pubnub-yml with: token: ${{ secrets.GH_TOKEN }} - - name: Cancel workflow runs for commit on error - if: failure() - uses: ./.github/.release/actions/actions/utils/fast-jobs-failure all-validations: name: Validations needs: [pubnub-yml, lint] From 404c63d08943078557604ef975c960b3b7acc877 Mon Sep 17 00:00:00 2001 From: Mateusz Wiktor Date: Thu, 12 Nov 2026 11:00:03 +0100 Subject: [PATCH 2/4] test: add live docs snippet tests for the sync and asyncio docs Adds docs-snippets/, a pytest suite that runs the code samples of the Python and asyncio docs against the real PubNub service and asserts the documented effect. The docs embed the snippet regions with EmbeddedCode. * 311 snippet ids in 193 steps (sync and asyncio), roles open and pam * assertions sit outside the snippet markers, a marker lint enforces it * honesty checks: each step must reach snippet.end and make an assertion * run-suffixed names per step, key and token redaction, janitor, no skips * coverage.json manifest with a --check mode * separate workflow: dry job on Python 3.9 to 3.13, live shards in series under one concurrency group, nightly, fork PRs skip live * setup.py excludes docs-snippets from the package Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/docs-snippets.yml | 182 ++ .gitignore | 5 + docs-snippets/.env.example | 6 + docs-snippets/AUTHORING.md | 45 + docs-snippets/README.md | 97 + docs-snippets/asyncio/access_manager.py | 364 +++ docs-snippets/asyncio/access_manager_v2.py | 320 +++ docs-snippets/asyncio/channel_groups.py | 146 ++ docs-snippets/asyncio/configuration.py | 321 +++ docs-snippets/asyncio/file_sharing.py | 248 ++ docs-snippets/asyncio/message_actions.py | 199 ++ docs-snippets/asyncio/message_persistence.py | 419 ++++ docs-snippets/asyncio/misc.py | 53 + docs-snippets/asyncio/mobile_push.py | 218 ++ docs-snippets/asyncio/presence.py | 252 ++ docs-snippets/asyncio/publish.py | 176 ++ docs-snippets/asyncio/subscribe.py | 1039 ++++++++ docs-snippets/asyncio/zz_application_level.py | 55 + docs-snippets/conftest.py | 98 + docs-snippets/coverage.json | 2183 +++++++++++++++++ docs-snippets/fixtures/sample.gif | Bin 0 -> 134 bytes docs-snippets/harness/__init__.py | 6 + docs-snippets/harness/assertions.py | 53 + docs-snippets/harness/audit.py | 136 + docs-snippets/harness/cli.py | 89 + docs-snippets/harness/env.py | 69 + docs-snippets/harness/janitor.py | 86 + docs-snippets/harness/lint_markers.py | 61 + docs-snippets/harness/listeners.py | 76 + docs-snippets/harness/manifest.py | 66 + docs-snippets/harness/markers.py | 72 + docs-snippets/harness/names.py | 68 + docs-snippets/harness/preflight.py | 99 + docs-snippets/harness/recorder.py | 51 + docs-snippets/harness/redact.py | 42 + docs-snippets/harness/registry.py | 54 + docs-snippets/harness/reporter.py | 80 + docs-snippets/harness/stage.py | 65 + docs-snippets/harness/step.py | 98 + docs-snippets/harness/wait.py | 33 + docs-snippets/harness/world.py | 86 + docs-snippets/pytest.ini | 16 + docs-snippets/selftest/test_harness.py | 103 + docs-snippets/sync/access_manager.py | 384 +++ docs-snippets/sync/access_manager_v2.py | 318 +++ docs-snippets/sync/channel_groups.py | 143 ++ docs-snippets/sync/configuration.py | 299 +++ docs-snippets/sync/encryption.py | 192 ++ docs-snippets/sync/file_sharing.py | 297 +++ docs-snippets/sync/message_actions.py | 192 ++ docs-snippets/sync/message_persistence.py | 403 +++ docs-snippets/sync/misc.py | 90 + docs-snippets/sync/mobile_push.py | 262 ++ docs-snippets/sync/objects.py | 579 +++++ docs-snippets/sync/objects_membership.py | 803 ++++++ docs-snippets/sync/presence.py | 329 +++ docs-snippets/sync/publish.py | 224 ++ docs-snippets/sync/subscribe.py | 995 ++++++++ docs-snippets/sync/zz_application_level.py | 46 + setup.py | 2 +- 60 files changed, 13492 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docs-snippets.yml create mode 100644 docs-snippets/.env.example create mode 100644 docs-snippets/AUTHORING.md create mode 100644 docs-snippets/README.md create mode 100644 docs-snippets/asyncio/access_manager.py create mode 100644 docs-snippets/asyncio/access_manager_v2.py create mode 100644 docs-snippets/asyncio/channel_groups.py create mode 100644 docs-snippets/asyncio/configuration.py create mode 100644 docs-snippets/asyncio/file_sharing.py create mode 100644 docs-snippets/asyncio/message_actions.py create mode 100644 docs-snippets/asyncio/message_persistence.py create mode 100644 docs-snippets/asyncio/misc.py create mode 100644 docs-snippets/asyncio/mobile_push.py create mode 100644 docs-snippets/asyncio/presence.py create mode 100644 docs-snippets/asyncio/publish.py create mode 100644 docs-snippets/asyncio/subscribe.py create mode 100644 docs-snippets/asyncio/zz_application_level.py create mode 100644 docs-snippets/conftest.py create mode 100644 docs-snippets/coverage.json create mode 100644 docs-snippets/fixtures/sample.gif create mode 100644 docs-snippets/harness/__init__.py create mode 100644 docs-snippets/harness/assertions.py create mode 100644 docs-snippets/harness/audit.py create mode 100644 docs-snippets/harness/cli.py create mode 100644 docs-snippets/harness/env.py create mode 100644 docs-snippets/harness/janitor.py create mode 100644 docs-snippets/harness/lint_markers.py create mode 100644 docs-snippets/harness/listeners.py create mode 100644 docs-snippets/harness/manifest.py create mode 100644 docs-snippets/harness/markers.py create mode 100644 docs-snippets/harness/names.py create mode 100644 docs-snippets/harness/preflight.py create mode 100644 docs-snippets/harness/recorder.py create mode 100644 docs-snippets/harness/redact.py create mode 100644 docs-snippets/harness/registry.py create mode 100644 docs-snippets/harness/reporter.py create mode 100644 docs-snippets/harness/stage.py create mode 100644 docs-snippets/harness/step.py create mode 100644 docs-snippets/harness/wait.py create mode 100644 docs-snippets/harness/world.py create mode 100644 docs-snippets/pytest.ini create mode 100644 docs-snippets/selftest/test_harness.py create mode 100644 docs-snippets/sync/access_manager.py create mode 100644 docs-snippets/sync/access_manager_v2.py create mode 100644 docs-snippets/sync/channel_groups.py create mode 100644 docs-snippets/sync/configuration.py create mode 100644 docs-snippets/sync/encryption.py create mode 100644 docs-snippets/sync/file_sharing.py create mode 100644 docs-snippets/sync/message_actions.py create mode 100644 docs-snippets/sync/message_persistence.py create mode 100644 docs-snippets/sync/misc.py create mode 100644 docs-snippets/sync/mobile_push.py create mode 100644 docs-snippets/sync/objects.py create mode 100644 docs-snippets/sync/objects_membership.py create mode 100644 docs-snippets/sync/presence.py create mode 100644 docs-snippets/sync/publish.py create mode 100644 docs-snippets/sync/subscribe.py create mode 100644 docs-snippets/sync/zz_application_level.py diff --git a/.github/workflows/docs-snippets.yml b/.github/workflows/docs-snippets.yml new file mode 100644 index 00000000..0ad4d9e5 --- /dev/null +++ b/.github/workflows/docs-snippets.yml @@ -0,0 +1,182 @@ +name: Docs snippets + +# Live tests of the Python and asyncio docs code samples (docs-snippets/README.md). Separate from run-tests.yml on +# purpose: this suite calls the real PubNub service and is never part of the unit and integration tests. +# +# No workflow-level paths filter: a required check behind a paths filter stays pending. The `changes` job decides +# instead, and the `gate` job is the one to require in branch protection. + +on: + pull_request: + push: + branches: [master] + schedule: + - cron: '41 4 * * *' + workflow_dispatch: + +defaults: + run: + shell: bash + +permissions: + contents: read + +jobs: + changes: + name: Relevant changes + runs-on: + group: organization/Default + outputs: + changed: ${{ steps.diff.outputs.changed }} + live: ${{ steps.diff.outputs.live }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - name: Decide with git diff + id: diff + env: + EVENT: ${{ github.event_name }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + PUSH_BEFORE: ${{ github.event.before }} + FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + run: | + changed=false + if [ "$EVENT" = "schedule" ] || [ "$EVENT" = "workflow_dispatch" ]; then + changed=true + else + base="$PR_BASE" + [ "$EVENT" = "push" ] && base="$PUSH_BEFORE" + if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ] || ! git cat-file -e "$base^{commit}" 2>/dev/null; then + changed=true + elif git diff --name-only "$base" HEAD | grep -qE '^(docs-snippets/|pubnub/|setup\.py$|requirements-dev\.txt$|\.github/workflows/docs-snippets\.yml$)'; then + changed=true + fi + fi + live=$changed + if [ "$FORK" = "true" ]; then + live=false + [ "$changed" = "true" ] && echo "live shards skipped: fork PR without secrets" >> "$GITHUB_STEP_SUMMARY" + fi + echo "changed=$changed" >> "$GITHUB_OUTPUT" + echo "live=$live" >> "$GITHUB_OUTPUT" + echo "relevant changes: $changed, live shards: $live" + + dry: + name: Dry run and manifest (Python ${{ matrix.python }}) + needs: [changes] + if: needs.changes.outputs.changed == 'true' + runs-on: + group: organization/Default + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install dependencies + run: ./scripts/install.sh + - name: Dry run (no network) with compile, selftest, marker lint, manifest and reporter self-test + working-directory: docs-snippets + run: python -u -m harness.cli dry + + live: + name: Live shards (open, pam) + needs: [changes, dry] + if: needs.changes.outputs.live == 'true' + runs-on: + group: organization/Default + # The shards run one after another in this one job, never in parallel. The pam shard opens its keyset with an + # application-level grant for a moment, and the janitor's revoke would break a shard running beside it. + timeout-minutes: 30 + # One live job at a time across the repository, so two workflow runs never share the keysets. A newer waiting run + # replaces an older waiting one, a running one is never cancelled. + concurrency: + group: docs-snippets-live + cancel-in-progress: false + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Setup Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + # No secret is set on the job or on this step, so installing dependencies never sees a key. + - name: Install dependencies + run: ./scripts/install.sh + # Each shard runs even when an earlier shard failed, but not after a cancel. Keys are set on the shard steps only. + - name: Shard open + id: open + if: '!cancelled()' + working-directory: docs-snippets + env: + SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_OPEN_SUBSCRIBE_KEY }} + PUBLISH_KEY: ${{ secrets.SDK_DOCS_OPEN_PUBLISH_KEY }} + DOCS_SNIPPETS_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}-open + run: python -u -m harness.cli run --shard open + - name: Shard pam + id: pam + if: '!cancelled()' + working-directory: docs-snippets + env: + SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_OPEN_SUBSCRIBE_KEY }} + PUBLISH_KEY: ${{ secrets.SDK_DOCS_OPEN_PUBLISH_KEY }} + PAM_SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_PAM_SUBSCRIBE_KEY }} + PAM_PUBLISH_KEY: ${{ secrets.SDK_DOCS_PAM_PUBLISH_KEY }} + PAM_SECRET_KEY: ${{ secrets.SDK_DOCS_PAM_SECRET_KEY }} + DOCS_SNIPPETS_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}-pam + run: python -u -m harness.cli run --shard pam + # After a failure or a cancel: revoke the application-level grants and remove leftovers. + - name: Revoke grants and clean up + if: cancelled() || failure() + working-directory: docs-snippets + env: + SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_OPEN_SUBSCRIBE_KEY }} + PUBLISH_KEY: ${{ secrets.SDK_DOCS_OPEN_PUBLISH_KEY }} + PAM_SUBSCRIBE_KEY: ${{ secrets.SDK_DOCS_PAM_SUBSCRIBE_KEY }} + PAM_PUBLISH_KEY: ${{ secrets.SDK_DOCS_PAM_PUBLISH_KEY }} + PAM_SECRET_KEY: ${{ secrets.SDK_DOCS_PAM_SECRET_KEY }} + run: python -u -m harness.cli janitor + - name: Show the result per run + if: always() + run: cat docs-snippets/report/*/summary.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + - name: Upload the reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: docs-snippets-reports + path: docs-snippets/report/ + if-no-files-found: warn + + gate: + name: Docs snippets + if: always() + needs: [changes, dry, live] + runs-on: + group: organization/Default + steps: + - name: Decide + env: + RESULTS: ${{ needs.changes.result }} ${{ needs.dry.result }} ${{ needs.live.result }} + CHANGED: ${{ needs.changes.outputs.changed }} + run: | + echo "changes, dry, live: $RESULTS (relevant changes: $CHANGED)" + for result in $RESULTS; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "a needed job ended with $result" + exit 1 + fi + done + echo "docs snippets gate passed" diff --git a/.gitignore b/.gitignore index ae82a593..dcb0bda7 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,8 @@ PubNubTwisted.ipynb venv/ reports/ + +# Docs snippet suite +docs-snippets/.run/ +docs-snippets/report/ +docs-snippets/.env.local diff --git a/docs-snippets/.env.example b/docs-snippets/.env.example new file mode 100644 index 00000000..06a8eb85 --- /dev/null +++ b/docs-snippets/.env.example @@ -0,0 +1,6 @@ +# Copy to .env.local and fill in. Never commit .env.local. +PUBLISH_KEY= +SUBSCRIBE_KEY= +PAM_PUBLISH_KEY= +PAM_SUBSCRIBE_KEY= +PAM_SECRET_KEY= diff --git a/docs-snippets/AUTHORING.md b/docs-snippets/AUTHORING.md new file mode 100644 index 00000000..7df47e68 --- /dev/null +++ b/docs-snippets/AUTHORING.md @@ -0,0 +1,45 @@ +# Authoring a page module + +Read `README.md` and `sync/publish.py` (the template) first. This page lists the rules for adding a page. + +## Files + +* `sync/.py` and, where the asyncio docs have the page, `asyncio/.py`. Page names follow the new docs names: `publish`, `subscribe`, `presence`, `message_persistence`, `message_actions`, `channel_groups`, `file_sharing`, `mobile_push`, `encryption`, `misc`, `configuration`, `access_manager`, `objects`. +* One step per operation. A step lists the snippet ids it contains: `@step('id_a', 'id_b', role='open')`. A step has a timeout (default 120 s, `timeout=` to change). +* Snippet ids are snake_case: `_`, for example `fetch_history_builder`, `fetch_history_named`. Builder pattern and Named Arguments are different code paths, so both get a snippet and the same step covers both when one result proves both. +* Do not edit anything outside your own page modules. Needed harness changes go into your report. + +## What goes between the markers + +* Only what a reader should copy: the call, and the lines that use its result. No `main()`, no `PNConfiguration`, no `os.getenv(..., 'demo')`, no imports that the page's other code does not need. The variable `pubnub` (and `pubnub_a`, `pubnub_b` only when the docs example needs a second client) already exists. Imports that the sample needs go in `# snippet.hide` ... `# snippet.show` lines or, if a reader needs to see them, inside the region. +* Start from the code in the current docs page (`docs/sdks/python/api-reference/*.md` and `docs/sdks/asyncio/api-reference/*.md` in `/Users/mateusz.wiktor/repos/documentation`, read only). Keep its idioms and literal names (`my_channel`, `ch1`, ...). Do not invent a new style, but fix code that is wrong. Report each docs claim the live service or the SDK source contradicts. +* Literal channel, group, user and file names inside a region are rewritten by staging to carry the run suffix. The names registered globally are in `harness/names.py` (`REGISTERED`). A literal that is not registered is not rewritten, so for your own names add a module-level list `REGISTERED_NAMES = ['name', ...]` near the top of your module. Outside the markers write `names.with_suffix('my_channel')` to get the same value the region uses. +* The extractor removes the common indentation. A `print()` of the result is fine. `pubnub.publish()` style calls stay on their own lines. +* Asyncio samples use `await ... .future()` for builders (the asyncio SDK has no `.sync()`). `unsubscribe_all()` is not awaited. + +## What goes outside the markers + +* Client creation with `clients.sync(role, 'name')` or `aclients.asyncio(role, 'name')`, listeners (`harness.listeners.listen / alisten`), seeding and every assertion. +* Assertions use `harness.assertions` (`ok`, `eq`, `contains`, `not_contains`, `timetoken`, `raises`). Reads of eventually consistent data use `eventually` (`aeventually` in asyncio). No fixed sleeps. Prove the effect through a second client, a listener or a read back, not through the response of the call alone. Do not repeat what `tests/` (cassettes) already checks. +* Never weaken an assertion to make a step pass. A failure the service causes is a finding for the report. +* Clean up what you create (channel groups, App Context objects, files, messages) in a `finally` or fixture, and register leftovers with `world.record(kind, name)` where the janitor knows the kind. +* Check return types and attribute names in the SDK source under `/Users/mateusz.wiktor/repos/python/pubnub/` before asserting. Cite nothing from memory. + +## Blocks that cannot run + +* Signature blocks: keep them out of the suite for now and list them in the report (a drift test comes later). +* Response blocks (JSON): assert the keys at runtime inside the step that makes the call, treat values such as timetokens as illustrative. +* A sample that cannot run in CI (push delivery, a host with no network) is registered with `@compile_only(...)` or `@stub(...)` and a reason, in a function that is never collected (its name must not start with `test_`). Never use skip, skipif or xfail. + +## Checks to run + +From `docs-snippets/`, with `PY=/tmp/pyvenv-docs/bin/python`: + +```bash +$PY -m harness.cli lint +$PY -m harness.cli manifest # regenerates coverage.json, the lead merges it, do not commit +$PY -m flake8 . --ignore F811,E402 --max-line-length 120 --exclude .run +$PY -u -m harness.cli run --shard open -k "" # or --shard pam +``` + +Live runs are shared with other people working at the same time, so keep each run small (your modules only). The run prints key status as set or missing and nothing else. Never print, log or open any env value or file. diff --git a/docs-snippets/README.md b/docs-snippets/README.md new file mode 100644 index 00000000..6e123158 --- /dev/null +++ b/docs-snippets/README.md @@ -0,0 +1,97 @@ +# Live docs snippet tests + +These tests run the code samples of the Python and asyncio SDK docs against the real PubNub service. They check that each sample does what its docs page says. + +## Why they exist + +A mocked test never runs the docs code and never talks to the service, so it cannot find a wrong docs sample. The existing tests in `tests/` check request shape against recorded cassettes. This suite checks what the service does. + +## Layout + +| Path | Purpose | +| :-- | :-- | +| `sync/` | One module per docs page for the sync `PubNub` client (threads and callbacks) | +| `asyncio/` | The same pages for `PubNubAsyncio` | +| `harness/` | Key loader, staging, marker lint, manifest, janitor, reporter, `eventually` | +| `selftest/` | Dry checks of the harness itself | +| `coverage.json` | Generated manifest, one entry per snippet id | +| `.run/`, `report/` | Git-ignored output | + +The `asyncio/` folder has no `__init__.py` on purpose, so it can never shadow the standard library module. + +## How a step is written + +A step is a test function. The docs code sits between `# snippet.` and `# snippet.end` inside the step body. Client setup and every assertion sit outside the markers. A marker lint enforces it. + +```python +@step('signal_message', role='open') +def test_signal(clients): + pubnub = clients.sync('open', 'signaler') + seen = listen(clients.sync('open', 'subscriber'), names.with_suffix('my_channel')) + + # snippet.signal_message + envelope = pubnub.signal() \ + .channel("my_channel") \ + .message("typing_on") \ + .sync() + # snippet.end + + a.timetoken(envelope.result.timetoken) + eventually(lambda: a.eq([s.message for s in seen.signals], ['typing_on'])) +``` + +Docs pages embed the region with `EmbeddedCode` and `snippets={['signal_message']}`. The extractor removes the common indentation. + +## What a run does + +| Stage | What happens | +| :-- | :-- | +| 1. Load keys | The loader reads the env names below and prints `set` or `missing` per role. It never falls back to demo keys | +| 2. Stage | The suite is copied to `.run//`. Registered names get the run suffix and every marker line is instrumented in place, so line numbers never move | +| 3. Steps | Each step runs its snippet, then asserts the effect through the service | +| 4. Honesty | A step fails when a snippet did not reach its `snippet.end` line or when no assertion ran after it started | +| 5. Janitor | Revokes application-level Access Manager (v2/v3) grants and removes leftovers older than six hours | +| 6. Report | `report//run.json` and `summary.md`, with every key and token masked | + +## Run it locally + +You need Python 3.9 or later, the dev requirements (`pip install -r requirements-dev.txt`) and your own keysets. The suite imports `pubnub` from this checkout. Run from `docs-snippets/`. + +```bash +cp .env.example .env.local # fill in your keys, never commit this file +python -u -m harness.cli env # prints set or missing per role +python -u -m harness.cli dry # no network: compile, selftest, marker lint, manifest check +python -u -m harness.cli run # all shards +python -u -m harness.cli run --shard open +python -u -m harness.cli manifest [--check] +python -u -m harness.cli janitor +``` + +## Keys + +| Role | Env names | Used for | +| :-- | :-- | :-- | +| `open` | `PUBLISH_KEY`, `SUBSCRIBE_KEY` | Publish, subscribe, presence, storage, message actions, App Context, files, push, channel groups, encryption, misc | +| `pam` | `PAM_PUBLISH_KEY`, `PAM_SUBSCRIBE_KEY`, `PAM_SECRET_KEY` | Access Manager (v2/v3) samples and the 403 branches | + +The names differ from the `PN_KEY_*` names of the existing tests on purpose, so the suite never reads those keys by accident. Real env vars win over the env file (`DOCS_SNIPPETS_ENV_FILE`, default `docs-snippets/.env.local`). + +The Python SDK has no DataSync API, so there is no `datasync` role. + +In CI the workflow [`docs-snippets.yml`](../.github/workflows/docs-snippets.yml) maps the `SDK_DOCS_OPEN_*` and `SDK_DOCS_PAM_*` secrets onto these names on the shard steps only. Without a GitHub Environment that has required reviewers, anyone with push access can read these secrets through a pull request. + +## Rules + +| Rule | Why | +| :-- | :-- | +| No skips, no `xfail`, no `-k` in CI | Every run must run all valid tests. A missing key fails the run with its exact name | +| Samples that cannot run are `compile_only` or `stub` with a reason in `coverage.json` | They are listed honestly, never skipped silently | +| Never weaken an assertion | Waiting longer with `eventually` is fine. Loosening a check is not | +| No fixed sleeps | Use `eventually(fn, timeout=15, interval=0.5)` for every read after a write | +| Names carry the run suffix and the marker `dsn` | Parallel and repeated runs never collide, and the janitor can tell leftovers apart | +| Assertions sit outside the markers | The region stays byte for byte what the docs show | +| Keys and tokens never reach output | The reporter masks values, URL-encoded forms and key shapes, and a self-test proves it | + +## Mutation check + +`DOCS_SNIPPETS_MUTATE= python -u -m harness.cli run --shard open` makes the named step throw at its start marker. That step must fail. It is refused in CI. diff --git a/docs-snippets/asyncio/access_manager.py b/docs-snippets/asyncio/access_manager.py new file mode 100644 index 00000000..9211cab5 --- /dev/null +++ b/docs-snippets/asyncio/access_manager.py @@ -0,0 +1,364 @@ +"""Docs page: access-manager (asyncio), Access Manager v3 tokens. Same steps as sync/access_manager.py with await. + +The asyncio client does not raise on an HTTP or validation error. future() resolves with a PubNubAsyncioException +object instead, so the helpers below inspect the resolved value. +""" +from pubnub.exceptions import PubNubAsyncioException +from pubnub.models.consumer.v3.channel import Channel + +from harness import assertions as a +from harness import names, redact +from harness.step import step +from harness.wait import aeventually + +REGISTERED_NAMES = ['readonly-channel', 'readwrite-channel', 'my-authorized-uuid', 'revoke-channel'] + +FLAGS = ('read', 'write', 'manage', 'delete', 'get', 'update', 'join') + + +def perms(*on): + """The seven flags parse_token returns for one resource, True only for the ones named.""" + return {flag: flag in on for flag in FLAGS} + + +def holder(aclients, user_id, token=None): + """A client that holds a token. It has no secret key, so every request is checked against the token.""" + client = aclients.asyncio('pam', user_id, secret=False) + if token: + client.set_token(token) + return client + + +async def publish(client, channel): + return await client.publish().channel(channel).message('docs').future() + + +async def refused(client, channel): + resolved = await publish(client, channel) + a.ok(isinstance(resolved, PubNubAsyncioException), 'the publish was refused') + a.eq(resolved.status.status_code, 403, 'status code of the refusal') + + +async def allowed(client, channel): + resolved = await publish(client, channel) + a.ok(not isinstance(resolved, PubNubAsyncioException), 'the publish went through') + a.timetoken(resolved.result.timetoken) + + +def failure(resolved): + """The error text of a future() that resolved with an error.""" + a.ok(isinstance(resolved, PubNubAsyncioException), 'the call resolved with an error object') + return str(resolved.value()) + + +@step('grant_token_builder', 'parse_token', role='pam') +async def test_grant_and_parse_token(aclients): + pubnub = aclients.asyncio('pam', 'admin') + + # snippet.grant_token_builder + from pubnub.models.consumer.v3.channel import Channel + + envelope = await pubnub.grant_token() \ + .channels([ + Channel.id('readonly-channel').read(), + Channel.id('readwrite-channel').read().write(), + ]) \ + .authorized_uuid("my-authorized-uuid") \ + .ttl(15) \ + .future() + # snippet.end + token = envelope.result.token + redact.register(token) + a.ok(isinstance(token, str) and len(token) > 40, 'the result carries a token') + a.eq(envelope.result.get_token(), token) + + # snippet.parse_token + parsed_token = pubnub.parse_token(token) + # snippet.end + + a.eq(parsed_token['version'], 2) + a.eq(parsed_token['ttl'], 15) + a.eq(parsed_token['authorized_uuid'], names.with_suffix('my-authorized-uuid')) + a.eq(parsed_token['resources']['channels'], { + names.with_suffix('readonly-channel'): perms('read'), + names.with_suffix('readwrite-channel'): perms('read', 'write'), + }) + a.eq(parsed_token['resources'].get('groups', {}), {}) + a.eq(parsed_token['resources'].get('uuids', {}), {}) + a.eq(parsed_token['patterns'].get('channels', {}), {}) + a.ok('categories' not in parsed_token, 'no category in a token granted without categories') + + +@step('set_token', role='pam') +async def test_set_token(aclients): + admin = aclients.asyncio('pam', 'admin') + readonly = names.with_suffix('readonly-channel') + readwrite = names.with_suffix('readwrite-channel') + fresh = names.with_suffix('set_token_other_channel') + granted = await admin.grant_token() \ + .channels([Channel.id(readonly).read(), Channel.id(readwrite).read().write()]) \ + .authorized_uuid(names.with_suffix('my-authorized-uuid')) \ + .ttl(15) \ + .future() + token = granted.result.token + redact.register(token) + without = holder(aclients, 'my-authorized-uuid') + pubnub = holder(aclients, 'my-authorized-uuid') + + # snippet.set_token + pubnub.set_token(token) + # snippet.end + + await aeventually(lambda: allowed(pubnub, readwrite), timeout=30, what='publish on the granted channel') + await refused(pubnub, readonly) + await refused(pubnub, fresh) + await refused(without, readwrite) + + +@step('revoke_token', role='pam', timeout=240) +async def test_revoke_token(aclients): + admin = aclients.asyncio('pam', 'admin') + pubnub = admin + channel = names.with_suffix('revoke-channel') + granted = await admin.grant_token() \ + .channels([Channel.id(channel).read().write()]) \ + .authorized_uuid(names.with_suffix('my-authorized-uuid')) \ + .ttl(15) \ + .future() + token = granted.result.token + redact.register(token) + holder_client = holder(aclients, 'my-authorized-uuid', token) + await aeventually(lambda: allowed(holder_client, channel), timeout=30, what='publish with the fresh token') + + # A client without a secret key cannot revoke. The SDK refuses before it sends the request. + keyless = holder(aclients, 'my-authorized-uuid') + a.contains(failure(await keyless.revoke_token(token).future()), 'Secret key is not configured') + + # snippet.revoke_token + envelope = await pubnub.revoke_token(token).future() + # snippet.end + + a.eq(envelope.result.status, 200) + a.eq(str(envelope.result), 'Revoke token success with status: 200') + await aeventually(lambda: refused(holder_client, channel), timeout=100, interval=2, + what='the revoked token is refused') + for number in (1, 2): + await refused(holder(aclients, 'my-authorized-uuid', token), channel) + + +@step('grant_token_categories', 'parse_token_categories', role='pam') +async def test_token_categories(aclients): + pubnub = aclients.asyncio('pam', 'admin') + + # snippet.grant_token_categories + from pubnub.models.consumer.v3.category import Category + + envelope = await pubnub.grant_token( + categories=[Category.channels().get(), Category.uuids().get()], + authorized_user_id='my_authorized_uuid', + ttl=15 + ).future() + token = envelope.result.get_token() + # snippet.end + redact.register(token) + + # snippet.parse_token_categories + categories = pubnub.parse_token(token)['categories'] + # snippet.end + + a.eq(categories, {'channels': {'get': True}, 'uuids': {'get': True}}) + parsed = pubnub.parse_token(token) + a.eq(parsed['authorized_uuid'], names.with_suffix('my_authorized_uuid')) + a.eq(parsed['resources'].get('channels', {}), {}) + a.eq(parsed['resources'].get('uuids', {}), {}) + + # A token with a single category carries only that category. + single = await pubnub.grant_token().categories([Category.channels().get()]).ttl(15).future() + redact.register(single.result.token) + a.eq(pubnub.parse_token(single.result.token)['categories'], {'channels': {'get': True}}) + + # The SDK refuses an invalid category before it sends the request. + resolved = await pubnub.grant_token().categories([Channel.id('x').get()]).ttl(15).future() + a.contains(failure(resolved), 'Unsupported category. Only Category.channels() and Category.uuids() are supported') + resolved = await pubnub.grant_token().categories([Category.channels()]).ttl(15).future() + a.contains(failure(resolved), "Category permissions must be exactly the 'get' permission") + + +@step('grant_token_multi', 'grant_token_regex', 'grant_token_multi_regex', role='pam') +async def test_grant_token_resources(aclients): + pubnub = aclients.asyncio('pam', 'admin') + authorized = names.with_suffix('my-authorized-uuid') + + # snippet.grant_token_multi + from pubnub.models.consumer.v3.channel import Channel + from pubnub.models.consumer.v3.group import Group + from pubnub.models.consumer.v3.uuid import UUID + + channels = [ + Channel.id("channel-a").read(), + Channel.id("channel-b").read().write(), + Channel.id("channel-c").read().write(), + Channel.id("channel-d").read().write() + ] + groups = [ + Group.id("channel-group-b").read() + ] + uuids = [ + UUID.id("uuid-c").get(), + UUID.id("uuid-d").get().update() + ] + envelope = await pubnub.grant_token() \ + .channels(channels) \ + .groups(groups) \ + .ttl(15) \ + .uuids(uuids) \ + .authorized_uuid("my-authorized-uuid") \ + .future() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['authorized_uuid'], authorized) + a.eq(parsed['resources']['channels'], { + 'channel-a': perms('read'), + 'channel-b': perms('read', 'write'), + 'channel-c': perms('read', 'write'), + 'channel-d': perms('read', 'write'), + }) + a.eq(parsed['resources']['groups'], {'channel-group-b': perms('read')}) + a.eq(parsed['resources']['uuids'], {'uuid-c': perms('get'), 'uuid-d': perms('get', 'update')}) + a.eq(parsed['patterns'].get('channels', {}), {}) + + # snippet.grant_token_regex + envelope = await pubnub.grant_token() \ + .channels([Channel.pattern("channel-[A-Za-z0-9]").read()]) \ + .ttl(15) \ + .authorized_uuid("my-authorized-uuid") \ + .future() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['patterns']['channels'], {'channel-[A-Za-z0-9]': perms('read')}) + a.eq(parsed['resources'].get('channels', {}), {}) + + # snippet.grant_token_multi_regex + channels = [ + Channel.id("channel-a").read(), + Channel.pattern("channel-[A-Za-z0-9]").read(), + Channel.id("channel-b").read().write(), + Channel.id("channel-c").read().write(), + Channel.id("channel-d").read().write() + ] + groups = [ + Group.id("channel-group-b").read() + ] + uuids = [ + UUID.id("uuid-c").get(), + UUID.id("uuid-d").get().update() + ] + envelope = await pubnub.grant_token() \ + .channels(channels) \ + .ttl(15) \ + .groups(groups) \ + .uuids(uuids) \ + .authorized_uuid("my-authorized-uuid") \ + .future() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['patterns']['channels'], {'channel-[A-Za-z0-9]': perms('read')}) + a.eq(sorted(parsed['resources']['channels']), ['channel-a', 'channel-b', 'channel-c', 'channel-d']) + a.eq(parsed['resources']['groups'], {'channel-group-b': perms('read')}) + a.eq(parsed['resources']['uuids'], {'uuid-c': perms('get'), 'uuid-d': perms('get', 'update')}) + + +@step('grant_token_spaces_builder', 'grant_token_spaces_multi', 'grant_token_spaces_regex', + 'grant_token_spaces_multi_regex', role='pam') +async def test_grant_token_spaces_users(aclients): + pubnub = aclients.asyncio('pam', 'admin') + + # snippet.grant_token_spaces_builder + from pubnub.models.consumer.v3.space import Space + from pubnub.models.consumer.v3.user import User + + envelope = await pubnub.grant_token() \ + .spaces([ + Space.id("my_space").read().write().manage(), + ]) \ + .users([ + User.id("my_user").get().update(), + ]) \ + .ttl(15) \ + .authorized_user("my-authorized-userId") \ + .future() + token = envelope.result.token + # snippet.end + redact.register(token) + parsed = pubnub.parse_token(token) + a.eq(parsed['ttl'], 15) + a.eq(parsed['authorized_uuid'], 'my-authorized-userId') + a.eq(parsed['resources']['channels'], {'my_space': perms('read', 'write', 'manage')}) + a.eq(parsed['resources']['uuids'], {names.with_suffix('my_user'): perms('get', 'update')}) + + # snippet.grant_token_spaces_multi + spaces = [ + Space.id("space-a").read(), + Space.id("space-b").read().write(), + Space.id("space-c").read().write(), + Space.id("space-d").read().write() + ] + users = [ + User.id("userId-c").get(), + User.id("userId-d").get().update() + ] + envelope = await pubnub.grant_token() \ + .spaces(spaces) \ + .ttl(15) \ + .users(users) \ + .authorized_user("my-authorized-userId") \ + .future() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['resources']['channels'], { + 'space-a': perms('read'), + 'space-b': perms('read', 'write'), + 'space-c': perms('read', 'write'), + 'space-d': perms('read', 'write'), + }) + a.eq(parsed['resources']['uuids'], {'userId-c': perms('get'), 'userId-d': perms('get', 'update')}) + + # snippet.grant_token_spaces_regex + envelope = await pubnub.grant_token() \ + .spaces([Space.pattern("space-[A-Za-z0-9]").read()]) \ + .ttl(15) \ + .authorized_user("my-authorized-userId") \ + .future() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['patterns']['channels'], {'space-[A-Za-z0-9]': perms('read')}) + + # snippet.grant_token_spaces_multi_regex + spaces = [ + Space.id("space-a").read(), + Space.pattern("space-[A-Za-z0-9]").read(), + Space.id("space-b").read().write(), + Space.id("space-c").read().write(), + Space.id("space-d").read().write() + ] + users = [ + User.id("user-c").get(), + User.id("user-d").get().update() + ] + envelope = await pubnub.grant_token() \ + .spaces(spaces) \ + .ttl(15) \ + .users(users) \ + .authorized_user("my-authorized-userId") \ + .future() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['patterns']['channels'], {'space-[A-Za-z0-9]': perms('read')}) + a.eq(sorted(parsed['resources']['channels']), ['space-a', 'space-b', 'space-c', 'space-d']) + a.eq(parsed['resources']['uuids'], {'user-c': perms('get'), 'user-d': perms('get', 'update')}) diff --git a/docs-snippets/asyncio/access_manager_v2.py b/docs-snippets/asyncio/access_manager_v2.py new file mode 100644 index 00000000..0fdba5e3 --- /dev/null +++ b/docs-snippets/asyncio/access_manager_v2.py @@ -0,0 +1,320 @@ +"""Docs page: access-manager-v2 (asyncio), the legacy grant(). Same steps as sync/access_manager_v2.py with await. + +Every grant is closed in a finally (all flags false, ttl 1). The step that grants at application level is the last one +of this file. The asyncio client does not raise on an HTTP error, future() resolves with a PubNubAsyncioException. +""" +from pubnub.exceptions import PubNubAsyncioException + +from harness import assertions as a +from harness import names +from harness.listeners import alisten +from harness.step import step +from harness.wait import aeventually + +REGISTERED_NAMES = ['my_channel-pnpres', 'girish', 'girish2', 'cg2', 'cg3'] + +ALL_FALSE = ('read', 'write', 'manage', 'delete', 'get', 'update', 'join') + + +def holder(aclients, user_id, auth_key=None): + """A client that has no secret key, so every request is checked against its auth key.""" + client = aclients.asyncio('pam', user_id, secret=False) + client.config.auth_key = auth_key + return client + + +async def close_grant(admin, channels=None, groups=None, uuids=None, auth_keys=None): + """Take back what a step granted: every flag false, ttl 1.""" + builder = admin.grant() + for flag in ALL_FALSE: + builder = getattr(builder, flag)(False) + if channels: + builder = builder.channels(channels) + if groups: + builder = builder.channel_groups(groups) + if uuids: + builder = builder.uuids(uuids) + if auth_keys: + builder = builder.auth_keys(auth_keys) + return await builder.ttl(1).future() + + +async def publish(client, channel): + return await client.publish().channel(channel).message('docs').future() + + +async def refused(client, channel): + resolved = await publish(client, channel) + a.ok(isinstance(resolved, PubNubAsyncioException), 'the publish was refused') + a.eq(resolved.status.status_code, 403, 'status code of the refusal') + + +async def allowed(client, channel): + resolved = await publish(client, channel) + a.ok(not isinstance(resolved, PubNubAsyncioException), 'the publish went through') + a.timetoken(resolved.result.timetoken) + + +def good(resolved): + """A future() that resolved with a result, not with an error object. Returns the result.""" + a.ok(not isinstance(resolved, PubNubAsyncioException), 'the call succeeded') + return resolved.result + + +def failure(resolved): + a.ok(isinstance(resolved, PubNubAsyncioException), 'the call resolved with an error object') + return str(resolved.value()) + + +def key_flags(key_data): + return (key_data.read_enabled, key_data.write_enabled, key_data.manage_enabled, key_data.delete_enabled) + + +@step('grant_channel_auth', role='pam') +async def test_grant_channel_and_auth_key(aclients): + pubnub = aclients.asyncio('pam', 'admin') + channels = [names.with_suffix(name) for name in ('ch1', 'ch2', 'ch3')] + groups = [names.with_suffix(name) for name in ('cg1', 'cg2')] + other = names.with_suffix('ch_no_grant') + try: + # snippet.grant_channel_auth + envelope = await pubnub.grant().channels(["ch1", "ch2", "ch3"]).\ + channel_groups(["cg1", "cg2"]).\ + auth_keys(["key1", "key2"]).\ + read(True).write(True).manage(True).\ + future() + # snippet.end + + result = good(envelope) + a.eq(result.level, 'channel-group+auth') + a.eq(result.ttl, 1440) + a.ok(isinstance(result.subscribe_key, str) and result.subscribe_key, 'the result names the subscribe key') + a.eq(sorted(result.channels), sorted(channels)) + a.eq(sorted(result.groups), sorted(groups)) + for data in list(result.channels.values()) + list(result.groups.values()): + a.eq(sorted(data.auth_keys), ['key1', 'key2']) + for key_data in data.auth_keys.values(): + a.eq(key_flags(key_data)[:3], (True, True, True)) + await aeventually(lambda: allowed(holder(aclients, 'granted', 'key1'), channels[0]), timeout=30, + what='a granted auth key publishes') + await refused(holder(aclients, 'stranger', 'key3'), channels[0]) + await refused(holder(aclients, 'granted', 'key1'), other) + finally: + await close_grant(pubnub, channels, groups, auth_keys=['key1', 'key2']) + + +@step('grant_channel_all_users', 'grant_channel_level', role='pam') +async def test_grant_channel_level(aclients): + pubnub = aclients.asyncio('pam', 'admin') + channel = names.with_suffix('my_channel') + other = names.with_suffix('ch_no_grant') + try: + # snippet.grant_channel_all_users + envelope = await pubnub.grant().channels("my_channel").read(True).write(True).future() + # snippet.end + + result = good(envelope) + a.eq(result.level, 'channel') + a.eq(result.ttl, 1440) + a.eq((result.channels[channel].read_enabled, result.channels[channel].write_enabled), (True, True)) + client = holder(aclients, 'anyone') + await aeventually(lambda: allowed(client, channel), timeout=30, what='publish without an auth key') + await refused(client, other) + + # snippet.grant_channel_level + envelope = await pubnub.grant().channels("my_channel"). \ + read(True).write(True). \ + future() + # snippet.end + + result = good(envelope) + a.eq(result.level, 'channel') + a.eq(result.channels[channel].read_enabled, True) + finally: + await close_grant(pubnub, [channel]) + + +@step('grant_channel_ro_auth', 'grant_user_level', role='pam', timeout=900) +async def test_grant_user_level(aclients): + pubnub = aclients.asyncio('pam', 'admin') + channel = names.with_suffix('my_channel') + other = names.with_suffix('ch_no_grant') + try: + # snippet.grant_channel_ro_auth + envelope = await pubnub.grant().channels("my_channel").read(True).write(False).\ + auth_keys("my_ro_authkey").ttl(5).future() + # snippet.end + + result = good(envelope) + a.eq(result.level, 'user') + a.eq(result.ttl, 5) + a.eq(list(result.channels), [channel]) + a.eq(list(result.channels[channel].auth_keys), ['my_ro_authkey']) + key_data = result.channels[channel].auth_keys['my_ro_authkey'] + a.eq((key_data.read_enabled, key_data.write_enabled), (True, False)) + reader = holder(aclients, 'reader', 'my_ro_authkey') + # The read grant is proved by a subscribe that connects. The missing write grant by a refused publish. + await alisten(reader, channel) + await aeventually(lambda: refused(reader, channel), timeout=300, + what='reader write refused, a channel level downgrade can take minutes') + await aeventually(lambda: refused(holder(aclients, 'stranger', 'other_authkey'), channel), timeout=300, + what='a stranger auth key refused, a channel level downgrade can take minutes') + finally: + await close_grant(pubnub, [channel], auth_keys=['my_ro_authkey']) + + try: + # snippet.grant_user_level + envelope = await pubnub.grant().channels("my_channel"). \ + auth_keys("my_authkey"). \ + read(True).write(True). \ + ttl(5). \ + future() + # snippet.end + + result = good(envelope) + a.eq(result.level, 'user') + a.eq(key_flags(result.channels[channel].auth_keys['my_authkey'])[:2], (True, True)) + await aeventually(lambda: allowed(holder(aclients, 'writer', 'my_authkey'), channel), timeout=30, + what='publish with the granted auth key') + await aeventually(lambda: refused(holder(aclients, 'writer', 'my_authkey'), other), timeout=300, + what='a channel without a grant refused, a channel level downgrade can take minutes') + finally: + await close_grant(pubnub, [channel], auth_keys=['my_authkey']) + + +@step('grant_presence', role='pam') +async def test_grant_presence_channel(aclients): + pubnub = aclients.asyncio('pam', 'admin') + presence = names.with_suffix('my_channel-pnpres') + try: + # snippet.grant_presence + envelope = await pubnub.grant(). \ + channels("my_channel-pnpres").read(True).write(True).future() + # snippet.end + + result = good(envelope) + a.eq(result.level, 'channel') + a.eq(list(result.channels), [presence]) + a.eq((result.channels[presence].read_enabled, result.channels[presence].write_enabled), (True, True)) + client = holder(aclients, 'presence_reader') + + async def here_now_not_refused(): + resolved = await client.here_now().channels(presence).future() + a.ok(not isinstance(resolved, PubNubAsyncioException), 'here_now on the presence channel was refused') + + await aeventually(here_now_not_refused, timeout=30, what='here_now on the presence channel') + finally: + await close_grant(pubnub, [presence]) + + +@step('grant_channel_group', role='pam') +async def test_grant_channel_group(aclients): + pubnub = aclients.asyncio('pam', 'admin') + groups = [names.with_suffix(name) for name in ('cg1', 'cg2', 'cg3')] + member = names.with_suffix('group_member') + try: + # snippet.grant_channel_group + envelope = await pubnub.grant(). \ + channel_groups(["cg1", "cg2", "cg3"]). \ + auth_keys(["auth1", "auth2", "auth3"]). \ + read(True).write(True).manage(True). \ + ttl(12237). \ + future() + # snippet.end + + result = good(envelope) + a.eq(result.level, 'channel-group+auth') + a.eq(result.ttl, 12237) + a.eq(sorted(result.groups), sorted(groups)) + for data in result.groups.values(): + a.eq(sorted(data.auth_keys), ['auth1', 'auth2', 'auth3']) + for key_data in data.auth_keys.values(): + a.eq(key_flags(key_data)[:3], (True, True, True)) + manager = holder(aclients, 'group_manager', 'auth1') + + async def add_member(): + resolved = await manager.add_channel_to_channel_group().channels(member).channel_group(groups[0]).future() + a.ok(not isinstance(resolved, PubNubAsyncioException), 'the manage grant lets the auth key edit the group') + + await aeventually(add_member, timeout=30, what='the manage grant lets the auth key edit the group') + listed = good(await pubnub.list_channels_in_channel_group().channel_group(groups[0]).future()) + a.eq(list(listed.channels), [member]) + resolved = await holder(aclients, 'stranger', 'auth9').add_channel_to_channel_group() \ + .channels(member).channel_group(groups[1]).future() + a.ok(isinstance(resolved, PubNubAsyncioException), 'an auth key without a grant is refused') + a.eq(resolved.status.status_code, 403) + finally: + try: + await pubnub.remove_channel_group().channel_group(groups[0]).future() + finally: + await close_grant(pubnub, groups=groups, auth_keys=['auth1', 'auth2', 'auth3']) + + +@step('grant_multiple_channels', role='pam') +async def test_grant_multiple_channels(aclients): + pubnub = aclients.asyncio('pam', 'admin') + channels = [names.with_suffix('girish'), names.with_suffix('girish2')] + try: + # snippet.grant_multiple_channels + envelope = await pubnub.grant().\ + auth_keys("my_rw_authkey").\ + channels(["girish", "girish2"]).\ + read(True).\ + write(True).\ + ttl(5).\ + future() + # snippet.end + + result = good(envelope) + a.eq(result.level, 'user') + a.eq(result.ttl, 5) + a.eq(sorted(result.channels), sorted(channels)) + for data in result.channels.values(): + a.eq(list(data.auth_keys), ['my_rw_authkey']) + a.eq(key_flags(data.auth_keys['my_rw_authkey'])[:2], (True, True)) + client = holder(aclients, 'rw', 'my_rw_authkey') + for channel in channels: + await aeventually(lambda: allowed(client, channel), timeout=30, what='publish on a granted channel') + finally: + await close_grant(pubnub, channels, auth_keys=['my_rw_authkey']) + + +@step('grant_categories', role='pam') +async def test_grant_categories(aclients): + pubnub = aclients.asyncio('pam', 'admin') + auth_key = 'my_auth_key' + try: + # snippet.grant_categories + envelope = await pubnub.grant() \ + .auth_keys(auth_key) \ + .categories(['channels', 'uuids']) \ + .get(True) \ + .ttl(15) \ + .future() + print(f"Granted category permissions, level: {envelope.result.level}") + # snippet.end + + result = good(envelope) + a.eq(result.level, 'category') + a.eq(result.ttl, 15) + a.eq(sorted(result.categories), ['channels', 'uuids']) + for name, data in result.categories.items(): + a.eq(data.name, name) + a.eq(list(data.auth_keys), [auth_key]) + a.eq(data.auth_keys[auth_key].get, True) + a.eq(data.auth_keys[auth_key].read_enabled, False) + closed = good(await pubnub.grant().auth_keys(auth_key).categories(['channels']).get(False).future()) + a.eq(closed.categories['channels'].auth_keys[auth_key].get, False) + finally: + await pubnub.grant().auth_keys(auth_key).categories(['channels', 'uuids']).get(False).future() + + # The SDK refuses these before it sends the request. + a.contains(failure(await pubnub.grant().auth_keys(auth_key).categories(['groups']).get(True).ttl(15).future()), + "Unsupported categories: groups. Only 'channels' and 'uuids' are supported") + a.contains(failure(await pubnub.grant().categories(['channels']).get(True).ttl(15).future()), + 'Category grants require providing non empty authKeys') + a.contains(failure(await pubnub.grant().auth_keys(auth_key).categories(['channels']).read(True).get(True) + .ttl(15).future()), + "Category grants allow only the 'get' permission") + a.contains(failure(await pubnub.grant().auth_keys(auth_key).categories(['channels']).get(True).ttl(1).future()), + 'One-minute (ttl=1) category grants are not supported') diff --git a/docs-snippets/asyncio/channel_groups.py b/docs-snippets/asyncio/channel_groups.py new file mode 100644 index 00000000..7bdebe8b --- /dev/null +++ b/docs-snippets/asyncio/channel_groups.py @@ -0,0 +1,146 @@ +"""Docs page: channel groups (asyncio). Same steps as sync/channel_groups.py with await. + +Every step registers its group with the janitor and deletes it in a finally. +""" +from harness import assertions as a +from harness import names, world +from harness.listeners import alisten +from harness.step import step +from harness.wait import aeventually + +REGISTERED_NAMES = ['son', 'daughter', 'channel_group'] + + +async def members(client, group): + envelope = await client.list_channels_in_channel_group().channel_group(group).future() + return sorted(envelope.result.channels) + + +async def eventually_members(client, group, expected): + async def check(): + a.eq(await members(client, group), sorted(expected), 'channels in the group') + + await aeventually(check, timeout=30, what='group membership') + + +@step('add_channels_builder', 'add_channels_named', role='open') +async def test_add_channels(aclients): + pubnub = aclients.asyncio('open', 'grouper') + group = names.with_suffix('cg1') + channels = [names.with_suffix('ch1'), names.with_suffix('ch2')] + world.record('channel_group', group) + try: + # snippet.add_channels_builder + envelope = await pubnub.add_channel_to_channel_group() \ + .channels(["ch1", "ch2"]) \ + .channel_group("cg1") \ + .future() + # snippet.end + a.eq(envelope.status.is_error(), False) + a.eq(str(envelope.result), 'Channel successfully added') + await eventually_members(pubnub, group, channels) + + # A subscriber on the group receives a publish to a member channel + seen = await alisten(aclients.asyncio('open', 'member_listener'), groups=[group]) + await pubnub.publish().channel(channels[0]).message('to-member').future() + await aeventually(lambda: a.eq([m.message for m in seen.messages], ['to-member'], 'group message received')) + a.eq((seen.messages[0].channel, seen.messages[0].subscription), (channels[0], group)) + + await pubnub.remove_channel_from_channel_group().channels(channels).channel_group(group).future() + await eventually_members(pubnub, group, []) + + # snippet.add_channels_named + envelope = await pubnub.add_channel_to_channel_group(channels=["ch1", "ch2"], channel_group="cg1").future() + # snippet.end + a.eq(envelope.status.is_error(), False) + await eventually_members(pubnub, group, channels) + finally: + await pubnub.remove_channel_group().channel_group(group).future() + + +@step('list_channels_builder', 'list_channels_named', role='open') +async def test_list_channels(aclients): + pubnub = aclients.asyncio('open', 'lister') + group = names.with_suffix('cg1') + channels = [names.with_suffix('ch1'), names.with_suffix('ch2')] + world.record('channel_group', group) + try: + await pubnub.add_channel_to_channel_group().channels(channels).channel_group(group).future() + await eventually_members(pubnub, group, channels) + + # snippet.list_channels_builder + envelope = await pubnub.list_channels_in_channel_group() \ + .channel_group("cg1") \ + .future() + print(envelope.result.channels) + # snippet.end + a.eq(sorted(envelope.result.channels), channels) + a.eq(type(envelope.result.channels), list, 'channels is a list, not a dictionary') + + # snippet.list_channels_named + envelope = await pubnub.list_channels_in_channel_group(channel_group="cg1").future() + # snippet.end + a.eq(sorted(envelope.result.channels), channels) + a.eq(await members(pubnub, names.with_suffix('missing_group')), [], 'an unknown group is empty') + finally: + await pubnub.remove_channel_group().channel_group(group).future() + + +@step('remove_channels_builder', 'remove_channels_named', role='open') +async def test_remove_channels(aclients): + pubnub = aclients.asyncio('open', 'remover') + group = names.with_suffix('channel_group') + kids = [names.with_suffix('son'), names.with_suffix('daughter')] + pair = [names.with_suffix('ch1'), names.with_suffix('ch2')] + other_group = names.with_suffix('cg1') + world.record('channel_group', group) + world.record('channel_group', other_group) + try: + await pubnub.add_channel_to_channel_group().channels(kids + pair).channel_group(group).future() + await pubnub.add_channel_to_channel_group().channels(pair).channel_group(other_group).future() + await eventually_members(pubnub, group, kids + pair) + + # snippet.remove_channels_builder + envelope = await pubnub.remove_channel_from_channel_group() \ + .channels(["son", "daughter"]) \ + .channel_group("channel_group") \ + .future() + # snippet.end + a.eq(envelope.status.is_error(), False) + a.eq(str(envelope.result), 'Channel successfully removed') + await eventually_members(pubnub, group, pair) + + # snippet.remove_channels_named + envelope = await pubnub.remove_channel_from_channel_group(channels=["ch1", "ch2"], + channel_group="cg1").future() + # snippet.end + a.eq(envelope.status.is_error(), False) + await eventually_members(pubnub, other_group, []) + a.eq(await members(pubnub, group), sorted(pair), 'the other group is untouched') + finally: + await pubnub.remove_channel_group().channel_group(group).future() + await pubnub.remove_channel_group().channel_group(other_group).future() + + +@step('delete_channel_group_builder', role='open') +async def test_delete_channel_group(aclients): + pubnub = aclients.asyncio('open', 'deleter') + group = names.with_suffix('cg1') + channels = [names.with_suffix('ch1'), names.with_suffix('ch2')] + world.record('channel_group', group) + try: + await pubnub.add_channel_to_channel_group().channels(channels).channel_group(group).future() + await eventually_members(pubnub, group, channels) + + # snippet.delete_channel_group_builder + envelope = await pubnub.remove_channel_group() \ + .channel_group("cg1") \ + .future() + # snippet.end + a.eq(envelope.status.is_error(), False) + a.eq(str(envelope.result), 'Group successfully removed') + await eventually_members(pubnub, group, []) + # The docs show remove_channel_group(channel_group="cg1"), but the method takes no argument + a.raises(TypeError, lambda: pubnub.remove_channel_group(channel_group=group)) + finally: + await pubnub.remove_channel_group().channel_group(group).future() diff --git a/docs-snippets/asyncio/configuration.py b/docs-snippets/asyncio/configuration.py new file mode 100644 index 00000000..073b0d24 --- /dev/null +++ b/docs-snippets/asyncio/configuration.py @@ -0,0 +1,321 @@ +"""Docs page: configuration (asyncio). Source: docs/sdks/asyncio/api-reference/configuration.md. + +The initialization samples use made up keys, so each step overwrites the keys on the same config object after the +sample ran and then proves the client works against the service. Validation branches replace the network call of the +request handler with a counter, so they prove that no request was sent. +""" +import asyncio +import re +from os import environ +from unittest import mock + +from pubnub.pubnub_asyncio import AsyncioSubscriptionManager, PubNubAsyncio +from pubnub.request_handlers.async_httpx import AsyncHttpxRequestHandler + +from harness import assertions as a +from harness import names, world +from harness.listeners import alisten +from harness.step import step +from harness.wait import aeventually + +USER_ID_ERROR = 'UUID missing or invalid type' # pubnub/pnconfiguration.py, validate_not_empty_string +UNREACHABLE = '127.0.0.1:1' + + +def real_keys(config): + """Copy the keyset of the open role onto a config the sample built with made up keys.""" + real = world.config('open') + config.subscribe_key = real.subscribe_key + config.publish_key = real.publish_key + + +async def works(aclients, pubnub, config): + """Overwrite the made up keys and prove the client talks to the service. Returns the time() envelope.""" + aclients._made.append(pubnub) + real_keys(config) + envelope = await pubnub.time().future() + a.ok(not envelope.status.is_error(), 'time() succeeded') + a.timetoken(str(envelope.result)) + return envelope + + +class RequestCounter: + """Replaces the network call of the request handler so that a validation branch can prove nothing was sent.""" + + def __init__(self): + self.calls = 0 + + async def __call__(self, *args, **kwargs): + self.calls += 1 + raise RuntimeError('a request was sent') + + +@step('configuration_full', role='open') +async def test_full_configuration(aclients): + real = world.config('open') + saved = {k: environ.get(k) for k in ('PUBNUB_SUBSCRIBE_KEY', 'PUBNUB_PUBLISH_KEY', 'PUBNUB_USER_ID')} + environ.update({'PUBNUB_SUBSCRIBE_KEY': real.subscribe_key, 'PUBNUB_PUBLISH_KEY': real.publish_key, + 'PUBNUB_USER_ID': names.with_suffix('configured_user')}) + try: + # snippet.configuration_full + import os + from pubnub.pnconfiguration import PNConfiguration + from pubnub.enums import PNHeartbeatNotificationOptions + + # Configuration for the client instance + pn_configuration = PNConfiguration() + + # Set configuration values + pn_configuration.subscribe_key = os.getenv('PUBNUB_SUBSCRIBE_KEY', 'demo') # required + pn_configuration.publish_key = os.getenv('PUBNUB_PUBLISH_KEY', 'demo') # only required if publishing + pn_configuration.secret_key = os.getenv('PUBNUB_SECRET_KEY', 'my_secret_key') # optional + pn_configuration.user_id = os.getenv('PUBNUB_USER_ID', 'my_custom_user_id') # must be set + pn_configuration.auth_key = os.getenv('PUBNUB_AUTH_KEY', 'my_auth_key') # optional + pn_configuration.ssl = True + pn_configuration.filter_expression = "such == 'wow'" + pn_configuration.heartbeat_notification_options = PNHeartbeatNotificationOptions.ALL + + # Adding the requested parameters + pn_configuration.enable_subscribe = True # required if you want to subscribe + pn_configuration.enable_presence_heartbeat = True # sends presence heartbeats for subscribed channels + + # The configuration is ready to pass to PubNub + # Simply create a PubNub instance using it: + pubnub = PubNubAsyncio(pn_configuration) + + print("Configuration set up complete.") + # snippet.end + finally: + for key, value in saved.items(): + if value is None: + environ.pop(key, None) + else: + environ[key] = value + aclients._made.append(pubnub) + a.eq(pn_configuration.subscribe_key, real.subscribe_key, 'keys come from the environment') + a.eq(pubnub.uuid, names.with_suffix('configured_user')) + a.eq(pn_configuration.heartbeat_notification_options, PNHeartbeatNotificationOptions.ALL) + a.eq((pn_configuration.enable_subscribe, pn_configuration.enable_presence_heartbeat), (True, True)) + a.eq((pn_configuration.auth_key, pn_configuration.filter_expression), ('my_auth_key', "such == 'wow'")) + + # The configured client publishes, and a plain client on the same channel reads the message + channel = names.with_suffix('my_channel') + seen = await alisten(aclients.asyncio('open', 'config_reader'), channel) + envelope = await pubnub.publish().channel(channel).message({'configured': True}).future() + a.timetoken(envelope.result.timetoken) + + async def received(): + a.eq([m.message for m in seen.messages], [{'configured': True}], 'reader received') + await aeventually(received) + + +@step('initialize_pubnub', 'initialize_non_secure', 'initialize_ssl', 'initialize_custom_user_id', role='open') +async def test_initialize(aclients): + # snippet.initialize_pubnub + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub_asyncio import PubNubAsyncio + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.ssl = True + pnconfig.user_id = "my_custom_user_id" + pubnub = PubNubAsyncio(pnconfig) + # snippet.end + a.eq((pubnub.uuid, pnconfig.ssl), ('my_custom_user_id', True)) + a.ok((await works(aclients, pubnub, pnconfig)).status.tls_enabled, 'requests use TLS') + + # snippet.initialize_non_secure + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub_asyncio import PubNubAsyncio + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.ssl = False + pnconfig.user_id = "my_custom_user_id" + pubnub = PubNubAsyncio(pnconfig) + # snippet.end + a.eq(pnconfig.scheme(), 'http') + a.eq((await works(aclients, pubnub, pnconfig)).status.tls_enabled, False, 'requests are plain http') + + # snippet.initialize_ssl + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub_asyncio import PubNubAsyncio + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.ssl = True + pnconfig.user_id = "my_custom_user_id" + pubnub = PubNubAsyncio(pnconfig) + # snippet.end + a.eq(pnconfig.scheme_and_host(), 'https://ps.pndsn.com') + a.ok((await works(aclients, pubnub, pnconfig)).status.tls_enabled, 'requests use TLS') + + # snippet.initialize_custom_user_id + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub_asyncio import PubNubAsyncio + + pnconfig = PNConfiguration() + + pnconfig.subscribe_key = 'mySubscribeKey' + pnconfig.publish_key = 'myPublishKey' + pnconfig.user_id = "my_custom_user_id" + + pubnub = PubNubAsyncio(pnconfig) + # snippet.end + a.eq(pubnub.uuid, 'my_custom_user_id') + a.eq(pnconfig.user_id, pnconfig.uuid, 'user_id and the deprecated uuid are the same value') + await works(aclients, pubnub, pnconfig) + + # The user ID is mandatory. The branches below never reach the network. + counter = RequestCounter() + with mock.patch.object(AsyncHttpxRequestHandler, '_request_with_wall_clock_deadline', counter): + missing = PNConfiguration() + missing.subscribe_key = 'mySubscribeKey' + missing.origin = UNREACHABLE + a.raises(AssertionError, PubNubAsyncio, missing, match=re.escape(USER_ID_ERROR)) + for bad in ('', ' ', None, 42): + a.raises(AssertionError, setattr, missing, 'user_id', bad, match=re.escape(USER_ID_ERROR)) + a.eq(missing.user_id, None, 'a rejected user_id is not stored') + a.eq(counter.calls, 0, 'no request was sent') + + +@step('initialize_custom_request_handler', role='open') +async def test_initialize_custom_request_handler(aclients): + # snippet.initialize_custom_request_handler + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub_asyncio import PubNubAsyncio + from pubnub.request_handlers.async_aiohttp import AsyncAiohttpRequestHandler + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.ssl = True + pnconfig.user_id = "my_custom_user_id" + pubnub = PubNubAsyncio(pnconfig, custom_request_handler=AsyncAiohttpRequestHandler) + # snippet.end + a.ok(isinstance(pubnub._request_handler, AsyncAiohttpRequestHandler), 'the custom handler is in use') + a.ok(not isinstance(pubnub._request_handler, AsyncHttpxRequestHandler), 'the default handler is replaced') + await works(aclients, pubnub, pnconfig) + a.raises(Exception, PubNubAsyncio, pnconfig, custom_request_handler=dict, match='subclass of BaseRequestHandler') + + +@step('initialize_read_only', role='open') +async def test_initialize_read_only(aclients): + channel = names.with_suffix('my_channel') + + # snippet.initialize_read_only + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub_asyncio import PubNubAsyncio + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.user_id = "my_custom_user_id" + + pubnub = PubNubAsyncio(pnconfig) + # snippet.end + aclients._made.append(pubnub) + a.eq(pnconfig.publish_key, None, 'no publish key') + + counter = RequestCounter() + pnconfig.origin = UNREACHABLE + with mock.patch.object(AsyncHttpxRequestHandler, '_request_with_wall_clock_deadline', counter): + failed = await pubnub.publish().channel(channel).message('x').future() + a.ok(failed.is_error(), 'publishing without a publish key fails') + a.eq(str(failed.status.error_data.exception), 'Publish key not configured', 'exact error text') + a.eq(counter.calls, 0, 'no request was sent') + + # It still reads: the same config with the real subscribe key receives a message another client published + pnconfig.origin = world.config('open').origin + pnconfig.subscribe_key = world.config('open').subscribe_key + seen = await alisten(pubnub, channel) + await aclients.asyncio('open', 'writer').publish().channel(channel).message('read-me').future() + + async def received(): + a.eq([m.message for m in seen.messages], ['read-me'], 'the read-only client received') + await aeventually(received) + + +@step('initialize_access_manager', role='open') +async def test_initialize_access_manager(aclients): + # snippet.initialize_access_manager + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub_asyncio import PubNubAsyncio + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.secret_key = "my_secretkey" + pnconfig.user_id = "my_custom_user_id" + pnconfig.ssl = True + + pubnub = PubNubAsyncio(pnconfig) + # snippet.end + a.eq(pnconfig.secret_key, 'my_secretkey') + aclients._made.append(pubnub) + + counter = RequestCounter() + pnconfig.origin = UNREACHABLE + with mock.patch.object(AsyncHttpxRequestHandler, '_request_with_wall_clock_deadline', counter): + with mock.patch('warnings.warn'): + with_secret = await pubnub.grant().channels('x').read(True).future() + a.eq(counter.calls, 1, 'the grant call reaches the network layer once the secret key is configured') + a.ok(with_secret.is_error(), 'the stand-in refused the request') + keyless = aclients.asyncio('open', 'no_secret') + without = await keyless.grant().channels('x').read(True).future() + a.eq(str(without.status.error_data.exception), 'Secret key is not configured', 'exact error text') + a.eq(counter.calls, 1, 'without a secret key no request is built') + + +@step('initialize_custom_event_loop', role='open') +async def test_initialize_custom_event_loop(aclients): + my_event_loop = asyncio.get_running_loop() + + # snippet.initialize_custom_event_loop + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub_asyncio import PubNubAsyncio + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.user_id = "my_custom_user_id" + + pubnub = PubNubAsyncio(pnconfig, custom_event_loop=my_event_loop) + # snippet.end + a.ok(pubnub.event_loop is my_event_loop, 'the client runs on the given loop') + await works(aclients, pubnub, pnconfig) + + +@step('set_filter_expression', 'get_filter_expression', role='open') +async def test_filter_expression(aclients): + channel = names.with_suffix('my_channel') + + # snippet.set_filter_expression + from pubnub.pnconfiguration import PNConfiguration + + pnconfig = PNConfiguration() + pnconfig.filter_expression = "such == 'wow'" + # snippet.end + real_keys(pnconfig) + pnconfig.user_id = names.with_suffix('filtered') + # The default event engine subscribe loop ignores filter_expression (SDK bug, see the report), so this step uses the + # documented subscription_manager argument of the constructor to select the loop that sends the filter. + filtered = PubNubAsyncio(pnconfig, subscription_manager=AsyncioSubscriptionManager) + aclients._made.append(filtered) + seen = await alisten(filtered, channel) + publisher = aclients.asyncio('open', 'filter_publisher') + await publisher.publish().channel(channel).message('no-meta').future() + await publisher.publish().channel(channel).message('other').meta({'such': 'meh'}).future() + await publisher.publish().channel(channel).message('matching').meta({'such': 'wow'}).future() + + async def only_matching(): + a.eq([m.message for m in seen.messages], ['matching'], 'only the matching message arrived') + await aeventually(only_matching) + + # snippet.get_filter_expression + filter = pnconfig.filter_expression + # snippet.end + a.eq(filter, "such == 'wow'") + a.eq(world.config('open').filter_expression, None, 'the default is no filter') diff --git a/docs-snippets/asyncio/file_sharing.py b/docs-snippets/asyncio/file_sharing.py new file mode 100644 index 00000000..f6eba40a --- /dev/null +++ b/docs-snippets/asyncio/file_sharing.py @@ -0,0 +1,248 @@ +"""Docs page: file sharing (asyncio). Source: docs/sdks/asyncio/api-reference/files.md. Same contract as sync.""" +import asyncio +import os +import tempfile +import urllib.request +from pathlib import Path + +from harness import assertions as a +from harness import names +from harness.step import step +from harness.wait import aeventually + +FIXTURE = Path(__file__).resolve().parent.parent / 'fixtures' / 'sample.gif' +FIXTURE_BYTES = FIXTURE.read_bytes() + + +def channel_name(): + return names.with_suffix('test_channel') + + +REGISTERED_NAMES = ['test_channel'] + + +async def upload(pubnub, store=True): + """Seed a file outside the snippet markers. Returns the PNSendFileResult.""" + with open(FIXTURE, 'rb') as sample_file: + envelope = await pubnub.send_file().channel(channel_name()).file_name('sample.gif') \ + .file_object(sample_file).should_store(store).future() + return envelope.result + + +async def remove(pubnub, *uploaded): + for item in uploaded: + try: + await pubnub.delete_file().channel(channel_name()).file_id(item.file_id).file_name(item.name).future() + except Exception as e: # cleanup must never hide the real failure + print('cleanup: could not delete a file (' + type(e).__name__ + ')') + + +async def stored_file_messages(pubnub, *file_ids): + """File events read back from Message Persistence, because the SDK cannot deliver file events to listeners.""" + envelope = await pubnub.fetch_messages().channels(channel_name()).include_meta(True).future() + items = envelope.result.channels.get(channel_name(), []) + files = [item for item in items if isinstance(item.message, dict) and 'file' in item.message] + return [item for item in files if not file_ids or item.message['file']['id'] in file_ids] + + +async def listed_ids(pubnub): + envelope = await pubnub.list_files().channel(channel_name()).future() + return [item['id'] for item in envelope.result.data] + + +def fetch(url): + with urllib.request.urlopen(url, timeout=30) as served: + return served.status, served.read() + + +@step('send_file_builder', 'send_file_options', role='open') +async def test_send_file(aclients): + pubnub = aclients.asyncio('open', 'file_sender') + sent = [] + try: + # snippet.send_file_builder + with open(str(FIXTURE), "rb") as fd: + envelope = await pubnub.send_file() \ + .channel("test_channel") \ + .file_name("knights_of_ni.jpg") \ + .message({"test_message": "test"}) \ + .should_store(True) \ + .file_object(fd) \ + .future() + + if envelope.status.is_error(): + print("Error sending file") + else: + print(f"File sent successfully: {envelope.result.name}") + # snippet.end + sent.append(envelope.result) + a.ok(not envelope.status.is_error(), 'status is not an error') + a.eq(envelope.result.name, 'knights_of_ni.jpg') + a.ok(envelope.result.file_id, 'file id returned') + a.timetoken(envelope.result.timestamp) + + # snippet.send_file_options + with open(str(FIXTURE), "rb") as fd: + envelope_2 = await pubnub.send_file() \ + .channel("test_channel") \ + .file_name("knights_of_ni.jpg") \ + .message({"test_message": "test"}) \ + .should_store(True) \ + .ttl(222) \ + .file_object(fd) \ + .future() + # snippet.end + sent.append(envelope_2.result) + both = (envelope.result.file_id, envelope_2.result.file_id) + a.ok(both[0] != both[1], 'every upload gets its own id') + + async def two_stored(): + a.eq(len(await stored_file_messages(pubnub, *both)), 2, 'file events stored') + await aeventually(two_stored) + stored = {m.message['file']['id']: m for m in await stored_file_messages(pubnub, *both)} + a.eq(stored[both[0]].message['message'], {'test_message': 'test'}) + a.eq(stored[both[0]].message['file']['name'], 'knights_of_ni.jpg') + a.eq(str(stored[both[0]].timetoken), str(envelope.result.timestamp), 'the message carries the timestamp') + + async def listed(): + a.contains(await listed_ids(pubnub), both[0], 'listed after upload') + await aeventually(listed) + listing = (await pubnub.list_files().channel(channel_name()).future()).result.data + entry = [item for item in listing if item['id'] == both[0]][0] + a.eq(entry['size'], len(FIXTURE_BYTES), 'listed size is the fixture size') + finally: + await remove(pubnub, *sent) + + +@step('list_files_builder', role='open') +async def test_list_files(aclients): + pubnub = aclients.asyncio('open', 'file_lister') + first, second = await upload(pubnub), await upload(pubnub) + try: + async def both_listed(): + a.contains(await listed_ids(pubnub), second.file_id, 'second upload listed') + await aeventually(both_listed) + + # snippet.list_files_builder + envelope = await pubnub.list_files().channel("test_channel").limit(1).future() + next_page = await pubnub.list_files().channel("test_channel").limit(1).next(envelope.result.next).future() + # snippet.end + a.eq(len(envelope.result.data), 1, 'first page has the limit') + a.eq(len(next_page.result.data), 1, 'second page has the limit') + a.eq({envelope.result.data[0]['id'], next_page.result.data[0]['id']}, {first.file_id, second.file_id}) + a.ok({'id', 'name', 'size', 'created'} <= set(envelope.result.data[0]), 'file entry keys') + everything = await pubnub.list_files().channel(channel_name()).future() + a.eq(everything.result.count, len(everything.result.data), 'count matches the page') + finally: + await remove(pubnub, first, second) + + +@step('get_file_url_builder', role='open') +async def test_get_file_url(aclients): + pubnub = aclients.asyncio('open', 'file_url_reader') + uploaded = await upload(pubnub) + try: + # snippet.get_file_url_builder + envelope = await pubnub.get_file_url(). \ + channel("test_channel"). \ + file_id(uploaded.file_id). \ + file_name(uploaded.name).future() + # snippet.end + url = envelope.result.file_url + a.ok(url.startswith('https://'), 'url scheme') + a.contains(url, uploaded.file_id) + status, body = await asyncio.get_running_loop().run_in_executor(None, fetch, url) + a.eq(status, 200, 'the url serves the file') + a.eq(body, FIXTURE_BYTES, 'the url serves the same bytes') + finally: + await remove(pubnub, uploaded) + + +@step('download_file_builder', role='open') +async def test_download_file(aclients): + pubnub = aclients.asyncio('open', 'file_downloader') + uploaded = await upload(pubnub) + target = tempfile.mkdtemp(prefix='docs-snippets-') + save_path = os.path.join(target, uploaded.name) + try: + # snippet.download_file_builder + download_envelope = await pubnub.download_file(). \ + channel("test_channel"). \ + file_id(uploaded.file_id). \ + file_name(uploaded.name).future() + + with open(save_path, "wb") as fw: + fw.write(download_envelope.result.data) + # snippet.end + a.eq(download_envelope.result.data, FIXTURE_BYTES, 'downloaded bytes equal the upload') + with open(save_path, 'rb') as saved: + a.eq(saved.read(), FIXTURE_BYTES, 'saved bytes equal the upload') + finally: + await remove(pubnub, uploaded) + if os.path.exists(save_path): + os.remove(save_path) + os.rmdir(target) + + +@step('delete_file_builder', role='open') +async def test_delete_file(aclients): + pubnub = aclients.asyncio('open', 'file_deleter') + one, two = await upload(pubnub), await upload(pubnub) + remaining = [one, two] + try: + async def both_listed(): + a.contains(await listed_ids(pubnub), two.file_id, 'uploads listed') + await aeventually(both_listed) + + # snippet.delete_file_builder + envelope = await pubnub.delete_file(). \ + channel("test_channel"). \ + file_id(one.file_id). \ + file_name(one.name).future() + # snippet.end + remaining.remove(one) + a.eq(envelope.result.status, 200, 'delete status') + + async def gone(): + a.not_contains(await listed_ids(pubnub), one.file_id, 'deleted file still listed') + await aeventually(gone) + a.contains(await listed_ids(pubnub), two.file_id, 'the other file is untouched') + missing = await pubnub.download_file().channel(channel_name()).file_id(one.file_id) \ + .file_name(one.name).future() + a.ok(missing.is_error(), 'a deleted file cannot be downloaded') + a.eq(missing.status.status_code, 404, 'the service answers not found') + finally: + await remove(pubnub, *remaining) + + +@step('publish_file_message_builder', role='open') +async def test_publish_file_message(aclients): + pubnub = aclients.asyncio('open', 'file_publisher') + uploaded = await upload(pubnub) + try: + async def uploaded_stored(): + a.eq(len(await stored_file_messages(pubnub, uploaded.file_id)), 1, 'upload stored a message') + await aeventually(uploaded_stored) + + # snippet.publish_file_message_builder + envelope = await pubnub.publish_file_message(). \ + channel("test_channel"). \ + meta({"test": "test"}). \ + message({"test_message": "test"}). \ + file_id(uploaded.file_id). \ + file_name(uploaded.name). \ + should_store(True). \ + ttl(22).future() + # snippet.end + a.timetoken(envelope.result.timestamp) + + async def message_stored(): + a.eq(len(await stored_file_messages(pubnub, uploaded.file_id)), 2, 'file message stored') + await aeventually(message_stored) + stored = await stored_file_messages(pubnub, uploaded.file_id) + event = [m for m in stored if str(m.timetoken) == str(envelope.result.timestamp)][0] + a.eq(event.message['file'], {'id': uploaded.file_id, 'name': uploaded.name}) + a.eq(event.message['message'], {'test_message': 'test'}) + a.eq(event.meta, {'test': 'test'}) + finally: + await remove(pubnub, uploaded) diff --git a/docs-snippets/asyncio/message_actions.py b/docs-snippets/asyncio/message_actions.py new file mode 100644 index 00000000..0d5acbb7 --- /dev/null +++ b/docs-snippets/asyncio/message_actions.py @@ -0,0 +1,199 @@ +"""Docs page: message actions (asyncio). Same steps as sync/message_actions.py with await. + +A message action needs a stored message, so every step waits until the published message is readable from history. +""" +from pubnub.models.consumer.message_actions import PNMessageAction + +from harness import assertions as a +from harness import names +from harness.listeners import alisten +from harness.step import step +from harness.wait import aeventually + +REGISTERED_NAMES = ['chats.room1'] + + +async def publish_stored(client, channel, count=1): + """Publish count messages and wait until history returns them. Returns their timetokens, oldest first.""" + tokens = [] + for i in range(count): + envelope = await client.publish().channel(channel).message({'n': i}).future() + tokens.append(int(envelope.result.timetoken)) + + async def stored(): + got = await client.fetch_messages(channels=[channel], start=tokens[-1] + 1, end=tokens[0]).future() + a.eq([int(i.timetoken) for i in got.result.channels.get(channel, [])], tokens, 'stored timetokens') + + await aeventually(stored, timeout=30, what='published messages stored') + return tokens + + +async def add_action(client, channel, message_timetoken, value, action_type='reaction'): + action = PNMessageAction() + action.type = action_type + action.value = value + action.message_timetoken = message_timetoken + return (await client.add_message_action().channel(channel).message_action(action).future()).result + + +async def remove_action(client, channel, message_timetoken, action_timetoken): + await client.remove_message_action(channel=channel, message_timetoken=message_timetoken, + action_timetoken=int(action_timetoken)).future() + + +@step('add_message_action_builder', 'add_message_action_named', role='open') +async def test_add_message_action(aclients): + pubnub = aclients.asyncio('open', 'reactor') + channel = names.with_suffix('chats.room1') + seen = await alisten(aclients.asyncio('open', 'watcher'), channel) + message_timetoken = (await publish_stored(pubnub, channel))[0] + + # snippet.add_message_action_builder + # snippet.hide + from pubnub.models.consumer.message_actions import PNMessageAction + # snippet.show + msg_action = PNMessageAction() + msg_action.type = "reaction" + msg_action.value = "smiley_face" + msg_action.message_timetoken = message_timetoken + + envelope = await pubnub.add_message_action() \ + .channel("chats.room1") \ + .message_action(msg_action) \ + .future() + + print(f"Added action at timetoken: {envelope.result.action_timetoken}") + # snippet.end + + added = envelope.result + a.eq(envelope.status.is_error(), False) + a.timetoken(added.action_timetoken) + a.eq((added.type, added.value, added.uuid), ('reaction', 'smiley_face', pubnub.uuid)) + a.eq(int(added.message_timetoken), message_timetoken) + await aeventually(lambda: a.eq(len(seen.message_actions), 1, 'action events received')) + event = seen.message_actions[0] + a.eq((event.type, event.value, event.uuid), ('reaction', 'smiley_face', pubnub.uuid)) + a.eq(event.action_timetoken, added.action_timetoken) + a.eq(int(event.message_timetoken), message_timetoken) + a.eq(event.channel, channel) + + # snippet.add_message_action_named + msg_action = PNMessageAction() + msg_action.type = "reaction" + msg_action.value = "thumbs_up" + msg_action.message_timetoken = message_timetoken + + envelope = await pubnub.add_message_action( + channel="chats.room1", + message_action=msg_action + ).future() + + print(f"Added action at timetoken: {envelope.result.action_timetoken}") + # snippet.end + a.eq(envelope.result.value, 'thumbs_up') + + async def read_back(): + got = await pubnub.fetch_messages(channels=[channel], count=1, include_message_actions=True).future() + actions = got.result.channels[channel][0].actions + a.eq(sorted(actions['reaction']), ['smiley_face', 'thumbs_up'], 'fetch shows both actions') + a.eq(actions['reaction']['smiley_face'][0]['actionTimetoken'], added.action_timetoken) + a.eq(actions['reaction']['smiley_face'][0]['uuid'], pubnub.uuid) + + await aeventually(read_back, timeout=30, what='actions visible through fetch_messages') + await aeventually(lambda: a.eq(len(seen.message_actions), 2, 'second action event received')) + for action_timetoken in (added.action_timetoken, envelope.result.action_timetoken): + await remove_action(pubnub, channel, message_timetoken, action_timetoken) + + +@step('get_message_actions_builder', 'get_message_actions_named', role='open') +async def test_get_message_actions(aclients): + pubnub = aclients.asyncio('open', 'reader') + channel = names.with_suffix('chats.room1') + tokens = await publish_stored(pubnub, channel, 3) + first, second, third = [(await add_action(pubnub, channel, tt, 'smiley_face')).action_timetoken for tt in tokens] + a.ok(first < second < third, 'action timetokens grow') + + async def all_visible(): + got = (await pubnub.get_message_actions(channel=channel).future()).result.actions + mine = [x.action_timetoken for x in got if int(x.message_timetoken) in tokens] + a.eq(mine, [first, second, third], 'ascending by action timetoken') + + await aeventually(all_visible, timeout=30, what='actions listed') + start, end = str(third), str(second) + + # snippet.get_message_actions_builder + envelope = await pubnub.get_message_actions() \ + .channel("chats.room1") \ + .start(start) \ + .end(end) \ + .limit("50") \ + .future() + + for action in envelope.result.actions: + print(f"Message Action type: {action.type}") + print(f"Message Action value: {action.value}") + print(f"Message Action timetoken: {action.message_timetoken}") + print(f"Message Action uuid: {action.uuid}") + print(f"Message Action timetoken: {action.action_timetoken}") + # snippet.end + + built = envelope.result.actions + a.eq(len(built) > 0 and built[-1].uuid, pubnub.uuid) + + # snippet.get_message_actions_named + envelope = await pubnub.get_message_actions(channel="chats.room1", start=start, end=end, limit="50").future() + # snippet.end + named = envelope.result.actions + limited = (await pubnub.get_message_actions(channel=channel, limit="2").future()).result.actions + a.eq(len(limited), 2, 'limit caps the list') + for token, action_timetoken in zip(tokens, (first, second, third)): + await remove_action(pubnub, channel, token, action_timetoken) + # Last on purpose: the SDK sends no end boundary when a start boundary is set (get_message_actions.py:50). + a.eq([x.action_timetoken for x in built], [second], 'builder: start excluded, end included') + a.eq(int(built[0].message_timetoken), tokens[1]) + a.eq([x.action_timetoken for x in named], [second], 'named arguments: same range') + + +@step('remove_message_action_builder', 'remove_message_action_named', role='open') +async def test_remove_message_action(aclients): + pubnub = aclients.asyncio('open', 'remover') + channel = names.with_suffix('chats.room1') + seen = await alisten(aclients.asyncio('open', 'watcher'), channel) + message_timetoken = (await publish_stored(pubnub, channel))[0] + first = (await add_action(pubnub, channel, message_timetoken, 'smiley_face')).action_timetoken + second = (await add_action(pubnub, channel, message_timetoken, 'thumbs_up')).action_timetoken + await aeventually(lambda: a.eq(len(seen.message_actions), 2, 'added events received')) + + async def listed(expected): + got = (await pubnub.get_message_actions(channel=channel).future()).result.actions + a.eq([x.action_timetoken for x in got if int(x.message_timetoken) == message_timetoken], expected) + + await aeventually(lambda: listed([first, second]), timeout=30, what='both actions listed') + action_timetoken = int(first) + + # snippet.remove_message_action_builder + envelope = await pubnub.remove_message_action() \ + .channel("chats.room1") \ + .action_timetoken(action_timetoken) \ + .message_timetoken(message_timetoken) \ + .future() + # snippet.end + a.eq(envelope.status.is_error(), False) + a.eq(envelope.result, {}) + await aeventually(lambda: listed([second]), timeout=30, what='first action removed') + await aeventually(lambda: a.eq(len(seen.message_actions), 3, 'removed event received')) + a.eq(seen.message_actions[2].action_timetoken, first) + + action_timetoken = int(second) + # snippet.remove_message_action_named + envelope = await pubnub.remove_message_action(channel="chats.room1", + action_timetoken=action_timetoken, + message_timetoken=message_timetoken) \ + .future() + # snippet.end + a.eq(envelope.result, {}) + await aeventually(lambda: listed([]), timeout=30, what='second action removed') + await aeventually(lambda: a.eq(len(seen.message_actions), 4, 'second removed event received')) + a.eq(seen.message_actions[3].action_timetoken, second) + got = await pubnub.fetch_messages(channels=[channel], count=1, include_message_actions=True).future() + a.eq(got.result.channels[channel][0].actions, {}, 'history shows no actions') diff --git a/docs-snippets/asyncio/message_persistence.py b/docs-snippets/asyncio/message_persistence.py new file mode 100644 index 00000000..a6277b9f --- /dev/null +++ b/docs-snippets/asyncio/message_persistence.py @@ -0,0 +1,419 @@ +"""Docs page: message persistence (asyncio). Same steps as sync/message_persistence.py with await. + +Each step seeds its own messages and reads them back through bounds made of the seeded timetokens, so leftovers on a +shared channel name from an earlier run of this page (for example the sync one) never change a result. +""" +import asyncio +import contextlib +import io + +from harness import assertions as a +from harness import names +from harness.step import step +from harness.wait import aeventually + +REGISTERED_NAMES = ['history_channel_2', 'history_paging_channel_2'] + + +async def seed(client, channel, count, **extra): + """Publish count messages one after the other and return [(message, timetoken)] oldest first.""" + seeded = [] + for index in range(count): + message = {'n': index, 'channel': channel} + builder = client.publish().channel(channel).message(message) + if extra.get('meta'): + builder = builder.meta(extra['meta']) + if extra.get('custom_message_type'): + builder = builder.custom_message_type(extra['custom_message_type']) + seeded.append((message, int((await builder.future()).result.timetoken))) + return seeded + + +async def seed_parallel(client, channel, count): + """Publish count messages at the same time. Only the timetokens matter, they come back sorted.""" + async def one(index): + # The asyncio SDK returns a failed request as an error object (result None), it does not raise. A publish that + # failed in transit is repeated, any other outcome is reported with the status instead of an AttributeError. + for attempt in range(3): + envelope = await client.publish().channel(channel).message({'n': index}).future() + if not envelope.is_error(): + return int(envelope.result.timetoken) + last = envelope.status + raise AssertionError(f'publish {index} failed 3 times: category={last.category} error={last.error_data}') + + return sorted(await asyncio.gather(*[one(i) for i in range(count)])) + + +async def window(client, channel, newest, oldest, **kwargs): + """Fetch the seeded range: start is exclusive, so one above the newest. Returns the items oldest first.""" + envelope = await client.fetch_messages(channels=[channel], start=newest + 1, end=oldest, **kwargs).future() + return envelope.result.channels.get(channel, []) + + +async def wait_stored(client, channel, timetokens): + async def check(): + stored = [int(item.timetoken) for item in await window(client, channel, max(timetokens), min(timetokens))] + a.eq(stored, sorted(timetokens), 'stored timetokens') + + await aeventually(check, timeout=30, what='seeded messages stored') + + +@step('fetch_history_builder', 'fetch_history_named', role='open') +async def test_fetch_history(aclients): + pubnub = aclients.asyncio('open', 'fetcher') + channel = names.with_suffix('my_channel') + seeded = await seed(pubnub, channel, 3, meta={'origin': 'docs'}, custom_message_type='text-message') + await wait_stored(pubnub, channel, [tt for _, tt in seeded]) + last_message, last_timetoken = seeded[-1] + + # snippet.fetch_history_builder + envelope = await pubnub.fetch_messages() \ + .channels(["my_channel"]) \ + .maximum_per_channel(1) \ + .include_message_actions(True) \ + .include_meta(True) \ + .include_message_type(True) \ + .include_uuid(True) \ + .future() + + for message in envelope.result.channels["my_channel"]: + print("Message: %s" % message.message) + print("Meta: %s" % message.meta) + print("Timetoken: %s" % message.timetoken) + + for action_type in message.actions: + print("Message Action type: " + action_type) + # snippet.end + + def check(items): + a.eq(len(items), 1, 'maximum_per_channel(1) returns one message') + a.eq(items[0].message, last_message) + a.eq(int(items[0].timetoken), last_timetoken) + a.eq(items[0].meta, {'origin': 'docs'}) + a.eq(items[0].uuid, pubnub.uuid) + a.ok(hasattr(items[0], 'message_type'), 'include_message_type adds message_type') + a.eq(items[0].actions, {}, 'no actions yet') + + check(envelope.result.channels[channel]) + + # snippet.fetch_history_named + envelope = await pubnub.fetch_messages(channels=["my_channel"], count=1, include_message_actions=True, + include_meta=True, include_message_type=True, include_uuid=True).future() + + for message in envelope.result.channels["my_channel"]: + print(f"Message: {message.message}") + print(f"Meta: {message.meta}") + print(f"Timetoken: {message.timetoken}") + # snippet.end + check(envelope.result.channels[channel]) + a.eq(sorted(envelope.result.channels), [channel]) + + +@step('fetch_history_range', role='open') +async def test_fetch_history_range(aclients): + pubnub = aclients.asyncio('open', 'ranger') + channel = names.with_suffix('my_channel') + seeded = await seed(pubnub, channel, 5) + tokens = [tt for _, tt in seeded] + messages = [m for m, _ in seeded] + await wait_stored(pubnub, channel, tokens) + newer_timetoken, older_timetoken = tokens[3], tokens[1] + + # snippet.fetch_history_range + envelope = await pubnub.fetch_messages() \ + .channels(["my_channel"]) \ + .start(newer_timetoken) \ + .end(older_timetoken) \ + .future() + # snippet.end + + got = envelope.result.channels[channel] + a.eq([int(i.timetoken) for i in got], [tokens[1], tokens[2]], 'start excluded, end included, oldest first') + a.eq([i.message for i in got], messages[1:3]) + only_start = await pubnub.fetch_messages(channels=[channel], start=tokens[3], end=tokens[0]).future() + a.eq([int(i.timetoken) for i in only_start.result.channels[channel]], tokens[0:3], 'older than start') + only_end = (await pubnub.fetch_messages(channels=[channel], end=tokens[2]).future()).result.channels[channel] + a.eq([int(i.timetoken) for i in only_end][:3], tokens[2:5], 'end only: newer than or equal to end') + a.eq(int(only_end[-1].timetoken), tokens[4], 'end only: includes the newest message') + + +@step('fetch_history_multiple', role='open', timeout=300) +async def test_fetch_history_caps(aclients): + pubnub = aclients.asyncio('open', 'capper') + channel = names.with_suffix('my_channel') + first = names.with_suffix('ch1') + second = names.with_suffix('ch2') + single = await seed_parallel(pubnub, channel, 101) + on_first = await seed_parallel(pubnub, first, 26) + on_second = await seed_parallel(pubnub, second, 26) + + async def single_stored(): + got = (await pubnub.fetch_messages(channels=[channel], count=150).future()).result.channels[channel] + a.eq([int(i.timetoken) for i in got], single[1:], 'newest 100 of 101') + + await aeventually(single_stored, timeout=30, what='seeded messages stored') + + async def newest_stored(): + got = (await pubnub.fetch_messages(channels=[first, second], count=30).future()).result.channels + a.eq([int(i.timetoken) for i in got[first]], on_first[1:], 'newest 25 of ch1') + a.eq([int(i.timetoken) for i in got[second]], on_second[1:], 'newest 25 of ch2') + + await aeventually(newest_stored, timeout=30, what='seeded messages on both channels stored') + + # snippet.fetch_history_multiple + envelope = await pubnub.fetch_messages() \ + .channels(["ch1", "ch2"]) \ + .maximum_per_channel(30) \ + .future() + + for channel_name, items in envelope.result.channels.items(): + print(channel_name, len(items)) + # snippet.end + + for name in (first, second): + a.eq(len(envelope.result.channels[name]), 25, 'several channels: at most 25 per channel') + one = (await pubnub.fetch_messages(channels=[channel], count=150).future()).result.channels[channel] + a.eq(len(one), 100, 'one channel: at most 100 messages') + a.eq([int(i.timetoken) for i in one], single[1:], 'the newest 100, oldest first') + + +@step('message_counts_builder', 'message_counts_named', 'message_counts_multiple_builder', + 'message_counts_multiple_named', role='open') +async def test_message_counts(aclients): + pubnub = aclients.asyncio('open', 'counter') + channel = names.with_suffix('my_channel') + other = names.with_suffix('my_channel_2') + seeded = await seed(pubnub, channel, 4) + tokens = [tt for _, tt in seeded] + on_other = [tt for _, tt in await seed(pubnub, other, 2)] + await wait_stored(pubnub, channel, tokens) + await wait_stored(pubnub, other, on_other) + timetoken = tokens[0] + other_timetoken = on_other[0] + + def counted(envelope, expected): + a.eq(envelope.result.channels, expected) + + async def counted_eventually(build, expected): + async def check(): + counted(await build(), expected) + + await aeventually(check, timeout=30, what='message counts') + + # The service counts messages newer than the timetoken, the message at the timetoken itself is not counted. + await counted_eventually( + lambda: pubnub.message_counts(channels=channel, channels_timetoken=[timetoken - 1]).future(), {channel: 4}) + await counted_eventually( + lambda: pubnub.message_counts(channels=channel, channels_timetoken=[timetoken]).future(), {channel: 3}) + + # snippet.message_counts_builder + envelope = await pubnub.message_counts() \ + .channel("my_channel") \ + .channel_timetokens([timetoken]) \ + .future() + print(envelope.result.channels["my_channel"]) + # snippet.end + counted(envelope, {channel: 3}) + + # snippet.message_counts_named + envelope = await pubnub.message_counts(channels="my_channel", channels_timetoken=[tokens[2]]).future() + # snippet.end + counted(envelope, {channel: 1}) + await counted_eventually( + lambda: pubnub.message_counts(channels=channel, channels_timetoken=[tokens[3]]).future(), {channel: 0}) + + # snippet.message_counts_multiple_builder + envelope = await pubnub.message_counts() \ + .channel(["my_channel", "my_channel_2"]) \ + .channel_timetokens([timetoken, other_timetoken]) \ + .future() + print(envelope.result.channels) + # snippet.end + counted(envelope, {channel: 3, other: 1}) + + # snippet.message_counts_multiple_named + envelope = await pubnub.message_counts(channels=["my_channel", "my_channel_2"], + channels_timetoken=[tokens[1], other_timetoken]).future() + # snippet.end + counted(envelope, {channel: 2, other: 1}) + + +@step('delete_messages_builder', 'delete_messages_named', role='open') +async def test_delete_messages(aclients): + pubnub = aclients.asyncio('open', 'deleter') + channel = names.with_suffix('my_channel') + seeded = await seed(pubnub, channel, 4) + tokens = [tt for _, tt in seeded] + await wait_stored(pubnub, channel, tokens) + newer_timetoken, older_timetoken = tokens[2], tokens[1] + + # snippet.delete_messages_builder + envelope = await pubnub.delete_messages() \ + .channel("my_channel") \ + .start(newer_timetoken) \ + .end(older_timetoken) \ + .future() + # snippet.end + detail = getattr(envelope.status.error_data, 'information', None) + a.eq(envelope.status.is_error(), False, 'delete failed: %s' % detail) + + async def left(expected): + got = [int(i.timetoken) for i in await window(pubnub, channel, tokens[3], tokens[0])] + a.eq(got, expected) + + await aeventually(lambda: left([tokens[0], tokens[2], tokens[3]]), timeout=30, + what='only the message at end is deleted') + + # snippet.delete_messages_named + envelope = await pubnub.delete_messages(channel="my_channel", start=tokens[3], end=tokens[2]).future() + # snippet.end + detail = getattr(envelope.status.error_data, 'information', None) + a.eq(envelope.status.is_error(), False, 'delete failed: %s' % detail) + await aeventually(lambda: left([tokens[0], tokens[3]]), timeout=30, what='the second message is deleted') + + +@step('delete_message_single_builder', 'delete_message_single_named', role='open') +async def test_delete_single_message(aclients): + pubnub = aclients.asyncio('open', 'single_deleter') + channel = names.with_suffix('my_channel') + seeded = await seed(pubnub, channel, 3) + tokens = [tt for _, tt in seeded] + await wait_stored(pubnub, channel, tokens) + publish_timetoken = tokens[1] + + # snippet.delete_message_single_builder + envelope = await pubnub.delete_messages() \ + .channel("my_channel") \ + .start(publish_timetoken - 1) \ + .end(publish_timetoken) \ + .future() + # snippet.end + detail = getattr(envelope.status.error_data, 'information', None) + a.eq(envelope.status.is_error(), False, 'delete failed: %s' % detail) + + async def left(expected): + got = [int(i.timetoken) for i in await window(pubnub, channel, tokens[2], tokens[0])] + a.eq(got, expected) + + await aeventually(lambda: left([tokens[0], tokens[2]]), timeout=30, + what='exactly the published message is deleted') + + publish_timetoken = tokens[2] + # snippet.delete_message_single_named + envelope = await pubnub.delete_messages(channel="my_channel", start=publish_timetoken - 1, + end=publish_timetoken).future() + # snippet.end + detail = getattr(envelope.status.error_data, 'information', None) + a.eq(envelope.status.is_error(), False, 'delete failed: %s' % detail) + await aeventually(lambda: left([tokens[0]]), timeout=30, what='the last message is deleted') + + +@step('history_basic', 'history_reverse', 'history_start_reverse', 'history_end', 'history_include_timetoken', + role='open') +async def test_history(aclients): + pubnub = aclients.asyncio('open', 'historian') + channel = names.with_suffix('history_channel_2') + seeded = await seed(pubnub, channel, 5) + tokens = [tt for _, tt in seeded] + messages = [m for m, _ in seeded] + + async def stored(): + got = await pubnub.history().channel(channel).count(100).future() + a.eq(len(got.result.messages), 5) + + await aeventually(stored, timeout=30, what='seeded messages stored') + + # snippet.history_basic + envelope = await pubnub.history() \ + .channel("history_channel_2") \ + .count(100) \ + .future() + # snippet.end + a.eq([m.entry for m in envelope.result.messages], messages, 'oldest first, newest last') + a.eq(envelope.result.start_timetoken, tokens[0]) + a.eq(envelope.result.end_timetoken, tokens[4]) + + # snippet.history_reverse + envelope = await pubnub.history() \ + .channel("history_channel_2") \ + .count(3) \ + .reverse(True) \ + .future() + # snippet.end + a.eq([m.entry for m in envelope.result.messages], messages[0:3], 'the three oldest') + + # snippet.history_start_reverse + envelope = await pubnub.history() \ + .channel("history_channel_2") \ + .start(tokens[1]) \ + .reverse(True) \ + .future() + # snippet.end + a.eq([m.entry for m in envelope.result.messages], messages[2:], 'newer than start, start excluded') + + # snippet.history_end + envelope = await pubnub.history() \ + .channel("history_channel_2") \ + .count(100) \ + .end(tokens[2]) \ + .future() + # snippet.end + a.eq([m.entry for m in envelope.result.messages], messages[2:], 'end included, newest last') + # The docs sample adds start(-1) and reverse(True). The service refuses start(-1) and with reverse(True) the end + # timetoken becomes the newer boundary, so the result is the other half of the timeline. + refused = await pubnub.history().channel(channel).count(100).start(-1).end(tokens[2]).future() + a.eq((refused.status.is_error(), refused.status.status_code), (True, 400), 'start(-1) is refused with HTTP 400') + flipped = await pubnub.history().channel(channel).count(100).end(tokens[2]).reverse(True).future() + a.eq([m.entry for m in flipped.result.messages], messages[:3], 'reverse with end returns the older half') + + # snippet.history_include_timetoken + envelope = await pubnub.history() \ + .channel("history_channel_2") \ + .count(100) \ + .include_timetoken(True) \ + .future() + # snippet.end + a.eq([m.timetoken for m in envelope.result.messages], tokens) + a.eq([m.entry for m in envelope.result.messages], messages) + + +@step('history_paging', role='open', timeout=300) +async def test_history_paging(aclients): + pubnub = aclients.asyncio('open', 'pager') + channel = names.with_suffix('history_paging_channel_2') + tokens = await seed_parallel(pubnub, channel, 101) + + async def stored(): + got = await pubnub.history().channel(channel).count(100).future() + a.eq(len(got.result.messages), 100) + + await aeventually(stored, timeout=30, what='seeded messages stored') + start_timetoken = tokens[-1] + 1 + + # snippet.history_paging + async def get_all_messages(start_tt): + envelope = await pubnub.history() \ + .channel("history_paging_channel_2") \ + .count(100) \ + .start(start_tt) \ + .future() + msgs = envelope.result.messages + start = envelope.result.start_timetoken + end = envelope.result.end_timetoken + count = len(msgs) + + if count > 0: + print("%d" % count) + print("start %d" % start) + print("end %d" % end) + + if count == 100: + await get_all_messages(start) + # snippet.end + + output = io.StringIO() + with contextlib.redirect_stdout(output): + await get_all_messages(start_timetoken) + lines = output.getvalue().split('\n') + a.eq(lines[0:3], ['100', 'start %d' % tokens[1], 'end %d' % tokens[-1]], 'first page: the newest 100') + a.eq(lines[3:6], ['1', 'start %d' % tokens[0], 'end %d' % tokens[0]], 'second page: the one older message') diff --git a/docs-snippets/asyncio/misc.py b/docs-snippets/asyncio/misc.py new file mode 100644 index 00000000..d61d552b --- /dev/null +++ b/docs-snippets/asyncio/misc.py @@ -0,0 +1,53 @@ +"""Docs page: miscellaneous utility methods (asyncio). Source: docs/sdks/asyncio/api-reference/misc.md.""" +import time + +from harness import assertions as a +from harness import names, world +from harness.listeners import alisten +from harness.step import step +from harness.wait import aeventually + + +@step('time_get', role='open') +async def test_time(aclients): + pubnub = aclients.asyncio('open', 'timekeeper') + + # snippet.time_get + envelope = await pubnub.time().future() + print(f"Current time: {envelope.result}") + # snippet.end + a.timetoken(str(envelope.result)) + a.eq(int(envelope.result), envelope.result.value_as_int) + a.ok(abs(int(envelope.result) / 10000000 - time.time()) < 120, 'server time is within two minutes of this host') + a.ok(abs(pubnub.timestamp() - int(time.time())) <= 1, 'timestamp() is int(time.time())') + a.eq(envelope.result.date_time().year, time.gmtime().tm_year) + + +@step('get_subscribed_channels', 'get_subscribed_channel_groups', role='open') +async def test_get_subscribed(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + admin = aclients.asyncio('open', 'group_admin') + channel_a, channel_b = names.with_suffix('ch1'), names.with_suffix('ch2') + group = names.with_suffix('cg1') + world.record('channel_group', group) + try: + await admin.add_channel_to_channel_group().channels([channel_b]).channel_group(group).future() + await alisten(pubnub, channel_a, groups=[group]) + + # snippet.get_subscribed_channels + channels = pubnub.get_subscribed_channels() + # snippet.end + a.eq(channels, [channel_a]) + + # snippet.get_subscribed_channel_groups + channel_groups = pubnub.get_subscribed_channel_groups() + # snippet.end + a.eq(channel_groups, [group]) + pubnub.unsubscribe().channels(channel_a).channel_groups(group).execute() + + async def left(): + a.eq(pubnub.get_subscribed_channels(), [], 'channels after unsubscribe') + await aeventually(left) + a.eq(pubnub.get_subscribed_channel_groups(), [], 'groups after unsubscribe') + finally: + await admin.remove_channel_group().channel_group(group).future() diff --git a/docs-snippets/asyncio/mobile_push.py b/docs-snippets/asyncio/mobile_push.py new file mode 100644 index 00000000..7ac5c32b --- /dev/null +++ b/docs-snippets/asyncio/mobile_push.py @@ -0,0 +1,218 @@ +"""Docs page: mobile push (asyncio). Source: docs/sdks/asyncio/api-reference/mobile-push.md. + +Only the registration side is testable. A push is delivered by FCM or APNs to a real device, so delivery is not tested. +The device tokens are made up and unique per run, and every gateway is cleaned in a finally block. +""" +import secrets + +from pubnub.enums import PNPushEnvironment, PNPushType + +from harness import assertions as a +from harness import names +from harness.step import step +from harness.wait import aeventually + +REGISTERED_NAMES = ['deviceId'] +TOPIC = 'myapptopic' +APNS2_DEVICE = secrets.token_hex(32) +CHANNELS = ['ch1', 'ch2', 'ch3'] + + +def suffixed(channels): + return sorted(names.with_suffix(c) for c in channels) + + +def fcm_device(): + return names.with_suffix('deviceId') + + +async def registered(pubnub, push_type): + """Channels the service holds for the device on one gateway.""" + if push_type == PNPushType.APNS2: + envelope = await pubnub.list_push_channels().push_type(push_type).device_id(APNS2_DEVICE) \ + .topic(TOPIC).environment(PNPushEnvironment.DEVELOPMENT).future() + else: + envelope = await pubnub.list_push_channels().push_type(push_type).device_id(fcm_device()).future() + return sorted(envelope.result.channels) + + +async def seed(pubnub, channels): + channels = suffixed(channels) + await pubnub.add_channels_to_push().push_type(PNPushType.FCM).channels(channels).device_id(fcm_device()).future() + await pubnub.add_channels_to_push().push_type(PNPushType.APNS2).channels(channels).device_id(APNS2_DEVICE) \ + .topic(TOPIC).environment(PNPushEnvironment.DEVELOPMENT).future() + + +async def clear(pubnub): + """Remove the test tokens on every gateway and environment so that nothing outlives the run.""" + for push_type in (PNPushType.FCM, PNPushType.GCM): + try: + await pubnub.remove_device_from_push().push_type(push_type).device_id(fcm_device()).future() + except Exception as e: # cleanup must never hide the real failure + print('cleanup: could not clear a gateway (' + type(e).__name__ + ')') + for push_type in (PNPushType.APNS, PNPushType.APNS2): + for environment in (PNPushEnvironment.DEVELOPMENT, PNPushEnvironment.PRODUCTION): + try: + await pubnub.remove_device_from_push().push_type(push_type).device_id(APNS2_DEVICE) \ + .topic(TOPIC).environment(environment).future() + except Exception as e: + print('cleanup: could not clear a gateway (' + type(e).__name__ + ')') + + +async def expect(pubnub, fcm, apns2, what): + async def fcm_state(): + a.eq(await registered(pubnub, PNPushType.FCM), fcm, 'fcm ' + what) + + async def apns2_state(): + a.eq(await registered(pubnub, PNPushType.APNS2), apns2, 'apns2 ' + what) + await aeventually(fcm_state) + await aeventually(apns2_state) + + +@step('add_channels_to_push_builder', 'add_channels_to_push_kwargs', role='open') +async def test_add_channels_to_push(aclients): + pubnub = aclients.asyncio('open', 'push_registrar') + try: + a.ok(len(APNS2_DEVICE) == 64 and set(APNS2_DEVICE) <= set('0123456789abcdef'), 'apns2 token is 64 hex chars') + + # snippet.add_channels_to_push_builder + # for FCM + envelope = await pubnub.add_channels_to_push() \ + .push_type(PNPushType.FCM) \ + .channels(["ch1", "ch2", "ch3"]) \ + .device_id("deviceId") \ + .future() + + # for APNS2 + envelope = await pubnub.add_channels_to_push() \ + .push_type(PNPushType.APNS2) \ + .channels(["ch1", "ch2", "ch3"]) \ + .device_id(APNS2_DEVICE) \ + .topic("myapptopic") \ + .environment(PNPushEnvironment.DEVELOPMENT) \ + .future() + + if envelope.status.is_error(): + print("Error adding device to channels for push notifications") + else: + print("Device added to channels successfully") + # snippet.end + a.ok(not envelope.status.is_error(), 'status is not an error') + await expect(pubnub, suffixed(CHANNELS), suffixed(CHANNELS), 'after add (builder)') + await clear(pubnub) + await expect(pubnub, [], [], 'after clearing') + + # snippet.add_channels_to_push_kwargs + kwargs = { + "push_type": PNPushType.FCM, + "channels": ["ch1", "ch2", "ch3"], + "device_id": "deviceId", + } + envelope = await pubnub.add_channels_to_push(**kwargs).future() + + kwargs = { + "push_type": PNPushType.APNS2, + "channels": ["ch1", "ch2", "ch3"], + "device_id": APNS2_DEVICE, + "topic": "myapptopic", + "environment": PNPushEnvironment.DEVELOPMENT, + } + envelope = await pubnub.add_channels_to_push(**kwargs).future() + # snippet.end + a.ok(not envelope.status.is_error(), 'status is not an error') + await expect(pubnub, suffixed(CHANNELS), suffixed(CHANNELS), 'after add (keyword arguments)') + finally: + await clear(pubnub) + + +@step('list_push_channels_builder', role='open') +async def test_list_push_channels(aclients): + pubnub = aclients.asyncio('open', 'push_lister') + try: + await seed(pubnub, CHANNELS) + await expect(pubnub, suffixed(CHANNELS), suffixed(CHANNELS), 'after seeding') + + # snippet.list_push_channels_builder + # for FCM + envelope = await pubnub.list_push_channels() \ + .push_type(PNPushType.FCM) \ + .device_id("deviceId") \ + .future() + fcm_channels = envelope.result.channels + + # for APNS2 + envelope = await pubnub.list_push_channels() \ + .push_type(PNPushType.APNS2) \ + .device_id(APNS2_DEVICE) \ + .topic("myapptopic") \ + .environment(PNPushEnvironment.DEVELOPMENT) \ + .future() + apns2_channels = envelope.result.channels + # snippet.end + a.eq(sorted(fcm_channels), suffixed(CHANNELS), 'fcm channels listed') + a.eq(sorted(apns2_channels), suffixed(CHANNELS), 'apns2 channels listed') + a.ok(not envelope.status.is_error(), 'status is not an error') + other = await pubnub.list_push_channels().push_type(PNPushType.FCM) \ + .device_id(names.with_suffix('unknown')).future() + a.eq(other.result.channels, [], 'unknown device') + finally: + await clear(pubnub) + + +@step('remove_channels_from_push_builder', role='open') +async def test_remove_channels_from_push(aclients): + pubnub = aclients.asyncio('open', 'push_remover') + keep = 'keep_me' + try: + await seed(pubnub, CHANNELS + [keep]) + await expect(pubnub, suffixed(CHANNELS + [keep]), suffixed(CHANNELS + [keep]), 'after seeding') + + # snippet.remove_channels_from_push_builder + # for FCM + envelope = await pubnub.remove_channels_from_push() \ + .push_type(PNPushType.FCM) \ + .channels(["ch1", "ch2", "ch3"]) \ + .device_id("deviceId") \ + .future() + + # for APNS2 + envelope = await pubnub.remove_channels_from_push() \ + .push_type(PNPushType.APNS2) \ + .channels(["ch1", "ch2", "ch3"]) \ + .device_id(APNS2_DEVICE) \ + .topic("myapptopic") \ + .environment(PNPushEnvironment.DEVELOPMENT) \ + .future() + # snippet.end + a.ok(not envelope.status.is_error(), 'status is not an error') + await expect(pubnub, suffixed([keep]), suffixed([keep]), 'after remove, only the other channel is left') + finally: + await clear(pubnub) + + +@step('remove_device_from_push_builder', role='open') +async def test_remove_device_from_push(aclients): + pubnub = aclients.asyncio('open', 'push_device_remover') + try: + await seed(pubnub, CHANNELS) + await expect(pubnub, suffixed(CHANNELS), suffixed(CHANNELS), 'after seeding') + + # snippet.remove_device_from_push_builder + # for FCM + envelope = await pubnub.remove_device_from_push() \ + .push_type(PNPushType.FCM) \ + .device_id("deviceId") \ + .future() + + # for APNS2 + envelope = await pubnub.remove_device_from_push() \ + .push_type(PNPushType.APNS2) \ + .device_id(APNS2_DEVICE) \ + .topic("myapptopic") \ + .environment(PNPushEnvironment.DEVELOPMENT) \ + .future() + # snippet.end + a.ok(not envelope.status.is_error(), 'status is not an error') + await expect(pubnub, [], [], 'after remove all') + finally: + await clear(pubnub) diff --git a/docs-snippets/asyncio/presence.py b/docs-snippets/asyncio/presence.py new file mode 100644 index 00000000..0c1d6aac --- /dev/null +++ b/docs-snippets/asyncio/presence.py @@ -0,0 +1,252 @@ +"""Docs page: presence (asyncio). Same steps as sync/presence.py with await. + +Presence needs channel tracking on the keyset. Joins of a second client are asserted, never the client's own join, +and here_now is polled with aeventually because the service reports it with a delay and caches it for 3 seconds. +""" +from harness import assertions as a +from harness import names +from harness import world +from harness.listeners import alisten +from harness.step import step +from harness.wait import aeventually + +REGISTERED_NAMES = ['cg2', 'cg3', 'gr1', 'gr2', 'gr3', 'such_uuid', 'some-other-uuid'] + + +class Printed: + """Everything the docs code printed so far.""" + + def __init__(self, capsys): + self._capsys = capsys + self._parts = [] + + def text(self): + self._parts.append(self._capsys.readouterr().out) + return ''.join(self._parts) + + +def occupants_of(here_now, channel): + """uuids of one channel in a PNHereNowResult, sorted. The result holds a list of PNHereNowChannelData.""" + found = [c for c in here_now.result.channels if c.channel_name == channel] + return sorted(o.uuid for o in found[0].occupants) if found and found[0].occupants is not None else None + + +def occupancy_of(here_now, channel): + found = [c for c in here_now.result.channels if c.channel_name == channel] + return found[0].occupancy if found else None + + +@step('here_now_builder', role='open') +async def test_here_now(aclients, capsys): + pubnub = aclients.asyncio('open', 'reader') + first, second = names.with_suffix('my_channel'), names.with_suffix('my_channel_2') + printed = Printed(capsys) + one = aclients.asyncio('open', 'user_one') + two = aclients.asyncio('open', 'user_two') + await alisten(one, first, second) + await alisten(two, first) + expected = {first: sorted([one.uuid, two.uuid]), second: [one.uuid]} + + async def builder(): + # snippet.here_now_builder + envelope = await pubnub.here_now() \ + .channels(["my_channel", "my_channel_2"]) \ + .include_uuids(True) \ + .future() + + if envelope.status.is_error(): + print("Error retrieving here_now data") + return + + for channel_data in envelope.result.channels: + print("---") + print(f"Channel: {channel_data.channel_name}") + print(f"Occupancy: {channel_data.occupancy}") + for occupant in channel_data.occupants: + print(f"UUID: {occupant.uuid}, State: {occupant.state}") + # snippet.end + return envelope + + async def check(): + result = await builder() + a.ok(result is not None, 'here_now returned an error') + a.eq({c: occupants_of(result, c) for c in expected}, expected, 'occupants per channel') + a.eq((result.result.total_channels, result.result.total_occupancy), (2, 3)) + await aeventually(check, what='here_now lists both clients') + text = printed.text() + a.contains(text, f'Channel: {first}\nOccupancy: 2\n') + a.contains(text, f'UUID: {one.uuid}, State: None') + a.contains(text, f'Channel: {second}\nOccupancy: 1\n') + + +@step('here_now_state', role='open') +async def test_here_now_state(aclients): + pubnub = aclients.asyncio('open', 'reader') + name = names.with_suffix('my_channel') + one = aclients.asyncio('open', 'user_one') + await alisten(one, name) + await one.set_state().channels(name).state({'abcd': {'age': 15}}).future() + + async def read(): + # snippet.here_now_state + envelope = await pubnub.here_now() \ + .channels("my_channel") \ + .include_uuids(True) \ + .include_state(True) \ + .future() + # snippet.end + return envelope + + async def check(): + envelope = await read() + found = [(o.uuid, o.state) for c in envelope.result.channels for o in c.occupants] + a.eq(found, [(one.uuid, {'abcd': {'age': 15}})], 'occupants with state') + a.eq((envelope.result.total_channels, envelope.result.total_occupancy), (1, 1)) + await aeventually(check, what='here_now returns the state') + + +@step('here_now_occupancy', role='open') +async def test_here_now_occupancy(aclients): + pubnub = aclients.asyncio('open', 'reader') + name = names.with_suffix('my_channel') + for who in ('user_one', 'user_two', 'user_three'): + await alisten(aclients.asyncio('open', who), name) + + async def read(): + # snippet.here_now_occupancy + envelope = await pubnub.here_now() \ + .channels("my_channel") \ + .include_uuids(False) \ + .include_state(False) \ + .future() + # snippet.end + return envelope + + async def check(): + envelope = await read() + a.eq([(c.channel_name, c.occupancy, c.occupants) for c in envelope.result.channels], [(name, 3, [])]) + a.eq((envelope.result.total_channels, envelope.result.total_occupancy), (1, 3)) + await aeventually(check, what='here_now counts three occupants') + + +@step('here_now_groups', role='open') +async def test_here_now_groups(aclients): + pubnub = aclients.asyncio('open', 'reader') + groups = [names.with_suffix(n) for n in ('cg1', 'cg2', 'cg3')] + members = [names.with_suffix(f'grouped_{i}') for i in (1, 2, 3)] + for group, member in zip(groups, members): + await pubnub.add_channel_to_channel_group().channels([member]).channel_group(group).future() + world.record('channel_group', group) + one = aclients.asyncio('open', 'user_one') + await alisten(one, *members) + + async def read(): + # snippet.here_now_groups + envelope = await pubnub.here_now() \ + .channel_groups(['cg1', 'cg2', 'cg3']) \ + .include_uuids(True) \ + .include_state(True) \ + .future() + # snippet.end + return envelope + + async def check(): + envelope = await read() + a.eq(sorted((c.channel_name, c.occupancy, [o.uuid for o in c.occupants]) for c in envelope.result.channels), + sorted((m, 1, [one.uuid]) for m in members), 'every channel of the groups with its occupant') + a.eq((envelope.result.total_channels, envelope.result.total_occupancy), (3, 3)) + await aeventually(check, what='here_now by channel group') + + +@step('where_now_self', 'where_now_other', role='open') +async def test_where_now(aclients): + pubnub = aclients.asyncio('open', 'reader') + first, second = names.with_suffix('my_channel'), names.with_suffix('my_channel_2') + await alisten(pubnub, first, second) + other = aclients.asyncio('open', 'some-other-uuid') + await alisten(other, first) + + async def own(): + # snippet.where_now_self + envelope = await pubnub.where_now() \ + .future() + # snippet.end + return envelope + + async def elsewhere(): + # snippet.where_now_other + envelope = await pubnub.where_now() \ + .uuid('some-other-uuid') \ + .future() + # snippet.end + return envelope + + async def check_own(): + a.eq(sorted((await own()).result.channels), sorted([first, second]), 'own channels') + + async def check_other(): + a.eq((await elsewhere()).result.channels, [first], 'channels of the other user ID') + await aeventually(check_own) + await aeventually(check_other) + + +@step('set_state_builder', 'get_state_builder', role='open') +async def test_state(aclients): + pubnub = aclients.asyncio('open', 'such_uuid') + reader = aclients.asyncio('open', 'reader') + channels = [names.with_suffix(n) for n in ('ch1', 'ch2', 'ch3')] + await alisten(pubnub, *channels) + + # snippet.set_state_builder + my_state = { + 'age': 20 + } + envelope = await pubnub.set_state() \ + .channels(['ch1', 'ch2', 'ch3']) \ + .state(my_state) \ + .future() + # snippet.end + a.eq(envelope.result.state, {'age': 20}, 'state returned by set_state') + + # snippet.get_state_builder + envelope = await pubnub.get_state() \ + .channels(['ch1', 'ch2', 'ch3']) \ + .uuid('such_uuid') \ + .future() + # snippet.end + a.eq(envelope.result.channels, {c: {'age': 20} for c in channels}, 'state per channel') + fetched = await reader.get_state().channels(channels[0]).uuid(pubnub.uuid).future() + a.eq(fetched.result.channels, {channels[0]: {'age': 20}}, 'another client reads the state') + + +@step('set_state_groups_builder', role='open') +async def test_state_for_groups(aclients): + pubnub = aclients.asyncio('open', 'such_uuid') + groups = [names.with_suffix(n) for n in ('gr1', 'gr2', 'gr3')] + members = [names.with_suffix(f'grouped_{i}') for i in (1, 2, 3)] + own = names.with_suffix('ch1') + for group, member in zip(groups, members): + await pubnub.add_channel_to_channel_group().channels([member]).channel_group(group).future() + world.record('channel_group', group) + await alisten(pubnub, own, *members, groups=groups) + # The SDK refuses a set_state() that names channel groups and no channel (PubNubException "State setter for + # channel groups is not supported yet" in sync, an error status in asyncio), so the sample below adds a + # channel next to the groups. + refusal = await pubnub.set_state().channel_groups(groups).state({'age': 20}).future() + a.ok(refusal.status.is_error(), 'set_state with only channel groups is refused') + + # snippet.set_state_groups_builder + my_state = { + 'age': 20 + } + envelope = await pubnub.set_state() \ + .channels(['ch1']) \ + .channel_groups(['gr1', 'gr2', 'gr3']) \ + .state(my_state) \ + .future() + # snippet.end + a.eq(envelope.result.state, {'age': 20}, 'state returned by set_state for groups') + own_state = await pubnub.get_state().channels(own).future() + a.eq(own_state.result.channels, {own: {'age': 20}}) + group_state = await pubnub.get_state().channel_groups(groups).future() + a.eq(group_state.result.channels, {m: {'age': 20} for m in members}, 'state read back through the channel groups') diff --git a/docs-snippets/asyncio/publish.py b/docs-snippets/asyncio/publish.py new file mode 100644 index 00000000..dd8168d7 --- /dev/null +++ b/docs-snippets/asyncio/publish.py @@ -0,0 +1,176 @@ +"""Docs page: publish (asyncio). Same steps as sync/publish.py with await.""" +from harness import assertions as a +from harness import names +from harness import world +from harness.listeners import alisten +from harness.step import step +from harness.wait import aeventually + + +@step('publish_builder', 'publish_named', role='open') +async def test_publish_message(aclients): + pubnub = aclients.asyncio('open', 'publisher') + channel = names.with_suffix('my_channel') + seen = await alisten(aclients.asyncio('open', 'subscriber'), channel) + + # snippet.publish_builder + envelope = await pubnub.publish() \ + .channel("my_channel") \ + .message({ + 'name': 'Alex', + 'online': True + }) \ + .custom_message_type("text-message") \ + .future() + # snippet.end + first_timetoken = envelope.result.timetoken + + # snippet.publish_named + envelope = await pubnub.publish( + channel="my_channel", + message={ + 'name': 'Sam', + 'online': False + }, + custom_message_type="text-message", + meta={'origin': 'docs'} + ).future() + # snippet.end + second_timetoken = envelope.result.timetoken + + a.timetoken(first_timetoken) + a.timetoken(second_timetoken) + await aeventually(lambda: a.eq(len(seen.messages), 2, 'messages received')) + by_token = {m.timetoken: m for m in seen.messages} + a.eq(by_token[first_timetoken].message, {'name': 'Alex', 'online': True}) + a.eq(by_token[first_timetoken].publisher, pubnub.uuid) + a.eq(by_token[first_timetoken].custom_message_type, 'text-message') + a.eq(by_token[second_timetoken].user_metadata, {'origin': 'docs'}) + + +@step('signal_message', role='open') +async def test_signal(aclients): + pubnub = aclients.asyncio('open', 'signaler') + channel = names.with_suffix('my_channel') + seen = await alisten(aclients.asyncio('open', 'subscriber'), channel) + + # snippet.signal_message + envelope = await pubnub.signal() \ + .channel("my_channel") \ + .message("typing_on") \ + .future() + # snippet.end + + a.timetoken(envelope.result.timetoken) + await aeventually(lambda: a.eq([s.message for s in seen.signals], ['typing_on'], 'signals received')) + a.eq(seen.messages, [], 'a signal is not a message') + + +@step('fire_message', role='open') +async def test_fire(aclients): + pubnub = aclients.asyncio('open', 'firer') + channel = names.with_suffix('my_channel') + seen = await alisten(aclients.asyncio('open', 'subscriber'), channel) + + # snippet.fire_message + envelope = await pubnub.fire() \ + .channel("my_channel") \ + .message({'command': 'refresh'}) \ + .future() + # snippet.end + + a.timetoken(envelope.result.timetoken) + # A fired message is not replicated to subscribers. A normal publish after it proves the subscriber was listening. + await pubnub.publish().channel(channel).message('after-fire').future() + await aeventually(lambda: a.eq([m.message for m in seen.messages], ['after-fire'], 'only the later one arrived')) + + +async def history_of(pubnub, channel): + fetched = await pubnub.fetch_messages(channels=[channel]).future() + return [item.message for item in fetched.result.channels.get(channel, [])] + + +async def stored_has(pubnub, channel, wanted): + a.contains(await history_of(pubnub, channel), wanted, 'message is stored in history') + + +@step('publish_metadata', role='open') +async def test_publish_metadata(aclients): + pubnub = aclients.asyncio('open', 'publisher') + channel = names.with_suffix('my_channel') + seen = await alisten(aclients.asyncio('open', 'subscriber'), channel) + + # snippet.publish_metadata + envelope = await pubnub.publish()\ + .channel("my_channel")\ + .message(["hello", "there"])\ + .meta({'name': 'Alex'})\ + .future() + # handle publish result, status always present, result if successful + # envelope.status.is_error() to see if error happened + # snippet.end + + a.ok(not envelope.status.is_error(), 'status is not an error') + a.timetoken(envelope.result.timetoken) + await aeventually(lambda: a.eq(len(seen.messages), 1, 'messages received')) + a.eq(seen.messages[0].message, ['hello', 'there']) + a.eq(seen.messages[0].user_metadata, {'name': 'Alex'}) + a.eq(seen.messages[0].timetoken, envelope.result.timetoken) + + +@step('publish_store_post', role='open') +async def test_publish_store_and_post(aclients): + pubnub = aclients.asyncio('open', 'publisher') + channel = names.with_suffix('my_channel') + seen = await alisten(aclients.asyncio('open', 'subscriber'), channel) + + # snippet.publish_store_post + stored = await pubnub.publish() \ + .channel("my_channel") \ + .message({'kept': True}) \ + .should_store(True) \ + .use_post(True) \ + .future() + + not_stored = await pubnub.publish() \ + .channel("my_channel") \ + .message({'kept': False}) \ + .should_store(False) \ + .use_post(True) \ + .future() + # snippet.end + + a.timetoken(stored.result.timetoken) + a.timetoken(not_stored.result.timetoken) + await aeventually(lambda: a.eq(len(seen.messages), 2, 'both messages are delivered live')) + a.eq(sorted(m.message['kept'] for m in seen.messages), [False, True], 'POST bodies arrive as JSON objects') + + await aeventually(lambda: stored_has(pubnub, channel, {'kept': True})) + a.not_contains(await history_of(pubnub, channel), {'kept': False}, 'the should_store(False) message is not stored') + world.record('history', channel) + refused = await pubnub.publish().channel(channel).message('x' * 33000).use_post(True).future() + a.ok(refused.status.is_error(), 'POST publish of a message over 32 KiB is refused') + a.eq(refused.status.status_code, 413, 'POST publish of a message over 32 KiB') + small = await pubnub.publish().channel(channel).message('x' * 30000).use_post(True).future() + a.timetoken(small.result.timetoken) + + +@step('fire_post', role='open') +async def test_fire_post(aclients): + pubnub = aclients.asyncio('open', 'firer') + channel = names.with_suffix('my_channel') + seen = await alisten(aclients.asyncio('open', 'subscriber'), channel) + + # snippet.fire_post + envelope = await pubnub.fire().channel('my_channel').message('hello there').use_post(True).future() + print('fire timetoken: %d' % envelope.result.timetoken) + # snippet.end + + a.timetoken(envelope.result.timetoken) + await pubnub.publish().channel(channel).message('after-fire-post').future() + await aeventually(lambda: a.contains([m.message for m in seen.messages], 'after-fire-post', 'later publish')) + a.not_contains([m.message for m in seen.messages], 'hello there', 'fired messages are not delivered') + + await aeventually(lambda: stored_has(pubnub, channel, 'after-fire-post')) + a.not_contains(await history_of(pubnub, channel), 'hello there', 'fired messages are not stored in history') + world.record('history', channel) diff --git a/docs-snippets/asyncio/subscribe.py b/docs-snippets/asyncio/subscribe.py new file mode 100644 index 00000000..e2abf795 --- /dev/null +++ b/docs-snippets/asyncio/subscribe.py @@ -0,0 +1,1039 @@ +"""Docs page: subscribe (asyncio). Subscribe, Entities, Event listeners, Unsubscribe and the old-style methods.""" +from pubnub.enums import PNStatusCategory +from pubnub.models.server.subscribe import SubscribeMessage + +from harness import assertions as a +from harness import names +from harness import world +from harness.listeners import Collector, alisten +from harness.step import compile_only, step +from harness.wait import aeventually +from pubnub.models.consumer.message_actions import PNMessageAction + +REGISTERED_NAMES = ['group-test', 'channelGroupName', 'channelMetadata', 'user_metadata', 'cg_1', + 'awesome_channel_group', 'my_channel1', 'my_channel2', 'my_group1', 'my_group2'] + + +def attach(client): + """Add a collector to a client without subscribing. The docs code does the subscribing.""" + collector = Collector() + client.subscribed_by_suite = True + client.add_listener(collector) + return collector + + +async def connected(collector): + await aeventually(lambda: a.ok(PNStatusCategory.PNConnectedCategory in collector.categories, 'not connected yet'), + what='PNConnectedCategory') + + +def texts(collector): + return [m.message for m in collector.messages] + + +@step('subscribe_single', role='open') +async def test_subscribe_single(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + name = names.with_suffix('my_channel') + seen = attach(pubnub) + + # snippet.subscribe_single + # single channel subscription + channel = pubnub.channel("my_channel") + t1_subscription = channel.subscription() + t1_subscription.subscribe() + # snippet.end + + a.eq(channel.name, name) + await connected(seen) + await publisher.publish().channel(name).message('hello single').future() + await aeventually(lambda: a.eq(texts(seen), ['hello single'], 'messages received')) + a.eq(seen.messages[0].channel, name) + + +@step('subscribe_multiple', role='open') +async def test_subscribe_multiple(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + name = names.with_suffix('my_channel') + group_name = names.with_suffix('group-test') + grouped = names.with_suffix('grouped_channel') + await publisher.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).future() + world.record('channel_group', group_name) + seen = attach(pubnub) + envelopes = [] + original_from_json = SubscribeMessage.from_json.__func__ + + def counting_from_json(cls, json_input): + if json_input.get('d') == 'second': + envelopes.append(json_input['c']) + return original_from_json(cls, json_input) + + SubscribeMessage.from_json = classmethod(counting_from_json) + try: + # snippet.subscribe_multiple + channel = "my_channel" + + # multiple channel subscription + channel_1 = pubnub.channel(channel).subscription() + channel_2 = pubnub.channel(f'{channel}.2').subscription(with_presence=True) + channel_x = pubnub.channel(f'{channel}.*').subscription(with_presence=True) + + group = pubnub.channel_group("group-test").subscription() + + subscription_set = pubnub.subscription_set([channel_1, channel_2, channel_x, group]) + + subscription_set.subscribe() + # snippet.end + + await connected(seen) + await publisher.publish().channel(name).message('plain').future() + await publisher.publish().channel(f'{name}.2').message('second').future() + await publisher.publish().channel(f'{name}.other').message('wildcard').future() + await publisher.publish().channel(grouped).message('group').future() + # The message on 'my_channel.2' matches two subscriptions: the channel itself and the wildcard + # 'my_channel.*'. The service sends one envelope per matching subscription, and for the same request the + # asyncio client got one or two of them, run to run (sync got two in every run). The SDK delivers every + # envelope it receives (pubnub/event_engine/effects.py:142 passes the whole 'm' list on, pubnub/workers.py:215 + # announces each one). The proof: copies at the listener equal envelopes received, and never more than + # the number of matching subscriptions. + matching = {f'{name}.2', f'{name}.*'} + await aeventually(lambda: a.eq(sorted(set(texts(seen))), ['group', 'plain', 'second', 'wildcard'], + 'distinct messages')) + await aeventually(lambda: a.eq({text: texts(seen).count(text) for text in set(texts(seen)) + if text != 'second'}, {'plain': 1, 'wildcard': 1, 'group': 1}, + 'messages with one matching subscription arrive once')) + await aeventually(lambda: a.eq(texts(seen).count('second'), envelopes.count(f'{name}.2'), + 'one delivery per envelope the SDK received')) + a.ok(1 <= envelopes.count(f'{name}.2') <= len(matching), 'at most one envelope per matching subscription') + second = [m for m in seen.messages if m.message == 'second'] + a.eq({m.timetoken for m in second}, {second[0].timetoken}, 'the copies are the same published message') + by_text = {m.message: m for m in seen.messages} + a.eq(by_text['plain'].channel, name) + a.eq(by_text['wildcard'].channel, f'{name}.other') + a.eq(by_text['wildcard'].subscription, f'{name}.*', 'wildcard match is reported as subscription') + a.eq(by_text['group'].channel, grouped) + a.eq(by_text['group'].subscription, group_name, 'channel group match is reported as subscription') + + other = aclients.asyncio('open', 'second_user') + await alisten(other, f'{name}.2') + await aeventually(lambda: a.contains([(p.event, p.uuid) for p in seen.presence_events + if p.channel == f'{name}.2'], + ('join', other.uuid), 'join of the second client')) + finally: + SubscribeMessage.from_json = classmethod(original_from_json) + + +class AppContextCollector(Collector): + """Collector that also keeps App Context events.""" + + def __init__(self): + super().__init__() + self.channel_events = [] + self.uuid_events = [] + + def channel(self, pubnub, channel): + with self._lock: + self.channel_events.append(channel) + + def uuid(self, pubnub, uuid): + with self._lock: + self.uuid_events.append(uuid) + + +@step('entity_channel', 'entity_channel_group', 'entity_channel_metadata', 'entity_user_metadata', role='open') +async def test_entities(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + other = aclients.asyncio('open', 'publisher') + name = names.with_suffix('my_channel') + group_name = names.with_suffix('channelGroupName') + meta_name = names.with_suffix('channelMetadata') + user_name = names.with_suffix('user_metadata') + grouped = names.with_suffix('grouped_channel') + seen = AppContextCollector() + pubnub.subscribed_by_suite = True + pubnub.add_listener(seen) + + # snippet.entity_channel + channel = pubnub.channel("my_channel") + # snippet.end + + # snippet.entity_channel_group + channel_group = pubnub.channel_group("channelGroupName") + # snippet.end + + # snippet.entity_channel_metadata + channel_metadata = pubnub.channel_metadata("channelMetadata") + # snippet.end + + # snippet.entity_user_metadata + user_metadata = pubnub.user_metadata("user_metadata") + # snippet.end + + a.eq((type(channel).__name__, channel.name), ('PubNubChannel', name)) + a.eq((type(channel_group).__name__, channel_group.name), ('PubNubChannelGroup', group_name)) + a.eq((type(channel_metadata).__name__, channel_metadata.name), ('PubNubChannelMetadata', meta_name)) + a.eq((type(user_metadata).__name__, user_metadata.name), ('PubNubUserMetadata', user_name)) + + await other.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).future() + world.record('channel_group', group_name) + # SDK bug: PubNubUserMetadata sets _types instead of _type, so its subscription would register the user ID as a + # channel group. Delivery for user metadata is therefore not exercised here, see the report. + pubnub.subscription_set([channel.subscription(), channel_group.subscription(), + channel_metadata.subscription()]).subscribe() + await connected(seen) + try: + await other.publish().channel(name).message('on the channel').future() + await other.publish().channel(grouped).message('through the group').future() + await other.set_channel_metadata().channel(meta_name).set_name('Entity channel').future() + await aeventually( + lambda: a.eq(sorted(texts(seen)), ['on the channel', 'through the group'], 'messages received')) + await aeventually( + lambda: a.eq([e.data['id'] for e in seen.channel_events], [meta_name], 'channel metadata event')) + finally: + await other.remove_channel_metadata().channel(meta_name).future() + + +class Printed: + """Everything the docs listeners printed so far. Callbacks run on SDK threads, capsys sees their output.""" + + def __init__(self, capsys): + self._capsys = capsys + self._parts = [] + + def text(self): + self._parts.append(self._capsys.readouterr().out) + return ''.join(self._parts) + + +@step('listeners_event_specific', role='open') +async def test_listeners_event_specific(aclients, capsys): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + name = names.with_suffix('my_channel') + seen = attach(pubnub) + printed = Printed(capsys) + + # snippet.listeners_event_specific + subscription = pubnub.channel("my_channel").subscription() + + # using closure for reusable listener + def on_message(listener): + def message_callback(message): + print(f"\033[94mMessage received on: {listener}: \n{message.message}\033[0m\n") + return message_callback + + # without closure + def on_message_action(message_action): + print(f"\033[5mMessageAction received: \n{message_action.value}\033[0m\n") + + def on_signal(listener): + def signal_callback(signal): + print(f"\033[0;36mSignal received on: {listener}: \n{signal.publisher} says: \t{signal.message}\033[0m") + return signal_callback + + subscription.on_message = on_message('message_listener') + subscription.on_message_action = on_message_action + subscription.on_signal = on_signal('signal_listener') + + subscription.subscribe() + # snippet.end + + await connected(seen) + envelope = await publisher.publish().channel(name).message('listener message').future() + await publisher.signal().channel(name).message('listener signal').future() + action = PNMessageAction().create(type='reaction', value='smile', message_timetoken=envelope.result.timetoken) + await publisher.add_message_action().channel(name).message_action(action).future() + await aeventually( + lambda: a.contains(printed.text(), 'Message received on: message_listener: \n', 'message printed')) + await aeventually(lambda: a.contains(printed.text(), 'listener message', 'message payload printed')) + await aeventually(lambda: a.contains(printed.text(), f'{publisher.uuid} says: \tlistener signal', 'signal printed')) + await aeventually(lambda: a.contains(printed.text(), 'MessageAction received: \nsmile', 'message action printed')) + + +@step('listeners_generic', role='open') +async def test_listeners_generic(aclients, capsys): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + name = names.with_suffix('my_channel') + seen = attach(pubnub) + printed = Printed(capsys) + + # snippet.listeners_generic + # snippet.hide + from pubnub.callbacks import SubscribeCallback + # snippet.show + + # add generic listener + class PrintListener(SubscribeCallback): + def status(self, _, status): + print(f'\033[92mPrintListener.status:\n{status.category.name}\033[0m') + + def message(self, _, message): + print(f'\033[94mPrintListener.message:\n{message.message}\033[0m') + + def presence(self, _, presence): + print(f'PrintListener.presence:\n{presence.uuid} {presence.event}s ' + f'{presence.subscription or presence.channel}\033[0m') + + def signal(self, _, signal): + print(f'PrintListener.signal:\n{signal.message} from {signal.publisher}\033[0m') + + def channel(self, _, channel): + print(f'\033[0;37mChannel Meta:\n{channel.__dict__}\033[0m') + + def uuid(self, _, uuid): + print(f'User Meta:\n{uuid.__dict__}\033[0m') + + def membership(self, _, membership): + print(f'Membership:\n{membership.__dict__}\033[0m') + + def message_action(self, _, message_action): + print(f'PrintListener.message_action {message_action}\033[0m') + + def file(self, _, file_message): + print(f' {file_message.__dict__}\033[0m') + + # The docs call subscription.add_listener(PrintListener()). The SDK then calls PrintListener.message(message) + # without the pubnub argument and raises TypeError, so the client-level add_listener is used here. + pubnub.add_listener(PrintListener()) + + subscription = pubnub.channel("my_channel").subscription() + subscription.subscribe() + # snippet.end + + await connected(seen) + await publisher.publish().channel(name).message('generic message').future() + await publisher.signal().channel(name).message('generic signal').future() + await aeventually(lambda: a.contains(printed.text(), 'PrintListener.message:\ngeneric message', 'message printed')) + await aeventually(lambda: a.contains(printed.text(), f'PrintListener.signal:\ngeneric signal from {publisher.uuid}', + 'signal printed')) + + +@step('listeners_subscription_set', role='open') +async def test_listeners_subscription_set(aclients, capsys): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + names_in_set = [names.with_suffix('my_channel'), names.with_suffix('my_channel_2')] + group_name = names.with_suffix('cg_1') + grouped = names.with_suffix('grouped_channel') + await publisher.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).future() + world.record('channel_group', group_name) + seen = attach(pubnub) + printed = Printed(capsys) + + # snippet.listeners_subscription_set + def on_message(listener): + def message_callback(message): + print(f"\033[94mMessage received on: {listener}: \n{message.message}\033[0m\n") + return message_callback + + subscription_set = pubnub.subscription_set([ + pubnub.channel("my_channel").subscription(), + pubnub.channel("my_channel_2").subscription(), + pubnub.channel_group("cg_1").subscription() + ]) + subscription_set.on_message = on_message('set_listener') + + subscription_set.subscribe() + # snippet.end + + await connected(seen) + for target, text in zip(names_in_set + [grouped], ['first', 'second', 'third']): + await publisher.publish().channel(target).message(text).future() + for text in ['first', 'second', 'third']: + await aeventually(lambda: a.contains(printed.text(), f'Message received on: set_listener: \n{text}\x1b', text)) + + +@step('listeners_presence', role='open') +async def test_listeners_presence(aclients, capsys): + pubnub = aclients.asyncio('open', 'subscriber') + name = names.with_suffix('my_channel') + seen = attach(pubnub) + printed = Printed(capsys) + + # snippet.listeners_presence + def on_presence(listener): + def presence_callback(presence): + print(f"\033[0;32mPresence received on: {listener}: \t{presence.uuid} {presence.event}s " + f"{presence.subscription or presence.channel}\033[0m") + return presence_callback + + subscription = pubnub.channel("my_channel").subscription(with_presence=True) + subscription.on_presence = on_presence('presence_listener') + + subscription.subscribe() + # snippet.end + + await connected(seen) + other = aclients.asyncio('open', 'second_user') + await alisten(other, name) + await aeventually( + lambda: a.contains(printed.text(), f'presence_listener: \t{other.uuid} joins {name}', 'join printed')) + + +@step('listeners_status', role='open') +async def test_listeners_status(aclients, capsys): + pubnub = aclients.asyncio('open', 'subscriber') + printed = Printed(capsys) + + # snippet.listeners_status + # snippet.hide + from pubnub.pubnub import SubscribeListener + # snippet.show + + class PrintListener(SubscribeListener): + def status(self, pubnub, status): + print(f'Status:\n{status.__dict__}') + + def message(self, pubnub, message): + pass + + def presence(self, pubnub, presence): + pass + + listener = PrintListener() + pubnub.add_listener(listener) + # snippet.end + + pubnub.subscribed_by_suite = True + pubnub.channel("my_channel").subscription().subscribe() + await aeventually(lambda: a.contains(printed.text(), 'PNConnectedCategory', 'connected status printed')) + a.contains(printed.text(), 'Status:\n{', 'status dictionary printed') + + +@step('unsubscribe_subscription', role='open') +async def test_unsubscribe_subscription(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + single, control = names.with_suffix('my_channel'), names.with_suffix('ch3') + seen = attach(pubnub) + pubnub.channel(control).subscription().subscribe() + + t1_subscription = pubnub.channel(single).subscription() + t1_subscription.subscribe() + await connected(seen) + await publisher.publish().channel(single).message('before').future() + await aeventually(lambda: a.eq(texts(seen), ['before'], 'message before unsubscribe')) + + # snippet.unsubscribe_subscription + t1_subscription.unsubscribe() + # snippet.end + + await aeventually(lambda: a.eq(pubnub.get_subscribed_channels(), [control], 'channels left')) + await publisher.publish().channel(single).message('after single').future() + await publisher.publish().channel(control).message('after control').future() + await aeventually(lambda: a.contains(texts(seen), 'after control', 'the control channel still delivers')) + a.not_contains(texts(seen), 'after single', 'no message after unsubscribe') + + +@step('unsubscribe_set', role='open') +async def test_unsubscribe_set(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + first, second, control = [names.with_suffix(n) for n in ('ch1', 'ch2', 'ch3')] + seen = attach(pubnub) + pubnub.channel(control).subscription().subscribe() + + subscription_set = pubnub.subscription_set([ + pubnub.channel(first).subscription(), + pubnub.channel(second).subscription() + ]) + subscription_set.subscribe() + await connected(seen) + await publisher.publish().channel(first).message('before first').future() + await publisher.publish().channel(second).message('before second').future() + await aeventually( + lambda: a.eq(sorted(texts(seen)), ['before first', 'before second'], 'messages before unsubscribe')) + + # snippet.unsubscribe_set + subscription_set.unsubscribe() + # snippet.end + + await aeventually(lambda: a.eq(pubnub.get_subscribed_channels(), [control], 'channels left')) + await publisher.publish().channel(first).message('after first').future() + await publisher.publish().channel(second).message('after second').future() + await publisher.publish().channel(control).message('after control').future() + await aeventually(lambda: a.contains(texts(seen), 'after control', 'the control channel still delivers')) + a.not_contains(texts(seen), 'after first', 'no message after unsubscribe') + a.not_contains(texts(seen), 'after second', 'no message after unsubscribe') + + +@step('unsubscribe_all', role='open') +async def test_unsubscribe_all(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + single, first = [names.with_suffix(n) for n in ('my_channel', 'ch1')] + seen = attach(pubnub) + pubnub.channel(single).subscription().subscribe() + pubnub.subscription_set([pubnub.channel(first).subscription()]).subscribe() + await connected(seen) + a.eq(sorted(pubnub.get_subscribed_channels()), sorted([single, first])) + + # snippet.unsubscribe_all + pubnub.unsubscribe_all() + # snippet.end + + # unsubscribe_all() leaves the registry unusable (see the report), so the harness must not call it again + pubnub.subscribed_by_suite = False + await aeventually(lambda: a.eq(pubnub.get_subscribed_channels(), [], 'channels left')) + await aeventually( + lambda: a.contains(seen.categories, PNStatusCategory.PNAcknowledgmentCategory, 'leave acknowledged')) + await publisher.publish().channel(single).message('after single').future() + await publisher.publish().channel(first).message('after first').future() + a.eq(texts(seen), [], 'no message after unsubscribe_all') + + +class StatusCollector(Collector): + """Collector that keeps the status objects, not only their categories.""" + + def __init__(self): + super().__init__() + self.statuses = [] + + def status(self, pubnub, status): + super().status(pubnub, status) + with self._lock: + self.statuses.append(status) + + +def attach_status(client): + collector = StatusCollector() + client.subscribed_by_suite = True + client.add_listener(collector) + return collector + + +@step('subscribe_old_channel', role='open') +async def test_subscribe_old_channel(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + name = names.with_suffix('my_channel') + seen = attach_status(pubnub) + + # snippet.subscribe_old_channel + pubnub.subscribe() \ + .channels('my_channel') \ + .execute() + # snippet.end + + await connected(seen) + connect = [s for s in seen.statuses if s.category == PNStatusCategory.PNConnectedCategory][0] + a.ok(not connect.is_error(), 'connected status is not an error') + a.eq(connect.error_data, None) + envelope = await publisher.publish().channel(name).message({'hello': 'old'}).meta({'k': 'v'}).future() + await publisher.signal().channel(name).message('old signal').future() + await aeventually(lambda: a.eq(len(seen.messages), 1, 'messages received')) + await aeventually(lambda: a.eq(len(seen.signals), 1, 'signals received')) + message = seen.messages[0] + a.eq((message.message, message.channel, message.subscription), ({'hello': 'old'}, name, None)) + a.eq((message.timetoken, message.user_metadata, message.publisher), + (envelope.result.timetoken, {'k': 'v'}, publisher.uuid)) + signal = seen.signals[0] + a.eq((signal.message, signal.channel, signal.publisher, signal.timetoken > 0), + ('old signal', name, publisher.uuid, True)) + + +@step('subscribe_old_multiple', role='open') +async def test_subscribe_old_multiple(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + both = [names.with_suffix('my_channel1'), names.with_suffix('my_channel2')] + seen = attach(pubnub) + + # snippet.subscribe_old_multiple + pubnub.subscribe() \ + .channels(["my_channel1", "my_channel2"]) \ + .execute() + # snippet.end + + await connected(seen) + for target in both: + await publisher.publish().channel(target).message(f'to {target}').future() + await aeventually(lambda: a.eq(sorted(texts(seen)), sorted(f'to {t}' for t in both), 'both channels deliver')) + + +@step('subscribe_old_wildcard', role='open') +async def test_subscribe_old_wildcard(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + matching = f"foo.{names.with_suffix('bar')}" + seen = attach(pubnub) + + # snippet.subscribe_old_wildcard + pubnub.subscribe() \ + .channels("foo.*") \ + .execute() + # snippet.end + + await connected(seen) + await publisher.publish().channel(matching).message('wildcard hit').future() + await aeventually(lambda: a.contains(texts(seen), 'wildcard hit', 'wildcard delivers')) + hit = [m for m in seen.messages if m.message == 'wildcard hit'][0] + a.eq((hit.channel, hit.subscription), (matching, 'foo.*')) + + +@step('subscribe_old_group', role='open') +async def test_subscribe_old_group(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + group_name = names.with_suffix('awesome_channel_group') + grouped = names.with_suffix('grouped_channel') + await publisher.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).future() + world.record('channel_group', group_name) + seen = attach(pubnub) + + # snippet.subscribe_old_group + pubnub.subscribe() \ + .channel_groups("awesome_channel_group") \ + .execute() + # snippet.end + + await connected(seen) + await publisher.publish().channel(grouped).message('group hit').future() + await aeventually(lambda: a.eq(texts(seen), ['group hit'], 'group delivers')) + a.eq((seen.messages[0].channel, seen.messages[0].subscription), (grouped, group_name)) + + +@step('subscribe_old_presence', 'subscribe_old_group_presence', role='open') +async def test_subscribe_old_presence(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + name = names.with_suffix('my_channel') + group_name = names.with_suffix('awesome_channel_group') + grouped = names.with_suffix('grouped_channel') + await aclients.asyncio('open', 'publisher').add_channel_to_channel_group().channels([grouped]) \ + .channel_group(group_name).future() + world.record('channel_group', group_name) + seen = attach(pubnub) + + # snippet.subscribe_old_presence + pubnub.subscribe()\ + .channels("my_channel")\ + .with_presence()\ + .execute() + # snippet.end + + # snippet.subscribe_old_group_presence + pubnub.subscribe() \ + .channel_groups("awesome_channel_group") \ + .with_presence() \ + .execute() + # snippet.end + + await connected(seen) + other = aclients.asyncio('open', 'second_user') + await alisten(other, name) + await aeventually(lambda: a.contains([(p.event, p.channel) for p in seen.presence_events if p.uuid == other.uuid], + ('join', name), 'join on the channel')) + # Presence of a channel group is announced per member channel: a client that subscribes to the group itself + # produces no join, a client that subscribes to a member channel does, and the event carries the group in + # `subscription` (workers.py _process_incoming_payload, subscription_match with the -pnpres suffix stripped). + member = aclients.asyncio('open', 'member_user') + await alisten(member, grouped) + await aeventually(lambda: a.contains([(p.event, p.channel, p.subscription) for p in seen.presence_events + if p.uuid == member.uuid], + ('join', grouped, group_name), 'join of a member channel through the group')) + join = [p for p in seen.presence_events if p.event == 'join' and p.uuid == other.uuid and p.channel == name][0] + a.ok(isinstance(join.timestamp, int) and join.occupancy >= 1, 'join carries timestamp and occupancy') + + +@step('subscribe_old_state', role='open') +async def test_subscribe_old_state(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + name = names.with_suffix('my_channel') + + # snippet.subscribe_old_state + import asyncio + from pubnub.enums import PNStatusCategory + from pubnub.pubnub_asyncio import SubscribeListener + + class MySubscribeListener(SubscribeListener): + def status(self, pubnub, status): + if status.category == PNStatusCategory.PNConnectedCategory: + state = { + 'field_a': 'awesome', + 'field_b': 10 + } + asyncio.ensure_future( + pubnub.set_state() + .channels('my_channel') + .state(state) + .future() + ) + + def message(self, pubnub, message): + pass + + def presence(self, pubnub, presence): + pass + + pubnub.add_listener(MySubscribeListener()) + pubnub.subscribe().channels("my_channel").execute() + # snippet.end + + pubnub.subscribed_by_suite = True + reader = aclients.asyncio('open', 'reader') + + async def check_state(): + fetched = await reader.get_state().channels(name).uuid(pubnub.uuid).future() + a.eq(fetched.result.channels, {name: {'field_a': 'awesome', 'field_b': 10}}, 'state set from the callback') + + await aeventually(check_state) + here = (await reader.here_now().channels(name).include_state(True).future()).result + a.eq([(o.uuid, o.state) for o in here.channels[0].occupants], + [(pubnub.uuid, {'field_a': 'awesome', 'field_b': 10})]) + + +@compile_only('subscribe_old_logging', reason='DEBUG logging writes every request URL with the keys to the output') +async def logging_sample(): + # snippet.subscribe_old_logging + import asyncio + import logging + import pubnub + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub_asyncio import PubNubAsyncio, SubscribeListener + + pubnub.set_stream_logger('pubnub', logging.DEBUG) + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = 'demo' + pnconfig.publish_key = 'demo' + pnconfig.user_id = 'my_user_id' + + async def main(): + client = PubNubAsyncio(pnconfig) + client.add_listener(SubscribeListener()) + client.subscribe().channels("my_channel").execute() + + # Keep the loop running so the subscription can receive messages. + await asyncio.sleep(500) + await client.stop() + + asyncio.run(main()) + # snippet.end + return main + + +@step('listeners_old_add', role='open') +async def test_listeners_old_add(aclients, capsys): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + name = names.with_suffix('my_channel') + printed = Printed(capsys) + seen = attach(pubnub) + + # snippet.listeners_old_add + from pubnub.callbacks import SubscribeCallback + from pubnub.enums import PNOperationType, PNStatusCategory + + class MySubscribeCallback(SubscribeCallback): + def message(self, pubnub, message): + print("Message channel: %s" % message.channel) + print("Message subscription: %s" % message.subscription) + print("Message timetoken: %s" % message.timetoken) + print("Message payload: %s" % message.message) + print("Message publisher: %s" % message.publisher) + + def presence(self, pubnub, presence): + # Can be join, leave, state-change, timeout, or interval + print("Presence event: %s" % presence.event) + + # The channel to which the message was published + print("Presence channel: %s" % presence.channel) + + # Number of users subscribed to the channel + print("Presence occupancy: %s" % presence.occupancy) + + # User state + print("Presence state: %s" % presence.state) + + # Channel group or wildcard subscription match, if any + print("Presence subscription: %s" % presence.subscription) + + # UUID to which this event is related + print("Presence UUID: %s" % presence.uuid) + + # Publish timetoken + print("Presence timestamp: %s" % presence.timestamp) + + # Current timetoken + print("Presence timetoken: %s" % presence.timetoken) + + # List of users that have joined the channel (if event is 'interval') + print("Presence joined: %s" % presence.join) + + # List of users that have left the channel (if event is 'interval') + print("Presence left: %s" % presence.leave) + + # List of users that have timed-out off the channel (if event is 'interval') + print("Presence timed out: %s" % presence.timeout) + + def status(self, pubnub, status): + # The status object returned is always related to subscribe but could contain + # information about subscribe, heartbeat, or errors + # use the operationType to switch on different options + if status.operation == PNOperationType.PNSubscribeOperation \ + or status.operation == PNOperationType.PNUnsubscribeOperation: + if status.category == PNStatusCategory.PNConnectedCategory: + pass + # This is expected for a subscribe, this means there is no error or issue whatsoever + elif status.category == PNStatusCategory.PNReconnectedCategory: + pass + # This usually occurs if subscribe temporarily fails but reconnects. This means + # there was an error but there is no longer any issue + elif status.category == PNStatusCategory.PNDisconnectedCategory: + pass + # This is the expected category for an unsubscribe. This means there + # was no error in unsubscribing from everything + elif status.category == PNStatusCategory.PNUnexpectedDisconnectCategory: + pass + # This is usually an issue with the internet connection, this is an error, handle + # appropriately retry will be called automatically + elif status.category == PNStatusCategory.PNAccessDeniedCategory: + pass + # This means that Access Manager does not allow this client to subscribe to this + # channel and channel group configuration. This is another explicit error + else: + pass + # This is usually an issue with the internet connection, this is an error, handle appropriately + # retry will be called automatically + else: + pass + # Encountered unknown status type + + def signal(self, pubnub, signal): + print("Signal channel: %s" % signal.channel) + print("Signal subscription: %s" % signal.subscription) + print("Signal timetoken: %s" % signal.timetoken) + print("Signal payload: %s" % signal.message) + print("Signal publisher: %s" % signal.publisher) + + def message_action(self, pubnub, message_action): + print("Message action type: %s" % message_action.type) + print("Message action value: %s" % message_action.value) + print("Message action uuid: %s" % message_action.uuid) + print("Message action action_timetoken: %s" % message_action.action_timetoken) + print("Message action message_timetoken: %s" % message_action.message_timetoken) + + pubnub.add_listener(MySubscribeCallback()) + # snippet.end + + pubnub.subscribe().channels(name).with_presence().execute() + await connected(seen) + envelope = await publisher.publish().channel(name).message({'n': 1}).future() + await publisher.signal().channel(name).message('sig').future() + action = PNMessageAction().create(type='reaction', value='smile', message_timetoken=envelope.result.timetoken) + await publisher.add_message_action().channel(name).message_action(action).future() + other = aclients.asyncio('open', 'second_user') + await alisten(other, name) + for text in (f'Message channel: {name}', "Message payload: {'n': 1}", f'Message publisher: {publisher.uuid}', + f'Message timetoken: {envelope.result.timetoken}', 'Signal payload: sig', + 'Message action type: reaction', 'Message action value: smile', + f'Message action message_timetoken: {envelope.result.timetoken}', + 'Presence event: join', f'Presence channel: {name}', f'Presence UUID: {other.uuid}'): + await aeventually(lambda: a.contains(printed.text(), text, 'printed by the docs listener')) + a.ok(len(seen.messages) == 1) + + +@step('listeners_old_remove', role='open') +async def test_listeners_old_remove(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + name = names.with_suffix('my_channel') + keeper = attach(pubnub) + + class MySubscribeCallback(Collector): + pass + + # snippet.listeners_old_remove + my_listener = MySubscribeCallback() + + pubnub.add_listener(my_listener) + + # some time later + pubnub.remove_listener(my_listener) + # snippet.end + + pubnub.add_listener(my_listener) + pubnub.subscribe().channels(name).execute() + await connected(keeper) + await publisher.publish().channel(name).message('first').future() + await aeventually(lambda: a.eq(texts(keeper), ['first'], 'the kept listener receives')) + await aeventually(lambda: a.eq(texts(my_listener), ['first'], 'a listener that is added receives')) + pubnub.remove_listener(my_listener) + await publisher.publish().channel(name).message('second').future() + await aeventually(lambda: a.eq(texts(keeper), ['first', 'second'], 'the kept listener still receives')) + a.eq(texts(my_listener), ['first'], 'a removed listener receives nothing more') + + +@step('listeners_old_disconnects', role='open') +async def test_listeners_old_disconnects(aclients, caplog): + pubnub = aclients.asyncio('open', 'subscriber') + seen = attach(pubnub) + + # snippet.listeners_old_disconnects + import logging + + from pubnub.callbacks import SubscribeCallback + from pubnub.enums import PNStatusCategory + + logger = logging.getLogger(__name__) + + class HandleDisconnectsCallback(SubscribeCallback): + def status(self, pubnub, status): + if status.category == PNStatusCategory.PNUnexpectedDisconnectCategory: + # internet got lost, do some magic and call reconnect when ready + pubnub.reconnect() + elif status.category == PNStatusCategory.PNTimeoutCategory: + # do some magic and call reconnect when ready + pubnub.reconnect() + else: + logger.debug(status) + + def presence(self, pubnub, presence): + pass + + def message(self, pubnub, message): + pass + + def signal(self, pubnub, signal): + pass + + disconnect_listener = HandleDisconnectsCallback() + + pubnub.add_listener(disconnect_listener) + # snippet.end + + with caplog.at_level(logging.DEBUG): + pubnub.subscribe().channels(names.with_suffix('my_channel')).execute() + await connected(seen) + await aeventually(lambda: a.ok( + any('PNConnectedCategory' in r.getMessage() or 'PNStatus' in r.getMessage() + for r in caplog.records if r.name == __name__ or r.name.endswith('subscribe')), + 'the docs logger saw the connected status')) + + +@step('unsubscribe_old_channel', role='open') +async def test_unsubscribe_old_channel(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + single, control = names.with_suffix('my_channel'), names.with_suffix('ch3') + seen = attach(pubnub) + observer = await alisten(aclients.asyncio('open', 'observer'), single, presence=True) + pubnub.subscribe().channels([single, control]).execute() + await connected(seen) + await publisher.publish().channel(single).message('before').future() + await aeventually(lambda: a.eq(texts(seen), ['before'], 'message before unsubscribe')) + await aeventually(lambda: a.contains([(p.event, p.uuid) for p in observer.presence_events], + ('join', pubnub.uuid), 'the observer sees the join')) + + # snippet.unsubscribe_old_channel + pubnub.unsubscribe() \ + .channels("my_channel") \ + .execute() + # snippet.end + + await aeventually(lambda: a.eq(pubnub.get_subscribed_channels(), [control], 'channels left')) + await publisher.publish().channel(single).message('after single').future() + await publisher.publish().channel(control).message('after control').future() + await aeventually(lambda: a.contains(texts(seen), 'after control', 'the control channel still delivers')) + a.not_contains(texts(seen), 'after single', 'no message after unsubscribe') + + # Documented: with the default configuration (enable_presence_heartbeat is False) the asyncio client sends no + # leave request (pubnub_asyncio.py adapt_unsubscribe_builder). The sentinel is the join of a client that + # subscribes after the unsubscribe: the observer has received a later event on the same channel, so a leave that + # had been sent would be in its list by now. + sentinel = aclients.asyncio('open', 'sentinel') + await alisten(sentinel, single) + await aeventually(lambda: a.contains([(p.event, p.uuid) for p in observer.presence_events], + ('join', sentinel.uuid), 'the observer sees the later sentinel join')) + a.not_contains([(p.event, p.uuid) for p in observer.presence_events], ('leave', pubnub.uuid), + 'no leave event with the default configuration') + + # Documented: with enable_presence_heartbeat and a heartbeat interval above 0 the client sends the leave. + beating = aclients.asyncio('open', 'beating_subscriber') + beating.config.enable_presence_heartbeat = True + beating.config.set_presence_timeout(20) + a.ok(beating.config.heartbeat_interval > 0, 'heartbeat interval is above 0') + beating_seen = attach(beating) + beating.subscribe().channels([single, control]).execute() + await connected(beating_seen) + await aeventually(lambda: a.contains([(p.event, p.uuid) for p in observer.presence_events], + ('join', beating.uuid), 'the observer sees the join of the heartbeat client')) + beating.unsubscribe() \ + .channels(single) \ + .execute() + await aeventually(lambda: a.contains([(p.event, p.uuid) for p in observer.presence_events], + ('leave', beating.uuid), 'leave event with the presence heartbeat enabled')) + leave = [p for p in observer.presence_events if p.event == 'leave' and p.uuid == beating.uuid][0] + a.ok(isinstance(leave.timestamp, int) and isinstance(leave.occupancy, int), 'leave carries timestamp, occupancy') + a.not_contains([(p.event, p.uuid) for p in observer.presence_events], ('leave', pubnub.uuid), + 'the default client still sent no leave') + + +@step('unsubscribe_old_multiple', 'unsubscribe_old_group', role='open') +async def test_unsubscribe_old_multiple_and_group(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + both = [names.with_suffix('my_channel1'), names.with_suffix('my_channel2')] + groups = [names.with_suffix('my_group1'), names.with_suffix('my_group2')] + control = names.with_suffix('ch3') + grouped = [names.with_suffix('grouped_one'), names.with_suffix('grouped_two')] + for group_name, channel_name in zip(groups, grouped): + await publisher.add_channel_to_channel_group().channels([channel_name]).channel_group(group_name).future() + world.record('channel_group', group_name) + seen = attach(pubnub) + pubnub.subscribe().channels(both + [control]).channel_groups(groups).execute() + await connected(seen) + for target in both + grouped: + await publisher.publish().channel(target).message(f'before {target}').future() + await aeventually(lambda: a.eq(len(seen.messages), 4, 'messages before unsubscribe')) + + # snippet.unsubscribe_old_multiple + pubnub.unsubscribe() \ + .channels(["my_channel1", "my_channel2"]) \ + .execute() + # snippet.end + + # snippet.unsubscribe_old_group + pubnub.unsubscribe() \ + .channel_groups(["my_group1", "my_group2"]) \ + .execute() + # snippet.end + + await aeventually(lambda: a.eq((pubnub.get_subscribed_channels(), pubnub.get_subscribed_channel_groups()), + ([control], []), 'subscriptions left')) + for target in both + grouped + [control]: + await publisher.publish().channel(target).message(f'after {target}').future() + await aeventually(lambda: a.contains(texts(seen), f'after {control}', 'the control channel still delivers')) + a.eq(sorted(m for m in texts(seen) if m.startswith('after')), [f'after {control}'], 'nothing else after') + + +@step('unsubscribe_all_old', role='open') +async def test_unsubscribe_all_old(aclients): + pubnub = aclients.asyncio('open', 'subscriber') + publisher = aclients.asyncio('open', 'publisher') + single = names.with_suffix('my_channel') + group_name = names.with_suffix('my_group1') + grouped = names.with_suffix('grouped_one') + await publisher.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).future() + world.record('channel_group', group_name) + seen = attach(pubnub) + pubnub.subscribe().channels(single).channel_groups(group_name).execute() + await connected(seen) + a.eq((pubnub.get_subscribed_channels(), pubnub.get_subscribed_channel_groups()), ([single], [group_name])) + + # snippet.unsubscribe_all_old + pubnub.unsubscribe_all() + # snippet.end + + # unsubscribe_all() leaves the registry unusable (see the report), so the harness must not call it again + pubnub.subscribed_by_suite = False + await aeventually(lambda: a.eq((pubnub.get_subscribed_channels(), pubnub.get_subscribed_channel_groups()), ([], []), + 'subscriptions left')) + await aeventually( + lambda: a.contains(seen.categories, PNStatusCategory.PNAcknowledgmentCategory, 'leave acknowledged')) + await publisher.publish().channel(single).message('after single').future() + await publisher.publish().channel(grouped).message('after group').future() + a.eq(texts(seen), [], 'no message after unsubscribe_all') diff --git a/docs-snippets/asyncio/zz_application_level.py b/docs-snippets/asyncio/zz_application_level.py new file mode 100644 index 00000000..9aa43e24 --- /dev/null +++ b/docs-snippets/asyncio/zz_application_level.py @@ -0,0 +1,55 @@ +"""Docs page: access-manager-v2 (asyncio), the application level grant. Kept in its own module so it runs last. + +Same as sync/zz_application_level.py. A downgrade is not asserted to apply at once. future() resolves with an +exception object on failure, so the result type is asserted instead of a try/except. +""" +from pubnub.exceptions import PubNubAsyncioException + +from harness import assertions as a +from harness.step import step + +FLAGS = ('read', 'write', 'manage', 'delete', 'get', 'update', 'join') + + +async def close_application_level(admin): + builder = admin.grant() + for flag in FLAGS: + builder = getattr(builder, flag)(False) + return await builder.ttl(1).future() + + +def accepted(resolved): + a.ok(not isinstance(resolved, PubNubAsyncioException), 'the grant was accepted') + return resolved.result + + +@step('grant_subkey_all', 'grant_application_level', role='pam', timeout=300) +async def test_grant_application_level(aclients): + pubnub = aclients.asyncio('pam', 'admin') + try: + # snippet.grant_subkey_all + envelope = await pubnub.grant().read(True).write(True).future() + # snippet.end + + result = accepted(envelope) + a.eq(result.level, 'subkey') + a.eq(result.ttl, 1440) + a.eq((result.read_enabled, result.write_enabled), (True, True)) + finally: + closed = await close_application_level(pubnub) + + try: + # snippet.grant_application_level + envelope = await pubnub.grant(). \ + read(True).write(True). \ + future() + # snippet.end + + result = accepted(envelope) + a.eq(result.level, 'subkey') + a.eq((result.read_enabled, result.write_enabled), (True, True)) + finally: + closed = await close_application_level(pubnub) + result = accepted(closed) + a.eq(result.level, 'subkey') + a.eq((result.read_enabled, result.write_enabled), (False, False)) diff --git a/docs-snippets/conftest.py b/docs-snippets/conftest.py new file mode 100644 index 00000000..55c2e4d7 --- /dev/null +++ b/docs-snippets/conftest.py @@ -0,0 +1,98 @@ +"""pytest wiring for the docs snippet suite: recorder, key loader, client fixtures, no-skip policy, reporter.""" +import os +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) # for "harness"; there is no __init__.py in sync or asyncio + +from harness import env, recorder, world # noqa: E402 + +recorder.install() +env.load() + +pytest_plugins = ['harness.reporter'] + + +def pytest_addoption(parser): + parser.addoption('--shard', action='store', default='all', help='open, pam or all. Refused: -k in CI') + + +def pytest_collection_modifyitems(config, items): + if os.environ.get('CI') and (config.getoption('-k') or config.getoption('-m') and not config.getoption('--shard')): + raise pytest.UsageError('subset selection (-k) is refused in CI, every run must run all valid tests') + for item in items: + for mark in item.iter_markers(): + if mark.name in ('skip', 'skipif', 'xfail'): + raise pytest.UsageError(f'{item.nodeid}: skip, skipif and xfail are not allowed in this suite') + shard = config.getoption('--shard') + if shard != 'all': + items[:] = [i for i in items if i.get_closest_marker(shard)] + live = [i for i in items if i.get_closest_marker('open') or i.get_closest_marker('pam')] + roles = {m for i in live for m in ('open', 'pam') if i.get_closest_marker(m)} + if config.option.collectonly: + return + live = [i for i in items if i.get_closest_marker('open') or i.get_closest_marker('pam')] + if live and os.environ.get('DOCS_SNIPPETS_STAGED') != '1': + raise pytest.UsageError('run the suite through "python -m harness.cli run" so that it is staged') + gone = [n for r in sorted(roles) for n in env.missing(r)] + if live and gone: + raise pytest.UsageError('missing env: ' + ', '.join(gone)) + + +@pytest.fixture +def clients(): + c = world.Clients() + yield c + c.close_sync() + + +@pytest.fixture +async def aclients(): + c = world.Clients() + yield c + await c.close_asyncio() + + +@pytest.fixture(scope='session', autouse=True) +def _no_lingering_threads(): + yield + left = world.lingering_threads() + assert not left, 'non-daemon threads still alive after the run: ' + ', '.join(left) + + +@pytest.fixture(autouse=True) +def _remove_channel_groups_after_each_step(): + """Channel groups a step recorded in the ledger are removed right after that step, so a failing step cannot leave + members behind for the next step. The end-of-run janitor still covers anything this misses.""" + import json + path = world.ledger_path() + before = len(path.read_text().splitlines()) if path.exists() else 0 + yield + if not path.exists(): + return + created = [json.loads(line) for line in path.read_text().splitlines()[before:]] + groups = [c['name'] for c in created if c['kind'] == 'channel_group'] + if not groups: + return + client = world.Clients() + try: + pubnub = client.sync('open', 'group-cleanup') + for group in groups: + try: + pubnub.remove_channel_group().channel_group(group).sync() + except Exception: # the step may already have removed it + pass + finally: + client.close_sync() + + +@pytest.fixture(autouse=True) +def _own_names_per_step(request): + """Every step gets its own channel, group and user names (see names.test_tag).""" + from harness import names + rel = Path(request.node.fspath).resolve().relative_to(Path(__file__).resolve().parent).as_posix() + names.set_tag(names.test_tag(rel, request.node.originalname or request.node.name)) + yield + names.set_tag('') diff --git a/docs-snippets/coverage.json b/docs-snippets/coverage.json new file mode 100644 index 00000000..15cf6841 --- /dev/null +++ b/docs-snippets/coverage.json @@ -0,0 +1,2183 @@ +{ + "asyncio": { + "add_channels_builder": { + "file": "asyncio/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_channels" + }, + "add_channels_named": { + "file": "asyncio/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_channels" + }, + "add_channels_to_push_builder": { + "file": "asyncio/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_channels_to_push" + }, + "add_channels_to_push_kwargs": { + "file": "asyncio/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_channels_to_push" + }, + "add_message_action_builder": { + "file": "asyncio/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_message_action" + }, + "add_message_action_named": { + "file": "asyncio/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_message_action" + }, + "configuration_full": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_full_configuration" + }, + "delete_channel_group_builder": { + "file": "asyncio/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_channel_group" + }, + "delete_file_builder": { + "file": "asyncio/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_file" + }, + "delete_message_single_builder": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_single_message" + }, + "delete_message_single_named": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_single_message" + }, + "delete_messages_builder": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_messages" + }, + "delete_messages_named": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_messages" + }, + "download_file_builder": { + "file": "asyncio/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_download_file" + }, + "entity_channel": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_entities" + }, + "entity_channel_group": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_entities" + }, + "entity_channel_metadata": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_entities" + }, + "entity_user_metadata": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_entities" + }, + "fetch_history_builder": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fetch_history" + }, + "fetch_history_multiple": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fetch_history_caps" + }, + "fetch_history_named": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fetch_history" + }, + "fetch_history_range": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fetch_history_range" + }, + "fire_message": { + "file": "asyncio/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fire" + }, + "fire_post": { + "file": "asyncio/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fire_post" + }, + "get_file_url_builder": { + "file": "asyncio/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_file_url" + }, + "get_filter_expression": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_filter_expression" + }, + "get_message_actions_builder": { + "file": "asyncio/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_message_actions" + }, + "get_message_actions_named": { + "file": "asyncio/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_message_actions" + }, + "get_state_builder": { + "file": "asyncio/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_state" + }, + "get_subscribed_channel_groups": { + "file": "asyncio/misc.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_subscribed" + }, + "get_subscribed_channels": { + "file": "asyncio/misc.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_subscribed" + }, + "grant_application_level": { + "file": "asyncio/zz_application_level.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_application_level" + }, + "grant_categories": { + "file": "asyncio/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_categories" + }, + "grant_channel_all_users": { + "file": "asyncio/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_channel_level" + }, + "grant_channel_auth": { + "file": "asyncio/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_channel_and_auth_key" + }, + "grant_channel_group": { + "file": "asyncio/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_channel_group" + }, + "grant_channel_level": { + "file": "asyncio/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_channel_level" + }, + "grant_channel_ro_auth": { + "file": "asyncio/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_user_level" + }, + "grant_multiple_channels": { + "file": "asyncio/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_multiple_channels" + }, + "grant_presence": { + "file": "asyncio/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_presence_channel" + }, + "grant_subkey_all": { + "file": "asyncio/zz_application_level.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_application_level" + }, + "grant_token_builder": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_and_parse_token" + }, + "grant_token_categories": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_token_categories" + }, + "grant_token_multi": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_resources" + }, + "grant_token_multi_regex": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_resources" + }, + "grant_token_regex": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_resources" + }, + "grant_token_spaces_builder": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_spaces_users" + }, + "grant_token_spaces_multi": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_spaces_users" + }, + "grant_token_spaces_multi_regex": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_spaces_users" + }, + "grant_token_spaces_regex": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_spaces_users" + }, + "grant_user_level": { + "file": "asyncio/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_user_level" + }, + "here_now_builder": { + "file": "asyncio/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now" + }, + "here_now_groups": { + "file": "asyncio/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now_groups" + }, + "here_now_occupancy": { + "file": "asyncio/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now_occupancy" + }, + "here_now_state": { + "file": "asyncio/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now_state" + }, + "history_basic": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "history_end": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "history_include_timetoken": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "history_paging": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history_paging" + }, + "history_reverse": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "history_start_reverse": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "initialize_access_manager": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize_access_manager" + }, + "initialize_custom_event_loop": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize_custom_event_loop" + }, + "initialize_custom_request_handler": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize_custom_request_handler" + }, + "initialize_custom_user_id": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize" + }, + "initialize_non_secure": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize" + }, + "initialize_pubnub": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize" + }, + "initialize_read_only": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize_read_only" + }, + "initialize_ssl": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize" + }, + "list_channels_builder": { + "file": "asyncio/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_channels" + }, + "list_channels_named": { + "file": "asyncio/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_channels" + }, + "list_files_builder": { + "file": "asyncio/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_files" + }, + "list_push_channels_builder": { + "file": "asyncio/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_push_channels" + }, + "listeners_event_specific": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_event_specific" + }, + "listeners_generic": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_generic" + }, + "listeners_old_add": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_old_add" + }, + "listeners_old_disconnects": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_old_disconnects" + }, + "listeners_old_remove": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_old_remove" + }, + "listeners_presence": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_presence" + }, + "listeners_status": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_status" + }, + "listeners_subscription_set": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_subscription_set" + }, + "message_counts_builder": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_message_counts" + }, + "message_counts_multiple_builder": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_message_counts" + }, + "message_counts_multiple_named": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_message_counts" + }, + "message_counts_named": { + "file": "asyncio/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_message_counts" + }, + "parse_token": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_and_parse_token" + }, + "parse_token_categories": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_token_categories" + }, + "publish_builder": { + "file": "asyncio/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_message" + }, + "publish_file_message_builder": { + "file": "asyncio/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_file_message" + }, + "publish_metadata": { + "file": "asyncio/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_metadata" + }, + "publish_named": { + "file": "asyncio/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_message" + }, + "publish_store_post": { + "file": "asyncio/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_store_and_post" + }, + "remove_channels_builder": { + "file": "asyncio/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channels" + }, + "remove_channels_from_push_builder": { + "file": "asyncio/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channels_from_push" + }, + "remove_channels_named": { + "file": "asyncio/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channels" + }, + "remove_device_from_push_builder": { + "file": "asyncio/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_device_from_push" + }, + "remove_message_action_builder": { + "file": "asyncio/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_message_action" + }, + "remove_message_action_named": { + "file": "asyncio/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_message_action" + }, + "revoke_token": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_revoke_token" + }, + "send_file_builder": { + "file": "asyncio/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_send_file" + }, + "send_file_options": { + "file": "asyncio/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_send_file" + }, + "set_filter_expression": { + "file": "asyncio/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_filter_expression" + }, + "set_state_builder": { + "file": "asyncio/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_state" + }, + "set_state_groups_builder": { + "file": "asyncio/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_state_for_groups" + }, + "set_token": { + "file": "asyncio/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_set_token" + }, + "signal_message": { + "file": "asyncio/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_signal" + }, + "subscribe_multiple": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_multiple" + }, + "subscribe_old_channel": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_channel" + }, + "subscribe_old_group": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_group" + }, + "subscribe_old_group_presence": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_presence" + }, + "subscribe_old_logging": { + "file": "asyncio/subscribe.py", + "mode": "compile-only", + "reason": "DEBUG logging writes every request URL with the keys to the output", + "role": "open", + "step": "logging_sample" + }, + "subscribe_old_multiple": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_multiple" + }, + "subscribe_old_presence": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_presence" + }, + "subscribe_old_state": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_state" + }, + "subscribe_old_wildcard": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_wildcard" + }, + "subscribe_single": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_single" + }, + "time_get": { + "file": "asyncio/misc.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_time" + }, + "unsubscribe_all": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_all" + }, + "unsubscribe_all_old": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_all_old" + }, + "unsubscribe_old_channel": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_old_channel" + }, + "unsubscribe_old_group": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_old_multiple_and_group" + }, + "unsubscribe_old_multiple": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_old_multiple_and_group" + }, + "unsubscribe_set": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_set" + }, + "unsubscribe_subscription": { + "file": "asyncio/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_subscription" + }, + "where_now_other": { + "file": "asyncio/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_where_now" + }, + "where_now_self": { + "file": "asyncio/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_where_now" + } + }, + "sync": { + "add_channels_builder": { + "file": "sync/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_channels" + }, + "add_channels_named": { + "file": "sync/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_channels" + }, + "add_channels_to_push_builder": { + "file": "sync/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_channels_to_push" + }, + "add_channels_to_push_named": { + "file": "sync/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_channels_to_push" + }, + "add_message_action_builder": { + "file": "sync/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_message_action" + }, + "add_message_action_named": { + "file": "sync/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_add_message_action" + }, + "configuration_full": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_full_configuration" + }, + "crypto_module_aes_cbc": { + "file": "sync/encryption.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_aes_cbc_crypto_module" + }, + "crypto_module_legacy": { + "file": "sync/encryption.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_legacy_crypto_module" + }, + "crypto_module_partial": { + "file": "sync/encryption.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_partial_crypto_module" + }, + "decrypt_file": { + "file": "sync/encryption.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_encrypt_and_decrypt_file" + }, + "delete_channel_group_builder": { + "file": "sync/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_channel_group" + }, + "delete_file_builder": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_file" + }, + "delete_file_named": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_file" + }, + "delete_message_single_builder": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_single_message" + }, + "delete_message_single_named": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_single_message" + }, + "delete_messages_builder": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_messages" + }, + "delete_messages_named": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_delete_messages" + }, + "disconnect_method": { + "file": "sync/misc.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_disconnect_and_reconnect" + }, + "download_file_builder": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_download_file" + }, + "encrypt_file": { + "file": "sync/encryption.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_encrypt_and_decrypt_file" + }, + "entity_channel": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_entities" + }, + "entity_channel_group": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_entities" + }, + "entity_channel_metadata": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_entities" + }, + "entity_user_metadata": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_entities" + }, + "fetch_history_builder": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fetch_history" + }, + "fetch_history_multiple": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fetch_history_caps" + }, + "fetch_history_named": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fetch_history" + }, + "fetch_history_range": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fetch_history_range" + }, + "fire_message": { + "file": "sync/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fire" + }, + "fire_named": { + "file": "sync/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fire_post_and_named" + }, + "fire_post": { + "file": "sync/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_fire_post_and_named" + }, + "get_all_channel_metadata_builder": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_all_channel_metadata" + }, + "get_all_channel_metadata_named": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_all_channel_metadata" + }, + "get_all_uuid_metadata_builder": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_all_uuid_metadata" + }, + "get_all_uuid_metadata_named": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_all_uuid_metadata" + }, + "get_channel_members_builder": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_channel_members" + }, + "get_channel_members_named": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_channel_members" + }, + "get_channel_metadata_builder": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_channel_metadata" + }, + "get_channel_metadata_named": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_channel_metadata" + }, + "get_file_url_builder": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_file_url" + }, + "get_file_url_named": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_file_url" + }, + "get_filter_expression": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_filter_expression" + }, + "get_memberships_builder": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_memberships" + }, + "get_memberships_named": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_memberships" + }, + "get_message_actions_builder": { + "file": "sync/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_message_actions" + }, + "get_message_actions_named": { + "file": "sync/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_message_actions" + }, + "get_state_builder": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_state" + }, + "get_state_named": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_state" + }, + "get_subscribed_channel_groups": { + "file": "sync/misc.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_subscribed" + }, + "get_subscribed_channels": { + "file": "sync/misc.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_subscribed" + }, + "get_uuid_metadata_builder": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_uuid_metadata" + }, + "get_uuid_metadata_named": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_get_uuid_metadata" + }, + "grant_application_level": { + "file": "sync/zz_application_level.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_application_level" + }, + "grant_categories": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_categories" + }, + "grant_channel_all_users": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_channel_level" + }, + "grant_channel_auth": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_channel_and_auth_key" + }, + "grant_channel_group": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_channel_group" + }, + "grant_channel_level": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_channel_level" + }, + "grant_channel_ro_auth": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_user_level" + }, + "grant_multiple_channels": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_multiple_channels" + }, + "grant_presence": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_presence_channel" + }, + "grant_subkey_all": { + "file": "sync/zz_application_level.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_application_level" + }, + "grant_token_builder": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_and_parse_token" + }, + "grant_token_categories": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_token_categories" + }, + "grant_token_multi": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_resources" + }, + "grant_token_multi_regex": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_resources" + }, + "grant_token_named": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_and_parse_token" + }, + "grant_token_regex": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_resources" + }, + "grant_token_spaces_builder": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_spaces_users" + }, + "grant_token_spaces_multi": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_spaces_users" + }, + "grant_token_spaces_multi_regex": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_spaces_users" + }, + "grant_token_spaces_named": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_spaces_users" + }, + "grant_token_spaces_regex": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_token_spaces_users" + }, + "grant_user_level": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_user_level" + }, + "grant_uuid": { + "file": "sync/access_manager_v2.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_uuid" + }, + "here_now_builder": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now" + }, + "here_now_groups": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now_groups" + }, + "here_now_named": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now" + }, + "here_now_occupancy": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now_occupancy" + }, + "here_now_paginate": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now_paginate" + }, + "here_now_state": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_here_now_state" + }, + "history_basic": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "history_end": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "history_include_timetoken": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "history_paging": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history_paging" + }, + "history_reverse": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "history_start_reverse": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_history" + }, + "initialize_access_manager": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize_access_manager" + }, + "initialize_custom_request_handler": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize_custom_request_handler" + }, + "initialize_custom_user_id": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize" + }, + "initialize_non_secure": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize" + }, + "initialize_pubnub": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize" + }, + "initialize_read_only": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize_read_only" + }, + "initialize_ssl": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_initialize" + }, + "list_channels_builder": { + "file": "sync/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_channels" + }, + "list_channels_named": { + "file": "sync/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_channels" + }, + "list_files_builder": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_files" + }, + "list_files_named": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_files" + }, + "list_push_channels_builder": { + "file": "sync/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_push_channels" + }, + "list_push_channels_named": { + "file": "sync/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_list_push_channels" + }, + "listeners_event_specific": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_event_specific" + }, + "listeners_generic": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_generic" + }, + "listeners_old_add": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_old_add" + }, + "listeners_old_disconnects": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_old_disconnects" + }, + "listeners_old_remove": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_old_remove" + }, + "listeners_presence": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_presence" + }, + "listeners_status": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_status" + }, + "listeners_subscription_set": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_listeners_subscription_set" + }, + "manage_channel_members_builder": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_manage_channel_members" + }, + "manage_channel_members_named": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_manage_channel_members" + }, + "manage_memberships_builder": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_manage_memberships" + }, + "manage_memberships_named": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_manage_memberships" + }, + "message_counts_builder": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_message_counts" + }, + "message_counts_multiple_builder": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_message_counts" + }, + "message_counts_multiple_named": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_message_counts" + }, + "message_counts_named": { + "file": "sync/message_persistence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_message_counts" + }, + "parse_token": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_grant_and_parse_token" + }, + "parse_token_categories": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_token_categories" + }, + "pnchannelmembership_class": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_channel_membership_class" + }, + "pnuuid_class": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_uuid_class" + }, + "publish_builder": { + "file": "sync/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_message" + }, + "publish_file_message_builder": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_file_message" + }, + "publish_file_message_callback": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_file_message" + }, + "publish_metadata": { + "file": "sync/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_metadata" + }, + "publish_named": { + "file": "sync/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_message" + }, + "publish_store_post": { + "file": "sync/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_publish_store_and_post" + }, + "reconnect_documented": { + "file": "sync/misc.py", + "mode": "compile-only", + "reason": "pubnub.reconnect() raises AttributeError: PNSubscriptionRegistry has no reconnect (SDK bug)", + "role": "open", + "step": "reconnect_documented" + }, + "reconnect_method": { + "file": "sync/misc.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_disconnect_and_reconnect" + }, + "remove_channel_members_builder": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channel_members" + }, + "remove_channel_members_named": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channel_members" + }, + "remove_channel_metadata_builder": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channel_metadata" + }, + "remove_channel_metadata_named": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channel_metadata" + }, + "remove_channels_builder": { + "file": "sync/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channels" + }, + "remove_channels_from_push_builder": { + "file": "sync/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channels_from_push" + }, + "remove_channels_from_push_named": { + "file": "sync/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channels_from_push" + }, + "remove_channels_named": { + "file": "sync/channel_groups.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_channels" + }, + "remove_device_from_push_builder": { + "file": "sync/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_device_from_push" + }, + "remove_device_from_push_named": { + "file": "sync/mobile_push.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_device_from_push" + }, + "remove_memberships_builder": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_memberships" + }, + "remove_memberships_named": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_memberships" + }, + "remove_message_action_builder": { + "file": "sync/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_message_action" + }, + "remove_message_action_named": { + "file": "sync/message_actions.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_message_action" + }, + "remove_uuid_metadata_builder": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_uuid_metadata" + }, + "remove_uuid_metadata_named": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_remove_uuid_metadata" + }, + "revoke_token": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_revoke_token" + }, + "send_file_builder": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_send_file" + }, + "send_file_custom_message_type": { + "file": "sync/file_sharing.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_send_file" + }, + "set_channel_members_builder": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_set_channel_members" + }, + "set_channel_members_named": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_set_channel_members" + }, + "set_channel_metadata_builder": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_set_channel_metadata" + }, + "set_channel_metadata_named": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_set_channel_metadata" + }, + "set_filter_expression": { + "file": "sync/configuration.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_filter_expression" + }, + "set_memberships_builder": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_set_memberships" + }, + "set_memberships_named": { + "file": "sync/objects_membership.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_set_memberships" + }, + "set_state_builder": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_state" + }, + "set_state_groups_builder": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_state_for_groups" + }, + "set_state_groups_named": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_state_for_groups" + }, + "set_state_named": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_state" + }, + "set_token": { + "file": "sync/access_manager.py", + "mode": "live", + "reason": "", + "role": "pam", + "step": "test_set_token" + }, + "set_uuid_metadata_builder": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_set_uuid_metadata" + }, + "set_uuid_metadata_named": { + "file": "sync/objects.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_set_uuid_metadata" + }, + "signal_message": { + "file": "sync/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_signal" + }, + "signal_named": { + "file": "sync/publish.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_signal_named" + }, + "subscribe_multiple": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_multiple" + }, + "subscribe_old_channel": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_channel" + }, + "subscribe_old_group": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_group" + }, + "subscribe_old_group_presence": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_presence" + }, + "subscribe_old_logging": { + "file": "sync/subscribe.py", + "mode": "compile-only", + "reason": "DEBUG logging writes every request URL with the keys to the output", + "role": "open", + "step": "logging_sample" + }, + "subscribe_old_multiple": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_multiple" + }, + "subscribe_old_presence": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_presence" + }, + "subscribe_old_state": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_state" + }, + "subscribe_old_wildcard": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_old_wildcard" + }, + "subscribe_single": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_subscribe_single" + }, + "time_get": { + "file": "sync/misc.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_time" + }, + "unsubscribe_all": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_all" + }, + "unsubscribe_all_old": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_all_old" + }, + "unsubscribe_old_channel": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_old_channel" + }, + "unsubscribe_old_group": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_old_multiple_and_group" + }, + "unsubscribe_old_multiple": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_old_multiple_and_group" + }, + "unsubscribe_set": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_set" + }, + "unsubscribe_subscription": { + "file": "sync/subscribe.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_unsubscribe_subscription" + }, + "where_now_other_builder": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_where_now" + }, + "where_now_other_named": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_where_now" + }, + "where_now_self": { + "file": "sync/presence.py", + "mode": "live", + "reason": "", + "role": "open", + "step": "test_where_now" + } + } +} diff --git a/docs-snippets/fixtures/sample.gif b/docs-snippets/fixtures/sample.gif new file mode 100644 index 0000000000000000000000000000000000000000..a5fd168ea128cb6be0da1d2fa73000f7ecc271d6 GIT binary patch literal 134 zcmZ?wbhEHbWMp7uXkY+=|Ns9h{$ybUF?B!$NQQxl$%T=@Iwe23ShqMYv!Ecgq*x&> zv!bN5C{+iJ)k/run.json. +""" +import ast +import json +import re +import sys +from collections import Counter + +from . import manifest, registry + +ASSERT_NAMES = {'ok', 'eq', 'contains', 'not_contains', 'timetoken', 'raises'} +WAIT_NAMES = {'eventually', 'aeventually'} + + +def _assertion_calls(node, helpers): + count = 0 + for n in ast.walk(node): + if isinstance(n, ast.Assert): + count += 1 + elif isinstance(n, ast.Call): + f = n.func + is_a_call = isinstance(f, ast.Attribute) and isinstance(f.value, ast.Name) and f.value.id == 'a' + if is_a_call and f.attr in ASSERT_NAMES: + count += 1 + elif isinstance(f, ast.Name) and (f.id in ASSERT_NAMES or f.id in WAIT_NAMES): + count += 1 + elif isinstance(f, ast.Name) and f.id in helpers: + count += helpers[f.id] + return count + + +def _module_helpers(tree): + """Assertion count of module-level helpers, resolved one level, so a step that asserts through one counts.""" + plain = {} + for n in tree.body: + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and not n.name.startswith('test_'): + plain[n.name] = _assertion_calls(n, {}) + return plain + + +def audit(): + rows, bad = [], [] + for e in registry.scan(): + if e.mode != 'live': + rows.append((e, None)) + continue + path = registry.ROOT / e.file + text = path.read_text() + tree = ast.parse(text) + helpers = _module_helpers(tree) + funcs = (ast.FunctionDef, ast.AsyncFunctionDef) + fn = next(n for n in ast.walk(tree) if isinstance(n, funcs) and n.name == e.func) + inside = [(i, line) for i, line in enumerate(text.splitlines(), 1) if fn.lineno < i <= fn.end_lineno] + ends = [i for i, line in inside if re.match(r'\s*# snippet\.end\s*$', line)] + marker = re.compile(r'\s*# snippet\.([A-Za-z0-9_-]+)\s*$') + starts = [i for i, line in inside + if marker.match(line) and marker.match(line).group(1) not in ('end', 'hide', 'show')] + first_start = min(starts) + local = dict(helpers) + for n in ast.walk(fn): # helpers defined inside the step count when the step calls them + if n is not fn and isinstance(n, funcs): + local[n.name] = _assertion_calls(n, helpers) + stmts = [n for n in fn.body if getattr(n, 'end_lineno', 0) > first_start and not isinstance(n, funcs)] + count = sum(_assertion_calls(n, local) for n in stmts) + rows.append((e, count)) + if count == 0 or not ends: + bad.append(e) + return rows, bad + + +def latest_results(): + out = {} + for shard in ('open', 'pam'): + p = registry.ROOT / 'report' / shard / 'run.json' + if p.exists(): + d = json.loads(p.read_text()) + for t in d['tests']: + out[t['test']] = t['outcome'] + for k, v in d.get('steps', {}).items(): + out.setdefault('_steps', {})[k] = v + return out + + +def report(): + rows, bad = audit() + results = latest_results() + steps = results.get('_steps', {}) + lines = ['# Coverage and assertion audit', ''] + modes = Counter((e.docset, e.mode) for e, _ in rows) + ids = Counter() + for e, _ in rows: + ids[(e.docset, e.mode)] += len(e.ids) + lines += ['## Snippet ids by docset and mode', '', + '| docset | live steps | live ids | compile-only ids | stub ids |', '| :-- | --: | --: | --: | --: |'] + for ds in registry.DOCSETS: + cells = [modes[(ds, 'live')], ids[(ds, 'live')], ids[(ds, 'compile-only')], ids[(ds, 'stub')]] + lines.append(f'| {ds} | ' + ' | '.join(str(c) for c in cells) + ' |') + lines += ['', f'Steps with no assertion after the snippet, or a snippet without an end marker: **{len(bad)}**', ''] + for e in bad: + lines.append(f'* {e.file}::{e.func}') + lines += ['', '## Per step', '', + '| step | role | snippet ids | assertion calls (static) | assertions (last run) | last result |', + '| :-- | :-- | :-- | --: | --: | :-- |'] + for e, count in sorted(rows, key=lambda r: (r[0].docset, r[0].file, r[0].first)): + node = f'{e.file}::{e.func}' + if e.mode != 'live': + lines.append(f"| {node} | {e.role} | {', '.join(e.ids)} | n/a | n/a | {e.mode} ({e.reason}) |") + continue + runtime = steps.get(node, {}).get('assertions', 'n/a') + outcome = results.get(node, 'not run') + lines.append(f"| {node} | {e.role} | {', '.join(e.ids)} | {count} | {runtime} | {outcome} |") + docs = manifest.docs_ids() + known = {d: set(v) for d, v in manifest.build().items()} + lines += ['', '## Docs pages versus the manifest', ''] + for ds, found in docs.items(): + missing = len(found - known.get(ds, set())) + lines.append(f'* {ds}: {len(found)} snippet ids referenced by docs pages, {missing} missing from the manifest') + return '\n'.join(lines) + '\n', bad + + +def main(argv): + text, bad = report() + if '--write' in argv: + (registry.ROOT / 'report').mkdir(exist_ok=True) + (registry.ROOT / 'report' / 'coverage.md').write_text(text) + print('wrote report/coverage.md') + else: + print(text) + return 1 if bad else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/docs-snippets/harness/cli.py b/docs-snippets/harness/cli.py new file mode 100644 index 00000000..84318f60 --- /dev/null +++ b/docs-snippets/harness/cli.py @@ -0,0 +1,89 @@ +"""Command line: python -m harness.cli """ +import compileall +import os +import subprocess +import sys + +from . import env, janitor, lint_markers, manifest, preflight, registry, redact, stage + + +def _compile(): + ok = compileall.compile_dir(str(registry.ROOT), quiet=1, rx=__import__('re').compile(r'(\.run|report)')) + print('compile:', 'ok' if ok else 'FAILED') + return 0 if ok else 1 + + +def _reporter_selftest(): + redact.register('pub-c-00000000-0000-0000-0000-000000000000') + text = redact.scrub('GET /publish/pub-c-00000000-0000-0000-0000-000000000000/x') + bad = redact.leaks(text) + print('reporter self-test:', 'ok' if not bad else 'FAILED, a secret survived scrub()') + return 1 if bad else 0 + + +def dry(): + rc = _compile() | main(['selftest']) | lint_markers.main() | manifest.main(['--check']) | _reporter_selftest() + if os.environ.get('DOCS_SNIPPETS_DRY_COLLECT', '1') == '1': + rc |= subprocess.call([sys.executable, '-m', 'pytest', '--collect-only', '-q', '--shard', 'all'], + cwd=registry.ROOT, env={**os.environ, 'DOCS_SNIPPETS_STAGED': '1'}) and 0 + print('dry:', 'ok' if rc == 0 else 'FAILED') + return rc + + +def run(argv): + shard = 'all' + if '--shard' in argv: + shard = argv[argv.index('--shard') + 1] + extra = [a for a in argv if a not in ('--shard', shard, '--keep')] + if os.environ.get('CI') and ('-k' in extra or os.environ.get('DOCS_SNIPPETS_MUTATE')): + print('subset selection and mutation are refused in CI') + return 2 + for line in env.status(): + print('keys', line) + if env.missing('admin'): + print('preflight: not run, PUBNUB_ADMIN_API_KEY is missing') + elif preflight.main(): + print('preflight: a keyset lacks a feature, see above. The steps that need it will fail.') + target = stage.stage() + print('staged:', target.name) + environ = {**os.environ, 'DOCS_SNIPPETS_STAGED': '1', 'PYTHONUNBUFFERED': '1', + 'DOCS_SNIPPETS_REPORT': str(registry.ROOT / 'report' / shard), + 'DOCS_SNIPPETS_LEDGER_DIR': str(registry.ROOT / '.run')} + import pubnub + from pubnub.pubnub_core import PubNubCore + print('sdk:', pubnub.__file__, PubNubCore.SDK_VERSION) + rc = subprocess.call([sys.executable, '-u', '-m', 'pytest', '--shard', shard, *extra], cwd=target, env=environ) + if '--keep' not in argv: + import shutil + shutil.rmtree(target, ignore_errors=True) + print('janitor...') + rc |= janitor.main(shard) + return rc + + +def main(argv): + cmd = argv[0] if argv else 'dry' + if cmd == 'env': + print('\n'.join(env.status())) + return 0 + if cmd == 'lint': + return lint_markers.main() + if cmd == 'manifest': + return manifest.main(argv[1:]) + if cmd == 'dry': + return dry() + if cmd == 'stage': + print(stage.stage()) + return 0 + if cmd == 'run': + return run(argv[1:]) + if cmd == 'selftest': + return subprocess.call([sys.executable, '-m', 'pytest', '-q', 'selftest'], cwd=registry.ROOT) + if cmd == 'janitor': + return janitor.main() + print(__doc__) + return 2 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/docs-snippets/harness/env.py b/docs-snippets/harness/env.py new file mode 100644 index 00000000..fa74b3fd --- /dev/null +++ b/docs-snippets/harness/env.py @@ -0,0 +1,69 @@ +"""Key loader. Fixed env names, grouped in roles. Real env vars win over the optional env file. + +The loader never falls back to demo keys and prints only "set" or "missing" per role. +""" +import os +from pathlib import Path + +from . import redact + +ROLES = { + 'open': ['PUBLISH_KEY', 'SUBSCRIBE_KEY'], + 'pam': ['PAM_PUBLISH_KEY', 'PAM_SUBSCRIBE_KEY', 'PAM_SECRET_KEY'], + # Read only Admin API access for the preflight. No test step uses it. + 'admin': ['PUBNUB_ADMIN_API_KEY'], +} + +_loaded = False + + +def _read_env_file(path): + values = {} + p = Path(path) + if not p.is_file(): + return values + for raw in p.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith('#') or '=' not in line: + continue + name, _, value = line.partition('=') + values[name.strip()] = value.strip().strip('"').strip("'") + return values + + +def load(): + """Fill os.environ from DOCS_SNIPPETS_ENV_FILE (or docs-snippets/.env.local) without overriding real env vars.""" + global _loaded + if _loaded: + return + path = os.environ.get('DOCS_SNIPPETS_ENV_FILE') or str(Path(__file__).resolve().parent.parent / '.env.local') + for name, value in _read_env_file(path).items(): + if name in {n for names in ROLES.values() for n in names} and not os.environ.get(name): + os.environ[name] = value + for names in ROLES.values(): + for name in names: + redact.register(os.environ.get(name, '')) + _loaded = True + + +def missing(role): + load() + return [name for name in ROLES[role] if not os.environ.get(name)] + + +def status(): + """One line per role: 'set' or 'missing: NAME, NAME'. Never prints a value.""" + load() + lines = [] + for role in ROLES: + gone = missing(role) + lines.append(f'{role}: ' + ('set' if not gone else 'missing: ' + ', '.join(gone))) + return lines + + +def require(role): + """Return the env values of a role. A missing key fails the run with the exact names, there is no fallback.""" + gone = missing(role) + if gone: + raise RuntimeError(f'role {role!r} is missing env: ' + ', '.join(gone)) + return {name: os.environ[name] for name in ROLES[role]} diff --git a/docs-snippets/harness/janitor.py b/docs-snippets/harness/janitor.py new file mode 100644 index 00000000..db64dcfa --- /dev/null +++ b/docs-snippets/harness/janitor.py @@ -0,0 +1,86 @@ +"""Janitor: removes what runs left behind and proves no application-level Access Manager v2 grant is open. + +Runs at the end of every live run and after a failure or cancel in CI. A leftover is a resource whose name carries the +marker "dsn-" AND a base36 timestamp older than six hours. Nothing else is touched. +""" +import json +import re +import sys +import time + +from . import env, names, world + +MAX_AGE = 6 * 3600 +PATTERN = re.compile(names.MARKER + r'-([0-9a-z]{6,8})') + + +def _age(name): + m = PATTERN.search(name) + if not m: + return None + try: + return time.time() - int(m.group(1)[:6], 36) + except ValueError: + return None + + +def revoke_application_grant(): + """Close the application-level v2 grant (no channel, no auth key) so no run leaves it open.""" + client = world.Clients().sync('pam', 'janitor') + envelope = client.grant() \ + .read(False).write(False).manage(False).delete(False).get(False).update(False).join(False) \ + .ttl(1).sync() + return envelope.status.is_error() is False + + +def clean_app_context(role='open'): + client = world.Clients().sync(role, 'janitor') + removed = 0 + for fetch, remove in ((client.get_all_channel_metadata, client.remove_channel_metadata), + (client.get_all_uuid_metadata, client.remove_uuid_metadata)): + for item in fetch().limit(100).sync().result.data: + ident = item['id'] if isinstance(item, dict) else item.id + age = _age(ident) + if age is not None and age > MAX_AGE: + remove(ident).sync() + removed += 1 + return removed + + +def clean_ledger(): + removed = 0 + path = world.ledger_path() + if not path.exists(): + return removed + client = world.Clients().sync('open', 'janitor') + for line in path.read_text().splitlines(): + item = json.loads(line) + try: + if item['kind'] == 'channel_group': + client.remove_channel_group().channel_group(item['name']).sync() + elif item['kind'] == 'history': + client.delete_messages().channel(item['name']).sync() + removed += 1 + except Exception as e: # keep going. The final report names what is left + print('janitor: could not remove', item['kind'], '(' + type(e).__name__ + ')') + path.unlink() + return removed + + +def main(shard='all'): + """An open-only run never touches the pam keyset, so it cannot break a pam run that goes on at the same time.""" + problems = 0 + tasks = [('open', clean_app_context), ('open', clean_ledger)] + if shard in ('all', 'pam'): + tasks.insert(0, ('pam', revoke_application_grant)) + for role, task in tasks: + if env.missing(role): + print(f'janitor: {task.__name__} not run, role {role} is missing env') + problems += 1 + continue + print(f'janitor: {task.__name__} ->', task()) + return 1 if problems else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docs-snippets/harness/lint_markers.py b/docs-snippets/harness/lint_markers.py new file mode 100644 index 00000000..6e318b85 --- /dev/null +++ b/docs-snippets/harness/lint_markers.py @@ -0,0 +1,61 @@ +"""Marker lint. Fails on: unclosed or implicit-ended regions, duplicate ids, a marker outside a registered step, +a step whose ids differ from the markers in its body, and assertions inside a visible region.""" +import re +import sys +from collections import Counter + +from . import markers as mk +from . import registry + +# Calls and statements that belong outside the markers (hidden lines are allowed) +FORBIDDEN = re.compile( + r'(^|\W)(assert\b|ok\(|eq\(|contains\(|not_contains\(|timetoken\(|raises\(|eventually\(|aeventually\(|__snip__)') + + +def lint(): + problems = [] + entries = registry.scan() + seen = Counter() + for entry in entries: + for i in entry.ids: + seen[(entry.docset, i)] += 1 + for (docset, i), n in seen.items(): + if n > 1: + problems.append(f'{docset}: snippet id {i!r} is registered {n} times') + + on_disk = {str(p.relative_to(registry.ROOT)) for d in registry.DOCSETS + for p in (registry.ROOT / d).glob('*.py') if not p.name.startswith('_')} + files = sorted({e.file for e in entries} | on_disk) + for rel in files: + text = (registry.ROOT / rel).read_text() + regions = mk.regions(text) + file_entries = [e for e in entries if e.file == rel] + for r in regions: + if not r.end: + problems.append(f'{rel}:{r.start}: snippet {r.id!r} has no snippet.end') + owner = [e for e in file_entries if e.first <= r.start and r.end <= e.last] + if not owner: + problems.append(f'{rel}:{r.start}: snippet {r.id!r} is not inside a registered step') + elif r.id not in owner[0].ids: + problems.append(f'{rel}:{r.start}: snippet {r.id!r} is not listed in {owner[0].func}') + for n, line in r.visible: + if FORBIDDEN.search(line): + problems.append(f'{rel}:{n}: assertion, helper or recorder call inside snippet {r.id!r}') + for e in file_entries: + inside = sorted(r.id for r in regions if e.first <= r.start and r.end <= e.last) + if inside != sorted(e.ids): + problems.append(f'{rel}: {e.func} lists {sorted(e.ids)} but its body has {inside}') + ids = Counter((e.docset, i) for e in entries for i in e.ids) + return problems, len(entries), sum(ids.values()) + + +def main(): + problems, steps, snippets = lint() + for p in problems: + print('lint:', p) + print(f'marker lint: {steps} entries, {snippets} snippet ids, {len(problems)} problems') + return 1 if problems else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docs-snippets/harness/listeners.py b/docs-snippets/harness/listeners.py new file mode 100644 index 00000000..c42823b3 --- /dev/null +++ b/docs-snippets/harness/listeners.py @@ -0,0 +1,76 @@ +"""Event collector used by steps outside the snippet markers. Works for PubNub and PubNubAsyncio.""" +import threading + +from pubnub.callbacks import SubscribeCallback +from pubnub.enums import PNStatusCategory + +from . import assertions +from .wait import aeventually, eventually + + +class Collector(SubscribeCallback): + def __init__(self): + self._lock = threading.Lock() + self.categories = [] + self.messages = [] + self.signals = [] + self.presence_events = [] + self.message_actions = [] + self.files = [] + + def status(self, pubnub, status): + with self._lock: + self.categories.append(status.category) + + def message(self, pubnub, message): + with self._lock: + self.messages.append(message) + + def signal(self, pubnub, signal): + with self._lock: + self.signals.append(signal) + + def presence(self, pubnub, presence): + with self._lock: + self.presence_events.append(presence) + + def message_action(self, pubnub, message_action): + with self._lock: + self.message_actions.append(message_action) + + def file(self, pubnub, file_message): + with self._lock: + self.files.append(file_message) + + def _connected(self): + assertions.ok(PNStatusCategory.PNConnectedCategory in self.categories, 'no PNConnectedCategory yet') + + +def listen(client, *channels, groups=None, presence=False): + """Sync client: add a collector, subscribe, wait for PNConnectedCategory.""" + collector = Collector() + client.subscribed_by_suite = True + client.add_listener(collector) + builder = client.subscribe().channels(list(channels)) + if groups: + builder = builder.channel_groups(list(groups)) + if presence: + builder = builder.with_presence() + builder.execute() + eventually(collector._connected, timeout=15, what='PNConnectedCategory') + return collector + + +async def alisten(client, *channels, groups=None, presence=False): + """Asyncio client: same as listen().""" + collector = Collector() + client.subscribed_by_suite = True + client.add_listener(collector) + builder = client.subscribe().channels(list(channels)) + if groups: + builder = builder.channel_groups(list(groups)) + if presence: + builder = builder.with_presence() + builder.execute() + await aeventually(collector._connected, timeout=15, what='PNConnectedCategory') + return collector diff --git a/docs-snippets/harness/manifest.py b/docs-snippets/harness/manifest.py new file mode 100644 index 00000000..4b7e712a --- /dev/null +++ b/docs-snippets/harness/manifest.py @@ -0,0 +1,66 @@ +"""Coverage manifest: one entry per snippet id with file, step, role, mode and reason. + +--check fails on a stale manifest. It also reports, without failing, the docs ids that the manifest does not know +(read from EmbeddedCode tags in a docs checkout, DOCS_REPO or ../documentation). +""" +import json +import re +import sys +from pathlib import Path + +from . import registry + +PATH = registry.ROOT / 'coverage.json' + + +def build(): + data = {} + for e in registry.scan(): + for i in e.ids: + data.setdefault(e.docset, {})[i] = { + 'file': e.file, + 'step': e.func, + 'role': e.role, + 'mode': e.mode, + 'reason': e.reason, + } + return {d: dict(sorted(v.items())) for d, v in sorted(data.items())} + + +def docs_ids(): + import os + root = Path(os.environ.get('DOCS_REPO') or registry.ROOT.parent.parent / 'documentation') + found = {} + for docset, sub in (('sync', 'python'), ('asyncio', 'asyncio')): + base = root / 'docs' / 'sdks' / sub + if not base.is_dir(): + continue + for md in base.rglob('*.md*'): + for tag in re.findall(r']*docs-snippets/[^>]*>', md.read_text()): + lists = ''.join(re.findall(r'snippets=\{\[[^\]]*\]\}', tag)) + found.setdefault(docset, set()).update(re.findall(r"'([a-zA-Z0-9_-]+)'", lists)) + return found + + +def main(argv): + text = json.dumps(build(), indent=2, sort_keys=True) + '\n' + if '--check' in argv: + current = PATH.read_text() if PATH.exists() else '' + if current != text: + print('manifest: coverage.json is stale, run "python -m harness.cli manifest"') + return 1 + total = sum(len(v) for v in build().values()) + print(f'manifest: ok, {total} snippet ids') + known = {d: set(v) for d, v in build().items()} + for docset, ids in docs_ids().items(): + gone = sorted(ids - known.get(docset, set())) + if gone: + print(f'manifest: docs ids without a snippet in {docset} (not a failure): ' + ', '.join(gone)) + return 0 + PATH.write_text(text) + print(f'manifest: wrote {PATH.name}') + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/docs-snippets/harness/markers.py b/docs-snippets/harness/markers.py new file mode 100644 index 00000000..09f0306c --- /dev/null +++ b/docs-snippets/harness/markers.py @@ -0,0 +1,72 @@ +"""Snippet marker parser. Follows the docs extractor (src/helper/codeSnippetFetcher.ts in the docs repo).""" +import re +from dataclasses import dataclass, field + +MARKER = re.compile(r'^(\s*)#\s*snippet\.([a-zA-Z0-9_-]+)\s*$') + + +@dataclass +class Region: + id: str + start: int # 1-based line of the start marker + end: int # 1-based line of the end marker, 0 if missing + visible: list = field(default_factory=list) # (line_no, text) of lines the docs show + hidden: list = field(default_factory=list) # (line_no, text) between hide and show + + +@dataclass +class Marker: + line: int + kind: str # 'start' | 'end' | 'hide' | 'show' + id: str + indent: str + + +def markers(text): + out = [] + for i, line in enumerate(text.splitlines(), 1): + m = MARKER.match(line) + if not m: + continue + name = m.group(2) + kind = name if name in ('end', 'hide', 'show') else 'start' + out.append(Marker(i, kind, name if kind == 'start' else '', m.group(1))) + return out + + +def regions(text): + lines = text.splitlines() + result, cur, hiding = [], None, False + for mk in markers(text): + if mk.kind == 'start': + if cur is not None: # the docs extractor ends a region at the next start. Lint reports it + cur.end = 0 + result.append(cur) + cur, hiding = Region(mk.id, mk.line, 0), False + elif mk.kind == 'end' and cur is not None: + cur.end = mk.line + result.append(cur) + cur = None + elif mk.kind == 'hide': + hiding = True + elif mk.kind == 'show': + hiding = False + if cur is not None and mk.kind in ('hide', 'show'): + pass + if cur is not None: + result.append(cur) + # fill body lines + for r in result: + stop = r.end or len(lines) + 1 + hiding = False + for n in range(r.start + 1, stop): + text_line = lines[n - 1] + m = MARKER.match(text_line) + if m and m.group(2) == 'hide': + hiding = True + continue + if m and m.group(2) == 'show': + hiding = False + continue + (r.hidden if hiding else r.visible).append((n, text_line)) + return result diff --git a/docs-snippets/harness/names.py b/docs-snippets/harness/names.py new file mode 100644 index 00000000..7e2dec0b --- /dev/null +++ b/docs-snippets/harness/names.py @@ -0,0 +1,68 @@ +"""Run suffix and the registered names that staging rewrites. + +Every channel, group, user and file name carries a base36 run suffix and the fixed marker "dsn", so the janitor can +recognise leftovers and never touches a name another run could use. +""" +import os +import random +import zlib +import string +import time + +MARKER = 'dsn' +_BASE36 = string.digits + string.ascii_lowercase + + +def _b36(n): + out = '' + while n: + n, r = divmod(n, 36) + out = _BASE36[r] + out + return out or '0' + + +def run_suffix(): + """Same value for the whole process. CI sets DOCS_SNIPPETS_RUN_ID to include run id, attempt and shard.""" + existing = os.environ.get('DOCS_SNIPPETS_SUFFIX') + if existing: + return existing + base = os.environ.get('DOCS_SNIPPETS_RUN_ID') or f'{_b36(int(time.time()))}{"".join(random.choices(_BASE36, k=3))}' + value = f'{MARKER}-{base}'.lower() + os.environ['DOCS_SNIPPETS_SUFFIX'] = value + return value + + +_tag = '' + + +def test_tag(relpath, func): + """Short tag of one step. Staging and the runtime derive it the same way, so every step gets its own names and a + client that lingers in presence after a step can never show up in the next one.""" + return format(zlib.crc32(f'{relpath}:{func}'.encode()), '08x')[:6] + + +def set_tag(tag): + global _tag + _tag = tag + + +def with_suffix(name): + return f'{name}-{run_suffix()}' + (f'-{_tag}' if _tag else '') + + +# Literal names the docs code uses. Staging appends the run suffix to these string literals in the staged copy only. +REGISTERED = [ + 'my_channel', + 'my_channel_2', + 'my-channel', + 'ch1', + 'ch2', + 'ch3', + 'cg1', + 'my_group', + 'my_user', + 'my_user_id', + 'my_uuid', + 'my_custom_user', + 'my_authorized_uuid', +] diff --git a/docs-snippets/harness/preflight.py b/docs-snippets/harness/preflight.py new file mode 100644 index 00000000..3d902286 --- /dev/null +++ b/docs-snippets/harness/preflight.py @@ -0,0 +1,99 @@ +"""Preflight: reads the keyset configuration through the Admin API (read only) and names every missing feature. + +Only the feature names and flags are printed, never a key value. The Admin API key is used here and nowhere else. +""" +import os +import sys + +import requests + +from . import env + +BASE = 'https://admin-api.pubnub.com' +VERSION = '2026-11-06' + +# role -> (label, [(feature name, path in the config, predicate)]) +REQUIRED = { + 'open': [ + ('Message Persistence', ('messagePersistence', 'enabled'), lambda v: v is True), + ('Message Persistence: Delete-From-History', ('messagePersistence', 'deleteFromHistory'), lambda v: v is True), + ('Message Persistence: retention of at least 1 day', ('messagePersistence', 'retention'), lambda v: bool(v)), + ('Presence', ('presence', 'enabled'), lambda v: v is True), + ('Presence: tracking rule for *', ('presence', 'rules'), lambda v: _tracks_everything(v)), + ('App Context', ('appContext', 'enabled'), lambda v: v is True), + ('App Context: user metadata events', ('appContext', 'userMetadataEvents'), lambda v: v is True), + ('App Context: channel metadata events', ('appContext', 'channelMetadataEvents'), lambda v: v is True), + ('App Context: membership events', ('appContext', 'membershipEvents'), lambda v: v is True), + ('App Context: referential integrity off', ('appContext', 'referentialIntegrity'), lambda v: not v), + ('Files', ('files', 'enabled'), lambda v: v is True), + ('Stream Controller', ('streamController', 'enabled'), lambda v: v is True), + ('Stream Controller: wildcard subscribe', ('streamController', 'wildcardSubscribe'), lambda v: v is True), + ('No Access Manager on the open keyset', ('accessManager', 'enabled'), lambda v: not v), + ], + 'pam': [ + ('Access Manager', ('accessManager', 'enabled'), lambda v: v is True), + ('Access Manager: token revocation', ('accessManager', 'tokenRevoke'), lambda v: v is True), + ('Message Persistence', ('messagePersistence', 'enabled'), lambda v: v is True), + ('Presence', ('presence', 'enabled'), lambda v: v is True), + ], +} + + +def _tracks_everything(rules): + return any(r.get('pattern') == '*' and (r.get('tracking') or {}).get('enabled') for r in (rules or [])) + + +SUBSCRIBE_ENV = {'open': 'SUBSCRIBE_KEY', 'pam': 'PAM_SUBSCRIBE_KEY'} + + +def _get(path, **params): + headers = {'Authorization': os.environ['PUBNUB_ADMIN_API_KEY'], 'PubNub-Version': VERSION} + response = requests.get(BASE + path, headers=headers, params=params, timeout=30) + response.raise_for_status() + return response.json() + + +def keyset_for(role): + sub = os.environ[SUBSCRIBE_ENV[role]] + page = 1 + while True: + data = _get('/v2/keysets', limit=100, page=page) + for item in data['keysets']: + if item.get('subscribeKey') == sub: + return item['id'], item.get('name', '') + if page * data['limit'] >= data['total']: + return None, '' + page += 1 + + +def check(role): + """Return (keyset name, [missing feature names]).""" + if env.missing('admin'): + raise RuntimeError('preflight needs PUBNUB_ADMIN_API_KEY') + keyset_id, name = keyset_for(role) + if keyset_id is None: + return '', ['keyset not found for the loaded subscribe key'] + config = _get(f'/v2/keysets/{keyset_id}/config') + missing = [] + for label, (group, key), ok in REQUIRED[role]: + value = (config.get(group) or {}).get(key) + if not ok(value): + missing.append(f'{label} ({group}.{key} = {value!r})') + return name, missing + + +def main(argv=None): + env.load() + roles = [r for r in ('open', 'pam') if not env.missing(r)] + bad = 0 + for role in roles: + name, missing = check(role) + print(f'preflight {role} ({name}): ' + ('ok' if not missing else f'{len(missing)} missing')) + for m in missing: + print(' missing:', m) + bad += len(missing) + return 1 if bad else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/docs-snippets/harness/recorder.py b/docs-snippets/harness/recorder.py new file mode 100644 index 00000000..ae0f155e --- /dev/null +++ b/docs-snippets/harness/recorder.py @@ -0,0 +1,51 @@ +"""Per step record: which snippet regions started and ended, how many assertions ran. Installed as builtin __snip__.""" +import builtins +import threading + +_local = threading.local() +_current = None + + +class Record: + def __init__(self, name, snippets): + self.name = name + self.snippets = list(snippets) + self.started = [] + self.ended = [] + self.assertions = 0 + self.after_snippet_assertions = 0 + + +def begin(name, snippets): + global _current + _current = Record(name, snippets) + return _current + + +def current(): + if _current is None: + raise RuntimeError('assertion used outside a step') + return _current + + +def finish(): + global _current + rec, _current = _current, None + return rec + + +class _Snip: + """Staged copies call __snip__.start(id) and __snip__.end(id) on the marker lines.""" + + def start(self, snippet_id): + import os + if os.environ.get('DOCS_SNIPPETS_MUTATE') == snippet_id: + raise RuntimeError(f'mutation: {snippet_id} was made to fail on purpose') + current().started.append(snippet_id) + + def end(self, snippet_id): + current().ended.append(snippet_id) + + +def install(): + builtins.__snip__ = _Snip() diff --git a/docs-snippets/harness/redact.py b/docs-snippets/harness/redact.py new file mode 100644 index 00000000..38b07398 --- /dev/null +++ b/docs-snippets/harness/redact.py @@ -0,0 +1,42 @@ +"""Mask key values, key shapes and Access Manager tokens in any text the suite prints or writes.""" +import re +import urllib.parse + +_secrets = set() + +_SHAPES = [ + re.compile(r'\b(?:pub|sub|sec)-c-[0-9a-fA-F-]{20,}'), + # Access Manager (v3) tokens are CBOR in base64url, so a real one starts with "qEF2". Granted tokens are also + # registered, which is the main protection. + re.compile(r'\bqEF2[A-Za-z0-9_\-%=]{40,}'), +] +MASK = '***' + + +def register(value): + """Remember a value (key, granted token) so that scrub() masks it and its URL-encoded forms.""" + if not value or len(value) < 6: + return + _secrets.add(value) + _secrets.add(urllib.parse.quote(value, safe='')) + _secrets.add(urllib.parse.quote(value)) + + +def registered(): + return set(_secrets) + + +def scrub(text): + if not text: + return text + out = str(text) + for secret in sorted(_secrets, key=len, reverse=True): + out = out.replace(secret, MASK) + for shape in _SHAPES: + out = shape.sub(MASK, out) + return out + + +def leaks(text): + """Return the registered secrets that still appear in text (used by the reporter self-test).""" + return [s for s in _secrets if s in str(text)] diff --git a/docs-snippets/harness/registry.py b/docs-snippets/harness/registry.py new file mode 100644 index 00000000..a3518bca --- /dev/null +++ b/docs-snippets/harness/registry.py @@ -0,0 +1,54 @@ +"""Static registry: reads the suite modules with ast, no import, so the dry stage needs no network or keys.""" +import ast +from dataclasses import dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DOCSETS = ('sync', 'asyncio') +DECORATORS = {'step': 'live', 'compile_only': 'compile-only', 'stub': 'stub'} + + +@dataclass +class Entry: + docset: str + file: str # relative to docs-snippets + func: str + first: int + last: int + ids: list + role: str + mode: str + reason: str + is_async: bool + + +def _literal(node): + return ast.literal_eval(node) + + +def scan(): + entries = [] + for docset in DOCSETS: + for path in sorted((ROOT / docset).glob('*.py')): + if path.name.startswith('_'): + continue + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for deco in node.decorator_list: + if isinstance(deco, ast.Call) and isinstance(deco.func, ast.Name) and deco.func.id in DECORATORS: + kwargs = {k.arg: _literal(k.value) for k in deco.keywords} + entries.append(Entry( + docset=docset, + file=str(path.relative_to(ROOT)), + func=node.name, + first=node.lineno, + last=node.end_lineno, + ids=[_literal(a) for a in deco.args], + role=kwargs.get('role', 'open'), + mode=DECORATORS[deco.func.id], + reason=kwargs.get('reason', ''), + is_async=isinstance(node, ast.AsyncFunctionDef), + )) + return entries diff --git a/docs-snippets/harness/reporter.py b/docs-snippets/harness/reporter.py new file mode 100644 index 00000000..865388c8 --- /dev/null +++ b/docs-snippets/harness/reporter.py @@ -0,0 +1,80 @@ +"""pytest plugin: writes report/run.json and report/summary.md, with every value scrubbed.""" +import json +import os +import platform +import time +from pathlib import Path + +from . import redact, registry, step + +_records = [] +_sections = {} # nodeid -> list of (phase, title, text), everything pytest captured for that step +_t0 = time.monotonic() + + +def pytest_runtest_logreport(report): + for title, text in getattr(report, 'sections', []): + _sections.setdefault(report.nodeid, []).append((report.when, title, text)) + if report.when == 'call' or (report.when == 'setup' and report.outcome != 'passed'): + _records.append({ + 'test': report.nodeid, + 'outcome': report.outcome, + 'duration': round(report.duration, 2), + 'error': redact.scrub(str(report.longreprtext))[-2000:] if report.outcome == 'failed' else '', + 'full_error': redact.scrub(str(report.longreprtext)) if report.outcome == 'failed' else '', + }) + + +def _write_step_logs(out): + """One log file per step in report//steps/: outcome, snippet ids, assertions, captured output, error.""" + folder = out / 'steps' + folder.mkdir(exist_ok=True) + for r in _records: + name = r['test'].split('::')[-1] + module = r['test'].split('::')[0].replace('/', '__').replace('.py', '') + info = step.results().get(r['test'], {}) + lines = [f"test: {r['test']}", f"outcome: {r['outcome']}", f"seconds: {r['duration']}", + f"snippet ids: {', '.join(info.get('ids', []))}", f"assertions: {info.get('assertions', 0)}", ''] + for phase, title, text in _sections.get(r['test'], []): + lines += [f'--- {title} ({phase}) ---', text.rstrip(), ''] + if r['full_error']: + lines += ['--- failure ---', r['full_error'].rstrip(), ''] + (folder / f'{module}__{name}.log').write_text(redact.scrub('\n'.join(lines)) + '\n') + + +def pytest_sessionfinish(session, exitstatus): + out = Path(os.environ.get('DOCS_SNIPPETS_REPORT', registry.ROOT / 'report')) + out.mkdir(parents=True, exist_ok=True) + try: + import pubnub + from pubnub.pubnub_core import PubNubCore + sdk = {'version': PubNubCore.SDK_VERSION, 'file': pubnub.__file__} + except Exception: # the report must never hide the real failure + sdk = {} + counts = {} + for r in _records: + counts[r['outcome']] = counts.get(r['outcome'], 0) + 1 + run = { + 'sdk': sdk, + 'python': platform.python_version(), + 'seconds': round(time.monotonic() - _t0, 1), + 'counts': counts, + 'steps': step.results(), + 'tests': [{k: v for k, v in r.items() if k != 'full_error'} for r in _records], + } + text = redact.scrub(json.dumps(run, indent=2)) + (out / 'run.json').write_text(text) + head = f"SDK {sdk.get('version', '?')} on Python {run['python']}, {run['seconds']} s" + lines = ['# Docs snippet run', '', head, '', '| outcome | count |', '| :-- | --: |'] + lines += [f'| {k} | {v} |' for k, v in sorted(counts.items())] + lines += ['', '| test | outcome | seconds | assertions | snippet ids |', '| :-- | :-- | --: | --: | :-- |'] + for r in sorted(_records, key=lambda x: -x['duration']): + ids = ', '.join(step.results().get(r['test'], {}).get('ids', [])) + n = step.results().get(r['test'], {}).get('assertions', 0) + lines.append(f"| {r['test']} | {r['outcome']} | {r['duration']} | {n} | {ids} |") + _write_step_logs(out) + (out / 'summary.md').write_text(redact.scrub('\n'.join(lines)) + '\n') + leaked = redact.leaks(text) + if leaked: + session.exitstatus = 1 + print('reporter self-test failed: a loaded secret appears in the report') diff --git a/docs-snippets/harness/stage.py b/docs-snippets/harness/stage.py new file mode 100644 index 00000000..3a4d9954 --- /dev/null +++ b/docs-snippets/harness/stage.py @@ -0,0 +1,65 @@ +"""Stage the suite: copy it to .run//, add the run suffix to registered names and instrument the markers. + +Staging rewrites only registered names and replaces each marker line in place, so line numbers never move and a +failure points at the same line of the source file. +""" +import ast +import re +import shutil + +from . import markers as mk +from . import names, registry + +IGNORE = shutil.ignore_patterns('.run', 'report', '__pycache__', '.env*', '*.pyc', '.pytest_cache') + + +def module_names(text): + """A module may declare more literal names the staging rewrites: REGISTERED_NAMES = ['a', 'b'] (a plain list).""" + m = re.search(r'^REGISTERED_NAMES\s*=\s*(\[[^\]]*\])', text, re.M) + return ast.literal_eval(m.group(1)) if m else [] + + +def _rewrite_names(line, suffix, extra=()): + for name in list(names.REGISTERED) + list(extra): + line = re.sub(r'(["\'])' + re.escape(name) + r'\1', lambda m: f'{m.group(1)}{name}-{suffix}{m.group(1)}', line) + return line + + +def instrument(text, suffix, relpath=''): + extra = module_names(text) + out = [] + current = None + func = '' + for line in text.splitlines(): + fm = re.match(r'(?:async )?def (test_\w+)', line) + if fm: + func = fm.group(1) + m = mk.MARKER.match(line) + if m: + name, indent = m.group(2), m.group(1) + if name == 'end': + out.append(f"{indent}__snip__.end({current!r}) # snippet.end") + current = None + elif name in ('hide', 'show'): + out.append(line) + else: + current = name + out.append(f"{indent}__snip__.start({name!r}) # snippet.{name}") + continue + # Only the lines the docs show carry the docs names. Test code outside the markers adds the suffix itself. + tagged = f'{suffix}-{names.test_tag(relpath, func)}' if relpath else suffix + out.append(_rewrite_names(line, tagged, extra) if current else line) + return '\n'.join(out) + '\n' + + +def stage(run_id=None): + suffix = names.run_suffix() + run_id = run_id or suffix + target = registry.ROOT / '.run' / run_id + if target.exists(): + shutil.rmtree(target) + shutil.copytree(registry.ROOT, target, ignore=IGNORE) + for docset in registry.DOCSETS: + for path in (target / docset).glob('*.py'): + path.write_text(instrument(path.read_text(), suffix, str(path.relative_to(target)).replace('\\', '/'))) + return target diff --git a/docs-snippets/harness/step.py b/docs-snippets/harness/step.py new file mode 100644 index 00000000..04d80196 --- /dev/null +++ b/docs-snippets/harness/step.py @@ -0,0 +1,98 @@ +"""Decorators that register a docs snippet step. The registry itself is read statically (registry.py). + +step a live test. The docs code sits between snippet markers in its body, assertions sit outside. +compile_only code that cannot run against the service. It is compiled and linted, not executed. +stub a snippet kept for the docs that no repository test can run (for example a push delivery). Needs a reason. +""" +import asyncio +import functools +import _thread +import os +import threading + +import pytest + +from . import recorder + +_RESULTS = {} + + +def results(): + return _RESULTS + + +def _check(rec): + problems = [] + if os.environ.get('DOCS_SNIPPETS_STAGED') != '1': + problems.append('the suite is not staged, run it through "python -m harness.cli run"') + missing_start = [i for i in rec.snippets if i not in rec.started] + missing_end = [i for i in rec.snippets if i not in rec.ended] + if missing_start: + problems.append('snippet never started: ' + ', '.join(missing_start)) + if missing_end: + problems.append('snippet did not reach its snippet.end line: ' + ', '.join(missing_end)) + if rec.after_snippet_assertions == 0: + problems.append('the step made no assertion after its snippet started') + _RESULTS[rec.name] = {'ids': rec.snippets, 'assertions': rec.assertions} + if problems: + raise AssertionError('honesty check failed: ' + '; '.join(problems)) + + +DEFAULT_TIMEOUT = 120 + + +def _key(fn): + """Same text as the pytest node id, for example sync/publish.py::test_signal.""" + path = os.path.abspath(fn.__code__.co_filename) + return f'{os.path.basename(os.path.dirname(path))}/{os.path.basename(path)}::{fn.__name__}' + + +def step(*snippets, role='open', reason='', timeout=DEFAULT_TIMEOUT): + if not snippets: + raise ValueError('step needs at least one snippet id') + + def deco(fn): + if asyncio.iscoroutinefunction(fn): + @functools.wraps(fn) + async def wrapper(*a, **k): + rec = recorder.begin(_key(fn), snippets) + try: + try: + await asyncio.wait_for(fn(*a, **k), timeout) + except asyncio.TimeoutError: + raise AssertionError(f'step timed out after {timeout}s') + finally: + rec = recorder.finish() + _check(rec) + else: + @functools.wraps(fn) + def wrapper(*a, **k): + rec = recorder.begin(_key(fn), snippets) + timer = threading.Timer(timeout, _thread.interrupt_main) + timer.daemon = True + timer.start() + try: + try: + fn(*a, **k) + except KeyboardInterrupt: + raise AssertionError(f'step timed out after {timeout}s') + finally: + timer.cancel() + rec = recorder.finish() + _check(rec) + wrapper.snippet_ids = list(snippets) + return getattr(pytest.mark, role)(wrapper) + + return deco + + +def compile_only(*snippets, role='open', reason=''): + if not reason: + raise ValueError('compile_only needs a reason') + return lambda fn: fn + + +def stub(*snippets, role='open', reason=''): + if not reason: + raise ValueError('stub needs a reason') + return lambda fn: fn diff --git a/docs-snippets/harness/wait.py b/docs-snippets/harness/wait.py new file mode 100644 index 00000000..54b26567 --- /dev/null +++ b/docs-snippets/harness/wait.py @@ -0,0 +1,33 @@ +"""eventually(): the only way to wait. No fixed sleeps. Retries on AssertionError until the timeout.""" +import asyncio +import time + + +def eventually(fn, timeout=15, interval=0.5, what='condition'): + deadline = time.monotonic() + timeout + last = None + while True: + try: + return fn() + except AssertionError as e: + last = e + if time.monotonic() >= deadline: + raise AssertionError(f'{what} not met within {timeout}s: {last}') + time.sleep(interval) + + +async def aeventually(fn, timeout=15, interval=0.5, what='condition'): + """fn may be a plain function or a coroutine function.""" + deadline = time.monotonic() + timeout + last = None + while True: + try: + result = fn() + if asyncio.iscoroutine(result): + result = await result + return result + except AssertionError as e: + last = e + if time.monotonic() >= deadline: + raise AssertionError(f'{what} not met within {timeout}s: {last}') + await asyncio.sleep(interval) diff --git a/docs-snippets/harness/world.py b/docs-snippets/harness/world.py new file mode 100644 index 00000000..462da745 --- /dev/null +++ b/docs-snippets/harness/world.py @@ -0,0 +1,86 @@ +"""Client factories and the ledger of created resources. Closes every client it made.""" +import json +import os +import threading +from pathlib import Path +import time + +from pubnub.pnconfiguration import PNConfiguration +from pubnub.pubnub import PubNub +from pubnub.pubnub_asyncio import PubNubAsyncio + +from . import env, names, registry + + +def ledger_path(): + base = os.environ.get('DOCS_SNIPPETS_LEDGER_DIR') or str(registry.ROOT / '.run') + return Path(base) / f'ledger-{names.run_suffix()}.jsonl' + + +def record(kind, name): + """Remember a resource this run created, so the janitor can remove it.""" + path = ledger_path() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('a') as f: + f.write(json.dumps({'kind': kind, 'name': name, 'ts': int(time.time())}) + '\n') + + +def config(role='open', user_id=None, secret=True): + keys = env.require(role) + cfg = PNConfiguration() + prefix = 'PAM_' if role == 'pam' else '' + cfg.publish_key = keys[prefix + 'PUBLISH_KEY'] + cfg.subscribe_key = keys[prefix + 'SUBSCRIBE_KEY'] + if role == 'pam' and secret: + cfg.secret_key = keys['PAM_SECRET_KEY'] + cfg.user_id = names.with_suffix(user_id or 'user') + # Request threads default to non-daemon. A long poll cannot be interrupted by stop(), so it would hold the process + # open for up to 310 seconds. The suite sets daemon=True so that the lingering thread check sees only real leaks. + cfg.daemon = True + return cfg + + +class Clients: + """Hands out clients and stops them all at the end of the step.""" + + def __init__(self): + self._made = [] + + def sync(self, role='open', user_id=None, secret=True): + client = PubNub(config(role, user_id, secret)) + self._made.append(client) + return client + + def asyncio(self, role='open', user_id=None, secret=True): + client = PubNubAsyncio(config(role, user_id, secret)) + self._made.append(client) + return client + + def close_sync(self): + for client in self._made: + if isinstance(client, PubNub): + try: + if getattr(client, 'subscribed_by_suite', False): + client.unsubscribe_all() + finally: + client.stop() + + async def close_asyncio(self): + for client in self._made: + if isinstance(client, PubNubAsyncio): + try: + # unsubscribe_all() raises AttributeError on a client that never subscribed (SDK finding) + if getattr(client, 'subscribed_by_suite', False): + client.unsubscribe_all() + finally: + await client.stop() + + +def lingering_threads(wait=10.0): + """Names of non-daemon threads that are still alive after waiting. The Python analog of an open handle check.""" + deadline = time.monotonic() + wait + while True: + alive = [t for t in threading.enumerate() if t is not threading.main_thread() and not t.daemon and t.is_alive()] + if not alive or time.monotonic() >= deadline: + return [t.name for t in alive] + time.sleep(0.25) diff --git a/docs-snippets/pytest.ini b/docs-snippets/pytest.ini new file mode 100644 index 00000000..be561661 --- /dev/null +++ b/docs-snippets/pytest.ini @@ -0,0 +1,16 @@ +[pytest] +# Own config so that "py.test tests" at the repository root never collects this folder. +python_files = *.py +python_functions = test_* +testpaths = sync asyncio +addopts = --import-mode=importlib -p no:cacheprovider -p no:randomly +asyncio_mode = auto +log_level = INFO +asyncio_default_fixture_loop_scope = function +markers = + open: steps that need the open keyset role + pam: steps that need the Access Manager keyset role +filterwarnings = + ignore:Mutable config will be deprecated in the future.:DeprecationWarning + ignore:Access management v2 is being deprecated.:DeprecationWarning + ignore:.*Usage of local cipher_keys is discouraged.*:UserWarning diff --git a/docs-snippets/selftest/test_harness.py b/docs-snippets/selftest/test_harness.py new file mode 100644 index 00000000..87913589 --- /dev/null +++ b/docs-snippets/selftest/test_harness.py @@ -0,0 +1,103 @@ +"""Dry checks of the harness itself. No network, no keys. Run: python -m harness.cli selftest""" +import os +import textwrap + +import pytest + +from harness import assertions as a +from harness import markers, names, recorder, redact, stage, step + + +def test_redaction_masks_values_encoded_forms_and_shapes(): + redact.register('secret/value+1234') + text = 'url=secret%2Fvalue%2B1234 raw=secret/value+1234 key=pub-c-12345678-1234-1234-1234-123456789012' + out = redact.scrub(text) + assert 'secret' not in out and 'pub-c-' not in out + assert not redact.leaks(out) + + +def test_markers_follow_the_docs_extractor_rules(): + src = textwrap.dedent('''\ + def f(): + # snippet.one + x = 1 + # snippet.hide + assert x + # snippet.show + y = 2 + # snippet.end + ''') + [r] = markers.regions(src) + assert r.id == 'one' and r.end == 8 + assert [t.strip() for _, t in r.visible] == ['x = 1', 'y = 2'] + assert [t.strip() for _, t in r.hidden] == ['assert x'] + + +def test_staging_keeps_line_numbers_and_adds_the_suffix(): + src = 'def f():\n # snippet.one\n c = "my_channel"\n # snippet.end\n' + out = stage.instrument(src, 'dsn-abc').splitlines() + assert len(out) == 4 + assert out[1].strip() == "__snip__.start('one') # snippet.one" + assert 'my_channel-dsn-abc' in out[2] + outside = stage.instrument('x = "my_channel"\n', 'dsn-abc') + assert outside.strip() == 'x = "my_channel"', 'names outside the markers are not rewritten' + assert out[3].strip() == "__snip__.end('one') # snippet.end" + + +def _run(fn_body, ids=('one',)): + recorder.install() + os.environ['DOCS_SNIPPETS_STAGED'] = '1' + + @step.step(*ids) + def sample(): + fn_body() + return sample + + +def test_honesty_fails_when_snippet_does_not_reach_its_end(): + def body(): + __snip__.start('one') # noqa: F821 + a.ok(True) + with pytest.raises(AssertionError, match='did not reach'): + _run(body)() + + +def test_honesty_fails_with_zero_assertions_after_the_snippet_started(): + def body(): + a.ok(True) # before the snippet, does not count + __snip__.start('one') # noqa: F821 + __snip__.end('one') # noqa: F821 + with pytest.raises(AssertionError, match='no assertion'): + _run(body)() + + +def test_honesty_passes_for_a_proper_step(): + def body(): + __snip__.start('one') # noqa: F821 + __snip__.end('one') # noqa: F821 + a.eq(1, 1) + _run(body)() + + +def test_mutation_makes_the_step_fail(): + def body(): + __snip__.start('one') # noqa: F821 + __snip__.end('one') # noqa: F821 + a.eq(1, 1) + os.environ['DOCS_SNIPPETS_MUTATE'] = 'one' + try: + with pytest.raises(RuntimeError, match='mutation'): + _run(body)() + finally: + del os.environ['DOCS_SNIPPETS_MUTATE'] + + +def test_a_weakened_assertion_is_still_counted_and_wrong_values_fail(): + recorder.begin('x', []) + with pytest.raises(AssertionError): + a.eq(1, 2) + recorder.finish() + + +def test_names_carry_marker_and_suffix(): + assert names.with_suffix('my_channel').startswith('my_channel-dsn-') diff --git a/docs-snippets/sync/access_manager.py b/docs-snippets/sync/access_manager.py new file mode 100644 index 00000000..b54bc48b --- /dev/null +++ b/docs-snippets/sync/access_manager.py @@ -0,0 +1,384 @@ +"""Docs page: access-manager (sync), Access Manager v3 tokens. + +Needs the pam keyset. The client that holds a token has no secret key, because a client with a secret key signs its +requests and is never checked against a token. Tokens are registered with the redactor so they never reach output. +""" +from pubnub.exceptions import PubNubException +from pubnub.models.consumer.v3.channel import Channel + +from harness import assertions as a +from harness import names, redact +from harness.step import step +from harness.wait import eventually + +REGISTERED_NAMES = ['readonly-channel', 'readwrite-channel', 'my-authorized-uuid', 'revoke-channel'] + +FLAGS = ('read', 'write', 'manage', 'delete', 'get', 'update', 'join') + + +def perms(*on): + """The seven flags parse_token returns for one resource, True only for the ones named.""" + return {flag: flag in on for flag in FLAGS} + + +def holder(clients, user_id, token=None): + """A client that holds a token. It has no secret key, so every request is checked against the token.""" + client = clients.sync('pam', user_id, secret=False) + if token: + client.set_token(token) + return client + + +def publish(client, channel): + return client.publish().channel(channel).message('docs').sync() + + +def refused(client, channel): + """A publish the service answers with 403.""" + error = a.raises(PubNubException, publish, client, channel) + a.eq(error.get_status_code(), 403, 'status code of the refusal') + + +def allowed(client, channel): + a.timetoken(publish(client, channel).result.timetoken) + + +@step('grant_token_builder', 'grant_token_named', 'parse_token', role='pam') +def test_grant_and_parse_token(clients): + pubnub = clients.sync('pam', 'admin') + + # snippet.grant_token_builder + from pubnub.models.consumer.v3.channel import Channel + + envelope = pubnub.grant_token() \ + .channels([ + Channel.id('readonly-channel').read(), + Channel.id('readwrite-channel').read().write(), + ]) \ + .authorized_uuid("my-authorized-uuid") \ + .ttl(15) \ + .sync() + # snippet.end + token = envelope.result.token + redact.register(token) + a.ok(isinstance(token, str) and len(token) > 40, 'the result carries a token') + a.eq(envelope.result.get_token(), token) + + # snippet.grant_token_named + envelope = pubnub.grant_token( + channels=[ + Channel.id('readonly-channel').read(), + Channel.id('readwrite-channel').read().write(), + ], + authorized_user_id="my-authorized-uuid", + ttl=15 + ).sync() + # snippet.end + named_token = envelope.result.token + redact.register(named_token) + + # snippet.parse_token + parsed_token = pubnub.parse_token(token) + # snippet.end + + for parsed in (parsed_token, pubnub.parse_token(named_token)): + a.eq(parsed['version'], 2) + a.eq(parsed['ttl'], 15) + a.eq(parsed['authorized_uuid'], names.with_suffix('my-authorized-uuid')) + a.eq(parsed['resources']['channels'], { + names.with_suffix('readonly-channel'): perms('read'), + names.with_suffix('readwrite-channel'): perms('read', 'write'), + }) + a.eq(parsed['resources'].get('groups', {}), {}) + a.eq(parsed['resources'].get('uuids', {}), {}) + a.eq(parsed['patterns'].get('channels', {}), {}) + a.ok('categories' not in parsed, 'no category in a token granted without categories') + + +@step('set_token', role='pam') +def test_set_token(clients): + admin = clients.sync('pam', 'admin') + readonly = names.with_suffix('readonly-channel') + readwrite = names.with_suffix('readwrite-channel') + fresh = names.with_suffix('set_token_other_channel') + authorized = names.with_suffix('my-authorized-uuid') + token = admin.grant_token() \ + .channels([ + Channel.id(readonly).read(), + Channel.id(readwrite).read().write(), + ]) \ + .authorized_uuid(authorized) \ + .ttl(15) \ + .sync().result.token + redact.register(token) + without = holder(clients, 'my-authorized-uuid') + pubnub = holder(clients, 'my-authorized-uuid') + + # snippet.set_token + pubnub.set_token(token) + # snippet.end + + eventually(lambda: allowed(pubnub, readwrite), timeout=30, what='publish on the granted channel') + refused(pubnub, readonly) + refused(pubnub, fresh) + refused(without, readwrite) + + +@step('revoke_token', role='pam', timeout=240) +def test_revoke_token(clients): + admin = clients.sync('pam', 'admin') + pubnub = admin + channel = names.with_suffix('revoke-channel') + token = admin.grant_token() \ + .channels([Channel.id(channel).read().write()]) \ + .authorized_uuid(names.with_suffix('my-authorized-uuid')) \ + .ttl(15) \ + .sync().result.token + redact.register(token) + holder_client = holder(clients, 'my-authorized-uuid', token) + eventually(lambda: allowed(holder_client, channel), timeout=30, what='publish with the fresh token') + + # A client without a secret key cannot revoke. The SDK refuses before it sends the request. + keyless = holder(clients, 'my-authorized-uuid') + error = a.raises(PubNubException, lambda: keyless.revoke_token(token).sync()) + a.contains(str(error), 'Secret key is not configured') + + # snippet.revoke_token + envelope = pubnub.revoke_token(token).sync() + # snippet.end + + a.eq(envelope.result.status, 200) + a.eq(str(envelope.result), 'Revoke token success with status: 200') + eventually(lambda: refused(holder_client, channel), timeout=100, interval=2, what='the revoked token is refused') + for number in (1, 2): + fresh_client = holder(clients, 'my-authorized-uuid', token) + refused(fresh_client, channel) + + +@step('grant_token_categories', 'parse_token_categories', role='pam') +def test_token_categories(clients): + pubnub = clients.sync('pam', 'admin') + + # snippet.grant_token_categories + from pubnub.models.consumer.v3.category import Category + + envelope = pubnub.grant_token() \ + .categories([Category.channels().get(), Category.uuids().get()]) \ + .authorized_uuid('my_authorized_uuid') \ + .ttl(15) \ + .sync() + token = envelope.result.get_token() + # snippet.end + redact.register(token) + + # snippet.parse_token_categories + categories = pubnub.parse_token(token)['categories'] + # snippet.end + + a.eq(categories, {'channels': {'get': True}, 'uuids': {'get': True}}) + parsed = pubnub.parse_token(token) + a.eq(parsed['authorized_uuid'], names.with_suffix('my_authorized_uuid')) + a.eq(parsed['resources'].get('channels', {}), {}) + a.eq(parsed['resources'].get('uuids', {}), {}) + + # A token with a single category carries only that category. + channels_only = pubnub.grant_token().categories([Category.channels().get()]).ttl(15).sync().result.token + redact.register(channels_only) + a.eq(pubnub.parse_token(channels_only)['categories'], {'channels': {'get': True}}) + + # The SDK refuses an invalid category before it sends the request. + error = a.raises(PubNubException, lambda: pubnub.grant_token().categories([Channel.id('x').get()]).ttl(15).sync()) + a.contains(str(error), 'Unsupported category. Only Category.channels() and Category.uuids() are supported') + error = a.raises(PubNubException, lambda: pubnub.grant_token().categories([Category.channels()]).ttl(15).sync()) + a.contains(str(error), "Category permissions must be exactly the 'get' permission") + + +@step('grant_token_multi', 'grant_token_regex', 'grant_token_multi_regex', role='pam') +def test_grant_token_resources(clients): + pubnub = clients.sync('pam', 'admin') + authorized = names.with_suffix('my-authorized-uuid') + + # snippet.grant_token_multi + from pubnub.models.consumer.v3.channel import Channel + from pubnub.models.consumer.v3.group import Group + from pubnub.models.consumer.v3.uuid import UUID + + channels = [ + Channel.id("channel-a").read(), + Channel.id("channel-b").read().write(), + Channel.id("channel-c").read().write(), + Channel.id("channel-d").read().write() + ] + groups = [ + Group.id("channel-group-b").read() + ] + uuids = [ + UUID.id("uuid-c").get(), + UUID.id("uuid-d").get().update() + ] + envelope = pubnub.grant_token() \ + .channels(channels) \ + .groups(groups) \ + .uuids(uuids) \ + .authorized_uuid("my-authorized-uuid") \ + .ttl(15) \ + .sync() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['authorized_uuid'], authorized) + a.eq(parsed['resources']['channels'], { + 'channel-a': perms('read'), + 'channel-b': perms('read', 'write'), + 'channel-c': perms('read', 'write'), + 'channel-d': perms('read', 'write'), + }) + a.eq(parsed['resources']['groups'], {'channel-group-b': perms('read')}) + a.eq(parsed['resources']['uuids'], {'uuid-c': perms('get'), 'uuid-d': perms('get', 'update')}) + a.eq(parsed['patterns'].get('channels', {}), {}) + + # snippet.grant_token_regex + envelope = pubnub.grant_token() \ + .channels([Channel.pattern("channel-[A-Za-z0-9]").read()]) \ + .authorized_uuid("my-authorized-uuid") \ + .ttl(15) \ + .sync() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['patterns']['channels'], {'channel-[A-Za-z0-9]': perms('read')}) + a.eq(parsed['resources'].get('channels', {}), {}) + + # snippet.grant_token_multi_regex + channels = [ + Channel.id("channel-a").read(), + Channel.pattern("channel-[A-Za-z0-9]").read(), + Channel.id("channel-b").read().write(), + Channel.id("channel-c").read().write(), + Channel.id("channel-d").read().write() + ] + groups = [ + Group.id("channel-group-b").read() + ] + uuids = [ + UUID.id("uuid-c").get(), + UUID.id("uuid-d").get().update() + ] + envelope = pubnub.grant_token() \ + .channels(channels) \ + .groups(groups) \ + .uuids(uuids) \ + .authorized_uuid("my-authorized-uuid") \ + .ttl(15) \ + .sync() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['patterns']['channels'], {'channel-[A-Za-z0-9]': perms('read')}) + a.eq(sorted(parsed['resources']['channels']), ['channel-a', 'channel-b', 'channel-c', 'channel-d']) + a.eq(parsed['resources']['groups'], {'channel-group-b': perms('read')}) + a.eq(parsed['resources']['uuids'], {'uuid-c': perms('get'), 'uuid-d': perms('get', 'update')}) + + +@step('grant_token_spaces_builder', 'grant_token_spaces_named', 'grant_token_spaces_multi', + 'grant_token_spaces_regex', 'grant_token_spaces_multi_regex', role='pam') +def test_grant_token_spaces_users(clients): + pubnub = clients.sync('pam', 'admin') + + # snippet.grant_token_spaces_builder + from pubnub.models.consumer.v3.space import Space + + envelope = pubnub.grant_token() \ + .authorized_user('some_user_id') \ + .spaces([ + Space.id('some_space_id').read().write(), + Space.pattern('some_*').read().write() + ]) \ + .ttl(60) \ + .sync() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['ttl'], 60) + a.eq(parsed['authorized_uuid'], 'some_user_id') + a.eq(parsed['resources']['channels'], {'some_space_id': perms('read', 'write')}) + a.eq(parsed['patterns']['channels'], {'some_*': perms('read', 'write')}) + + # snippet.grant_token_spaces_named + grant = pubnub.grant_token( + spaces=[Space.id('some_space_id').read().write(), Space.pattern('some_*').read().write()], + authorized_user_id="some_user_id", + ttl=60 + ).sync() + # snippet.end + redact.register(grant.result.get_token()) + named = pubnub.parse_token(grant.result.get_token()) + for key in ('ttl', 'authorized_uuid', 'resources', 'patterns'): + a.eq(named[key], parsed[key], key) + + # snippet.grant_token_spaces_multi + from pubnub.models.consumer.v3.user import User + + spaces = [ + Space.id("space-a").read(), + Space.id("space-b").read().write(), + Space.id("space-c").read().write(), + Space.id("space-d").read().write() + ] + users = [ + User.id("userId-c").get(), + User.id("userId-d").get().update() + ] + envelope = pubnub.grant_token() \ + .spaces(spaces) \ + .ttl(15) \ + .users(users) \ + .authorized_user("my-authorized-userId") \ + .sync() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['authorized_uuid'], 'my-authorized-userId') + a.eq(parsed['resources']['channels'], { + 'space-a': perms('read'), + 'space-b': perms('read', 'write'), + 'space-c': perms('read', 'write'), + 'space-d': perms('read', 'write'), + }) + a.eq(parsed['resources']['uuids'], {'userId-c': perms('get'), 'userId-d': perms('get', 'update')}) + + # snippet.grant_token_spaces_regex + envelope = pubnub.grant_token() \ + .spaces([Space.pattern("space-[A-Za-z0-9]").read()]) \ + .ttl(15) \ + .authorized_user("my-authorized-userId") \ + .sync() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['patterns']['channels'], {'space-[A-Za-z0-9]': perms('read')}) + + # snippet.grant_token_spaces_multi_regex + spaces = [ + Space.id("space-a").read(), + Space.pattern("space-[A-Za-z0-9]").read(), + Space.id("space-b").read().write(), + Space.id("space-c").read().write(), + Space.id("space-d").read().write() + ] + users = [ + User.id("user-c").get(), + User.id("user-d").get().update() + ] + envelope = pubnub.grant_token() \ + .spaces(spaces) \ + .ttl(15) \ + .users(users) \ + .authorized_user("my-authorized-userId") \ + .sync() + # snippet.end + parsed = pubnub.parse_token(envelope.result.token) + redact.register(envelope.result.token) + a.eq(parsed['patterns']['channels'], {'space-[A-Za-z0-9]': perms('read')}) + a.eq(sorted(parsed['resources']['channels']), ['space-a', 'space-b', 'space-c', 'space-d']) + a.eq(parsed['resources']['uuids'], {'user-c': perms('get'), 'user-d': perms('get', 'update')}) diff --git a/docs-snippets/sync/access_manager_v2.py b/docs-snippets/sync/access_manager_v2.py new file mode 100644 index 00000000..36fde8ca --- /dev/null +++ b/docs-snippets/sync/access_manager_v2.py @@ -0,0 +1,318 @@ +"""Docs page: access-manager-v2 (sync), the legacy grant(). + +Needs the pam keyset. Every grant is closed in a finally (all flags false, ttl 1). The step that grants at application +level is the last one of this file, because a downgrade is not guaranteed to apply at once and would otherwise leak +into the 403 checks of later steps. Tokens and keys never reach output. +""" +from pubnub.exceptions import PubNubException + +from harness import assertions as a +from harness import names +from harness.listeners import listen +from harness.step import step +from harness.wait import eventually + +REGISTERED_NAMES = ['my_channel-pnpres', 'girish', 'girish2', 'cg2', 'cg3'] + +ALL_FALSE = ('read', 'write', 'manage', 'delete', 'get', 'update', 'join') + + +def holder(clients, user_id, auth_key=None): + """A client that has no secret key, so every request is checked against its auth key.""" + client = clients.sync('pam', user_id, secret=False) + client.config.auth_key = auth_key + return client + + +def close_grant(admin, channels=None, groups=None, uuids=None, auth_keys=None): + """Take back what a step granted: every flag false, ttl 1.""" + builder = admin.grant() + for flag in ALL_FALSE: + builder = getattr(builder, flag)(False) + if channels: + builder = builder.channels(channels) + if groups: + builder = builder.channel_groups(groups) + if uuids: + builder = builder.uuids(uuids) + if auth_keys: + builder = builder.auth_keys(auth_keys) + return builder.ttl(1).sync() + + +def publish(client, channel): + return client.publish().channel(channel).message('docs').sync() + + +def refused(client, channel): + error = a.raises(PubNubException, publish, client, channel) + a.eq(error.get_status_code(), 403, 'status code of the refusal') + + +def allowed(client, channel): + a.timetoken(publish(client, channel).result.timetoken) + + +def key_flags(key_data): + return (key_data.read_enabled, key_data.write_enabled, key_data.manage_enabled, key_data.delete_enabled) + + +@step('grant_channel_auth', role='pam') +def test_grant_channel_and_auth_key(clients): + pubnub = clients.sync('pam', 'admin') + channels = [names.with_suffix(name) for name in ('ch1', 'ch2', 'ch3')] + groups = [names.with_suffix(name) for name in ('cg1', 'cg2')] + other = names.with_suffix('ch_no_grant') + try: + # snippet.grant_channel_auth + envelope = pubnub.grant().channels(["ch1", "ch2", "ch3"]).\ + channel_groups(["cg1", "cg2"]).\ + auth_keys(["key1", "key2"]).\ + read(True).write(True).manage(True).delete(True).\ + ttl(12337).\ + sync() + # snippet.end + + result = envelope.result + a.eq(result.level, 'channel-group+auth') + a.eq(result.ttl, 12337) + a.ok(isinstance(result.subscribe_key, str) and result.subscribe_key, 'the result names the subscribe key') + a.eq(sorted(result.channels), sorted(channels)) + a.eq(sorted(result.groups), sorted(groups)) + for data in list(result.channels.values()) + list(result.groups.values()): + a.eq(sorted(data.auth_keys), ['key1', 'key2']) + for key_data in data.auth_keys.values(): + a.eq(key_flags(key_data), (True, True, True, True)) + eventually(lambda: allowed(holder(clients, 'granted', 'key1'), channels[0]), timeout=30, + what='a granted auth key publishes') + refused(holder(clients, 'stranger', 'key3'), channels[0]) + refused(holder(clients, 'granted', 'key1'), other) + finally: + close_grant(pubnub, channels, groups, auth_keys=['key1', 'key2']) + + +@step('grant_channel_all_users', 'grant_channel_level', role='pam') +def test_grant_channel_level(clients): + pubnub = clients.sync('pam', 'admin') + channel = names.with_suffix('my_channel') + other = names.with_suffix('ch_no_grant') + try: + # snippet.grant_channel_all_users + envelope = pubnub.grant().channels("my_channel").read(True).write(True).sync() + # snippet.end + + a.eq(envelope.result.level, 'channel') + a.eq(envelope.result.ttl, 1440) + a.eq(envelope.result.channels[channel].read_enabled, True) + a.eq(envelope.result.channels[channel].write_enabled, True) + client = holder(clients, 'anyone') + eventually(lambda: allowed(client, channel), timeout=30, what='publish without an auth key') + refused(client, other) + + # snippet.grant_channel_level + envelope = pubnub.grant().channels("my_channel").\ + read(True).write(True).\ + sync() + # snippet.end + + a.eq(envelope.result.level, 'channel') + a.eq(envelope.result.channels[channel].read_enabled, True) + finally: + close_grant(pubnub, [channel]) + + +@step('grant_channel_ro_auth', 'grant_user_level', role='pam', timeout=900) +def test_grant_user_level(clients): + pubnub = clients.sync('pam', 'admin') + channel = names.with_suffix('my_channel') + other = names.with_suffix('ch_no_grant') + try: + # snippet.grant_channel_ro_auth + envelope = pubnub.grant().channels("my_channel").read(True).write(False).\ + auth_keys("my_ro_authkey").ttl(5).sync() + # snippet.end + + result = envelope.result + a.eq(result.level, 'user') + a.eq(result.ttl, 5) + a.eq(list(result.channels), [channel]) + a.eq(list(result.channels[channel].auth_keys), ['my_ro_authkey']) + key_data = result.channels[channel].auth_keys['my_ro_authkey'] + a.eq((key_data.read_enabled, key_data.write_enabled), (True, False)) + reader = holder(clients, 'reader', 'my_ro_authkey') + # The read grant is proved by a subscribe that connects. The missing write grant by a refused publish. + listen(reader, channel) + eventually(lambda: refused(reader, channel), timeout=300, + what='reader write refused, a channel level downgrade can take minutes') + eventually(lambda: refused(holder(clients, 'stranger', 'other_authkey'), channel), timeout=300, + what='a stranger auth key refused, a channel level downgrade can take minutes') + finally: + close_grant(pubnub, [channel], auth_keys=['my_ro_authkey']) + + try: + # snippet.grant_user_level + envelope = pubnub.grant().channels("my_channel"). \ + auth_keys("my_authkey"). \ + read(True).write(True). \ + ttl(5). \ + sync() + # snippet.end + + a.eq(envelope.result.level, 'user') + a.eq(key_flags(envelope.result.channels[channel].auth_keys['my_authkey'])[:2], (True, True)) + eventually(lambda: allowed(holder(clients, 'writer', 'my_authkey'), channel), timeout=30, + what='publish with the granted auth key') + eventually(lambda: refused(holder(clients, 'writer', 'my_authkey'), other), timeout=300, + what='a channel without a grant refused, a channel level downgrade can take minutes') + finally: + close_grant(pubnub, [channel], auth_keys=['my_authkey']) + + +@step('grant_presence', role='pam') +def test_grant_presence_channel(clients): + pubnub = clients.sync('pam', 'admin') + presence = names.with_suffix('my_channel-pnpres') + try: + # snippet.grant_presence + envelope = pubnub.grant(). \ + channels("my_channel-pnpres").read(True).write(True).sync() + # snippet.end + + a.eq(envelope.result.level, 'channel') + a.eq(list(envelope.result.channels), [presence]) + a.eq(envelope.result.channels[presence].read_enabled, True) + a.eq(envelope.result.channels[presence].write_enabled, True) + client = holder(clients, 'presence_reader') + eventually(lambda: a.eq(client.here_now().channels(presence).sync().status.is_error(), False), timeout=30, + what='a here_now on the presence channel is not refused') + finally: + close_grant(pubnub, [presence]) + + +@step('grant_channel_group', role='pam') +def test_grant_channel_group(clients): + pubnub = clients.sync('pam', 'admin') + groups = [names.with_suffix(name) for name in ('cg1', 'cg2', 'cg3')] + member = names.with_suffix('group_member') + try: + # snippet.grant_channel_group + envelope = pubnub.grant(). \ + channel_groups(["cg1", "cg2", "cg3"]). \ + auth_keys(["auth1", "auth2", "auth3"]). \ + read(True).write(True).manage(True). \ + ttl(12237). \ + sync() + # snippet.end + + result = envelope.result + a.eq(result.level, 'channel-group+auth') + a.eq(result.ttl, 12237) + a.eq(sorted(result.groups), sorted(groups)) + for data in result.groups.values(): + a.eq(sorted(data.auth_keys), ['auth1', 'auth2', 'auth3']) + for key_data in data.auth_keys.values(): + a.eq(key_flags(key_data)[:3], (True, True, True)) + manager = holder(clients, 'group_manager', 'auth1') + eventually(lambda: manager.add_channel_to_channel_group().channels(member).channel_group(groups[0]).sync(), + timeout=30, what='the manage grant lets the auth key edit the group') + listed = pubnub.list_channels_in_channel_group().channel_group(groups[0]).sync().result.channels + a.eq(list(listed), [member]) + error = a.raises(PubNubException, holder(clients, 'stranger', 'auth9').add_channel_to_channel_group() + .channels(member).channel_group(groups[1]).sync) + a.eq(error.get_status_code(), 403) + finally: + try: + pubnub.remove_channel_group().channel_group(groups[0]).sync() + finally: + close_grant(pubnub, groups=groups, auth_keys=['auth1', 'auth2', 'auth3']) + + +@step('grant_multiple_channels', role='pam') +def test_grant_multiple_channels(clients): + pubnub = clients.sync('pam', 'admin') + channels = [names.with_suffix('girish'), names.with_suffix('girish2')] + try: + # snippet.grant_multiple_channels + envelope = pubnub.grant().\ + auth_keys("my_rw_authkey").\ + channels(["girish", "girish2"]).\ + read(True).\ + write(True).\ + ttl(5).\ + sync() + # snippet.end + + a.eq(envelope.result.level, 'user') + a.eq(envelope.result.ttl, 5) + a.eq(sorted(envelope.result.channels), sorted(channels)) + for data in envelope.result.channels.values(): + a.eq(list(data.auth_keys), ['my_rw_authkey']) + a.eq(key_flags(data.auth_keys['my_rw_authkey'])[:2], (True, True)) + client = holder(clients, 'rw', 'my_rw_authkey') + for channel in channels: + eventually(lambda: allowed(client, channel), timeout=30, what='publish on a granted channel') + finally: + close_grant(pubnub, channels, auth_keys=['my_rw_authkey']) + + +@step('grant_uuid', role='pam') +def test_grant_uuid(clients): + pubnub = clients.sync('pam', 'admin') + uuid = names.with_suffix('my_uuid') + try: + # snippet.grant_uuid + envelope = pubnub.grant().uuids("my_uuid").get(True).update(True) \ + .auth_keys("my_ro_authkey").ttl(1440).sync() + # snippet.end + + a.eq(envelope.result.level, 'uuid') + a.eq(envelope.result.ttl, 1440) + a.eq(list(envelope.result.uuids), [uuid]) + key_data = envelope.result.uuids[uuid].auth_keys['my_ro_authkey'] + a.eq((key_data.get, key_data.update), (True, True)) + a.eq((key_data.read_enabled, key_data.write_enabled), (False, False)) + finally: + close_grant(pubnub, uuids=[uuid], auth_keys=['my_ro_authkey']) + + +@step('grant_categories', role='pam') +def test_grant_categories(clients): + pubnub = clients.sync('pam', 'admin') + auth_key = 'my_auth_key' + try: + # snippet.grant_categories + envelope = pubnub.grant() \ + .auth_keys(auth_key) \ + .categories(['channels', 'uuids']) \ + .get(True) \ + .ttl(15) \ + .sync() + print(f"Granted category permissions, level: {envelope.result.level}") + # snippet.end + + result = envelope.result + a.eq(result.level, 'category') + a.eq(result.ttl, 15) + a.eq(sorted(result.categories), ['channels', 'uuids']) + for name, data in result.categories.items(): + a.eq(data.name, name) + a.eq(list(data.auth_keys), [auth_key]) + a.eq(data.auth_keys[auth_key].get, True) + a.eq(data.auth_keys[auth_key].read_enabled, False) + parsed_back = pubnub.grant().auth_keys(auth_key).categories(['channels']).get(False).sync().result + a.eq(parsed_back.categories['channels'].auth_keys[auth_key].get, False) + finally: + pubnub.grant().auth_keys(auth_key).categories(['channels', 'uuids']).get(False).sync() + + # The SDK refuses these before it sends the request. + def invalid(builder): + return a.raises(PubNubException, builder.sync) + + a.contains(str(invalid(pubnub.grant().auth_keys(auth_key).categories(['groups']).get(True).ttl(15))), + "Unsupported categories: groups. Only 'channels' and 'uuids' are supported") + a.contains(str(invalid(pubnub.grant().categories(['channels']).get(True).ttl(15))), + 'Category grants require providing non empty authKeys') + a.contains(str(invalid(pubnub.grant().auth_keys(auth_key).categories(['channels']).read(True).get(True).ttl(15))), + "Category grants allow only the 'get' permission") + a.contains(str(invalid(pubnub.grant().auth_keys(auth_key).categories(['channels']).get(True).ttl(1))), + 'One-minute (ttl=1) category grants are not supported') diff --git a/docs-snippets/sync/channel_groups.py b/docs-snippets/sync/channel_groups.py new file mode 100644 index 00000000..2cb8ae0f --- /dev/null +++ b/docs-snippets/sync/channel_groups.py @@ -0,0 +1,143 @@ +"""Docs page: channel groups (sync). Add, list, remove channels and delete the group. + +Every step registers its group with the janitor and deletes it in a finally. +""" +from harness import assertions as a +from harness import names, world +from harness.listeners import listen +from harness.step import step +from harness.wait import eventually + +REGISTERED_NAMES = ['son', 'daughter', 'channel_group'] + + +def members(client, group): + return sorted(client.list_channels_in_channel_group().channel_group(group).sync().result.channels) + + +def eventually_members(client, group, expected): + eventually(lambda: a.eq(members(client, group), sorted(expected), 'channels in the group'), timeout=30, + what='group membership') + + +@step('add_channels_builder', 'add_channels_named', role='open') +def test_add_channels(clients): + pubnub = clients.sync('open', 'grouper') + group = names.with_suffix('cg1') + channels = [names.with_suffix('ch1'), names.with_suffix('ch2')] + world.record('channel_group', group) + try: + # snippet.add_channels_builder + envelope = pubnub.add_channel_to_channel_group() \ + .channels(["ch1", "ch2"]) \ + .channel_group("cg1") \ + .sync() + # snippet.end + a.eq(envelope.status.is_error(), False) + a.eq(str(envelope.result), 'Channel successfully added') + eventually_members(pubnub, group, channels) + + # A subscriber on the group receives a publish to a member channel + seen = listen(clients.sync('open', 'member_listener'), groups=[group]) + pubnub.publish().channel(channels[0]).message('to-member').sync() + eventually(lambda: a.eq([m.message for m in seen.messages], ['to-member'], 'group message received')) + a.eq((seen.messages[0].channel, seen.messages[0].subscription), (channels[0], group)) + + pubnub.remove_channel_from_channel_group().channels(channels).channel_group(group).sync() + eventually_members(pubnub, group, []) + + # snippet.add_channels_named + envelope = pubnub.add_channel_to_channel_group(channels=["ch1", "ch2"], channel_group="cg1").sync() + # snippet.end + a.eq(envelope.status.is_error(), False) + eventually_members(pubnub, group, channels) + finally: + pubnub.remove_channel_group().channel_group(group).sync() + + +@step('list_channels_builder', 'list_channels_named', role='open') +def test_list_channels(clients): + pubnub = clients.sync('open', 'lister') + group = names.with_suffix('cg1') + channels = [names.with_suffix('ch1'), names.with_suffix('ch2')] + world.record('channel_group', group) + try: + pubnub.add_channel_to_channel_group().channels(channels).channel_group(group).sync() + eventually_members(pubnub, group, channels) + + # snippet.list_channels_builder + result = pubnub.list_channels_in_channel_group() \ + .channel_group("cg1") \ + .sync() + print(result.result.channels) + # snippet.end + a.eq(sorted(result.result.channels), channels) + a.eq(type(result.result.channels), list, 'channels is a list, not a dictionary') + + # snippet.list_channels_named + result = pubnub.list_channels_in_channel_group(channel_group="cg1").sync() + # snippet.end + a.eq(sorted(result.result.channels), channels) + a.eq(members(pubnub, names.with_suffix('missing_group')), [], 'an unknown group is empty') + finally: + pubnub.remove_channel_group().channel_group(group).sync() + + +@step('remove_channels_builder', 'remove_channels_named', role='open') +def test_remove_channels(clients): + pubnub = clients.sync('open', 'remover') + group = names.with_suffix('channel_group') + kids = [names.with_suffix('son'), names.with_suffix('daughter')] + pair = [names.with_suffix('ch1'), names.with_suffix('ch2')] + other_group = names.with_suffix('cg1') + world.record('channel_group', group) + world.record('channel_group', other_group) + try: + pubnub.add_channel_to_channel_group().channels(kids + pair).channel_group(group).sync() + pubnub.add_channel_to_channel_group().channels(pair).channel_group(other_group).sync() + eventually_members(pubnub, group, kids + pair) + + # snippet.remove_channels_builder + envelope = pubnub.remove_channel_from_channel_group() \ + .channels(["son", "daughter"]) \ + .channel_group("channel_group") \ + .sync() + # snippet.end + a.eq(envelope.status.is_error(), False) + a.eq(str(envelope.result), 'Channel successfully removed') + eventually_members(pubnub, group, pair) + + # snippet.remove_channels_named + envelope = pubnub.remove_channel_from_channel_group(channels=["ch1", "ch2"], channel_group="cg1").sync() + # snippet.end + a.eq(envelope.status.is_error(), False) + eventually_members(pubnub, other_group, []) + a.eq(members(pubnub, group), sorted(pair), 'the other group is untouched') + finally: + pubnub.remove_channel_group().channel_group(group).sync() + pubnub.remove_channel_group().channel_group(other_group).sync() + + +@step('delete_channel_group_builder', role='open') +def test_delete_channel_group(clients): + pubnub = clients.sync('open', 'deleter') + group = names.with_suffix('cg1') + channels = [names.with_suffix('ch1'), names.with_suffix('ch2')] + world.record('channel_group', group) + try: + pubnub.add_channel_to_channel_group().channels(channels).channel_group(group).sync() + eventually_members(pubnub, group, channels) + + # snippet.delete_channel_group_builder + envelope = pubnub.remove_channel_group() \ + .channel_group("cg1") \ + .sync() + # snippet.end + a.eq(envelope.status.is_error(), False) + a.eq(str(envelope.result), 'Group successfully removed') + eventually_members(pubnub, group, []) + + # The docs show remove_channel_group(channel_group="cg1"), but the method takes no argument + a.raises(TypeError, lambda: pubnub.remove_channel_group(channel_group=group)) + finally: + pubnub.remove_channel_group().channel_group(group).sync() diff --git a/docs-snippets/sync/configuration.py b/docs-snippets/sync/configuration.py new file mode 100644 index 00000000..f9da661a --- /dev/null +++ b/docs-snippets/sync/configuration.py @@ -0,0 +1,299 @@ +"""Docs page: configuration (sync). Source: docs/sdks/python/api-reference/configuration.md. + +The initialization samples use made up keys, so each step overwrites the keys on the same config object after the +sample ran and then proves the client works against the service. Validation branches run against an unreachable origin +with the request handler replaced by a counter, so they prove that no request was sent. +""" +import re +from os import environ +from unittest import mock + +from pubnub.exceptions import PubNubException +from pubnub.pubnub import PubNub +from pubnub.request_handlers.httpx import HttpxRequestHandler + +from harness import assertions as a +from harness import names, world +from harness.listeners import listen +from harness.step import step +from harness.wait import eventually + +USER_ID_ERROR = 'UUID missing or invalid type' # pubnub/pnconfiguration.py, validate_not_empty_string +UNREACHABLE = '127.0.0.1:1' + + +def real_keys(config): + """Copy the keyset of the open role onto a config the sample built with made up keys.""" + real = world.config('open') + config.subscribe_key = real.subscribe_key + config.publish_key = real.publish_key + + +def works(clients, pubnub, config): + """Overwrite the made up keys and prove the client talks to the service. Returns the time() envelope.""" + clients._made.append(pubnub) + real_keys(config) + config.daemon = True + envelope = pubnub.time().sync() + a.timetoken(str(envelope.result)) + return envelope + + +class RequestCounter: + """Replaces the request handler call so that a validation branch can prove no request was sent.""" + + def __init__(self): + self.calls = 0 + + def __call__(self, *args, **kwargs): + self.calls += 1 + raise RuntimeError('a request was sent') + + +@step('configuration_full', role='open') +def test_full_configuration(clients): + channel = names.with_suffix('my_channel') + real = world.config('open') + saved = {k: environ.get(k) for k in ('SUBSCRIBE_KEY', 'PUBLISH_KEY', 'USER_ID')} + environ.update({'SUBSCRIBE_KEY': real.subscribe_key, 'PUBLISH_KEY': real.publish_key, + 'USER_ID': names.with_suffix('configured_user')}) + try: + # snippet.configuration_full + import os + from pubnub.pnconfiguration import PNConfiguration + from pubnub.enums import PNHeartbeatNotificationOptions + from pubnub.pubnub import PubNub + from pubnub.crypto import AesCbcCryptoModule + + # Configuration for PubNub instance + pn_configuration = PNConfiguration() + + # Set configuration values + pn_configuration.subscribe_key = os.getenv('SUBSCRIBE_KEY', 'demo') # required + pn_configuration.publish_key = os.getenv('PUBLISH_KEY', 'demo') # only required if publishing + pn_configuration.secret_key = os.getenv('SECRET_KEY', 'my_secret_key') # optional + pn_configuration.cipher_key = os.getenv('CIPHER_KEY', 'my_cipher_key') # for encryption/decryption + pn_configuration.crypto_module = AesCbcCryptoModule(pn_configuration) # for encryption/decryption + pn_configuration.user_id = os.getenv('USER_ID', 'my_custom_user_id') # must be set + pn_configuration.auth_key = os.getenv('AUTH_KEY', 'my_auth_key') # optional + pn_configuration.ssl = True + pn_configuration.connect_timeout = 100 + pn_configuration.subscribe_request_timeout = 310 + pn_configuration.non_subscribe_request_timeout = 300 + pn_configuration.filter_expression = "such == 'wow'" + pn_configuration.heartbeat_notification_options = PNHeartbeatNotificationOptions.ALL + + # Initialize the PubNub client + pubnub = PubNub(pn_configuration) + + print("Configuration set up complete and PubNub client initialized.") + # snippet.end + finally: + for key, value in saved.items(): + if value is None: + environ.pop(key, None) + else: + environ[key] = value + clients._made.append(pubnub) + a.eq(pn_configuration.subscribe_key, real.subscribe_key, 'keys come from the environment') + a.eq(pubnub.uuid, names.with_suffix('configured_user')) + a.eq((pn_configuration.connect_timeout, pn_configuration.subscribe_request_timeout, + pn_configuration.non_subscribe_request_timeout), (100, 310, 300), 'timeouts') + a.eq(pn_configuration.heartbeat_notification_options, PNHeartbeatNotificationOptions.ALL) + a.eq((pn_configuration.auth_key, pn_configuration.filter_expression), ('my_auth_key', "such == 'wow'")) + + # The same crypto module on a reader: a message published with the sample client is readable + reader_config = world.config('open', 'config_reader') + reader_config.cipher_key = 'my_cipher_key' + reader_config.crypto_module = pn_configuration.crypto_module.__class__(reader_config) + reader = PubNub(reader_config) + clients._made.append(reader) + seen = listen(reader, channel) + envelope = pubnub.publish().channel(channel).message({'configured': True}).sync() + a.timetoken(envelope.result.timetoken) + eventually(lambda: a.eq([m.message for m in seen.messages], [{'configured': True}], 'reader decrypts')) + + +@step('initialize_pubnub', 'initialize_non_secure', 'initialize_ssl', 'initialize_custom_user_id', role='open') +def test_initialize(clients): + # snippet.initialize_pubnub + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub import PubNub + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.ssl = True + pnconfig.user_id = "my_custom_user_id" + pubnub = PubNub(pnconfig) + # snippet.end + a.eq((pubnub.uuid, pnconfig.ssl), ('my_custom_user_id', True)) + a.ok(works(clients, pubnub, pnconfig).status.tls_enabled, 'requests use TLS') + + # snippet.initialize_non_secure + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub import PubNub + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.ssl = False + pnconfig.user_id = "my_custom_user_id" + pubnub = PubNub(pnconfig) + # snippet.end + a.eq(pnconfig.scheme(), 'http') + a.eq(works(clients, pubnub, pnconfig).status.tls_enabled, False, 'requests are plain http') + + # snippet.initialize_ssl + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub import PubNub + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.ssl = True + pnconfig.user_id = "my_custom_user_id" + pubnub = PubNub(pnconfig) + # snippet.end + a.eq(pnconfig.scheme_and_host(), 'https://ps.pndsn.com') + a.ok(works(clients, pubnub, pnconfig).status.tls_enabled, 'requests use TLS') + + # snippet.initialize_custom_user_id + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub import PubNub + + pnconfig = PNConfiguration() + + pnconfig.subscribe_key = 'mySubscribeKey' + pnconfig.publish_key = 'myPublishKey' + pnconfig.user_id = "my_custom_user_id" + + pubnub = PubNub(pnconfig) + # snippet.end + a.eq(pubnub.uuid, 'my_custom_user_id') + a.eq(pnconfig.user_id, pnconfig.uuid, 'user_id and the deprecated uuid are the same value') + works(clients, pubnub, pnconfig) + + # The user ID is mandatory. The branches below never reach the network. + counter = RequestCounter() + with mock.patch.object(HttpxRequestHandler, 'sync_request', counter): + missing = PNConfiguration() + missing.subscribe_key = 'mySubscribeKey' + missing.origin = UNREACHABLE + a.raises(AssertionError, PubNub, missing, match=re.escape(USER_ID_ERROR)) + for bad in ('', ' ', None, 42): + a.raises(AssertionError, setattr, missing, 'user_id', bad, match=re.escape(USER_ID_ERROR)) + a.eq(missing.user_id, None, 'a rejected user_id is not stored') + a.eq(counter.calls, 0, 'no request was sent') + + +@step('initialize_custom_request_handler', role='open') +def test_initialize_custom_request_handler(clients): + # snippet.initialize_custom_request_handler + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub import PubNub + from pubnub.request_handlers.requests import RequestsRequestHandler + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.user_id = "my_custom_user_id" + + pubnub = PubNub(pnconfig, custom_request_handler=RequestsRequestHandler) + # snippet.end + a.ok(isinstance(pubnub._request_handler, RequestsRequestHandler), 'the custom handler is in use') + a.ok(not isinstance(pubnub._request_handler, HttpxRequestHandler), 'the default handler is replaced') + works(clients, pubnub, pnconfig) + a.raises(Exception, PubNub, pnconfig, custom_request_handler=dict, match='subclass of BaseRequestHandler') + + +@step('initialize_read_only', role='open') +def test_initialize_read_only(clients): + channel = names.with_suffix('my_channel') + + # snippet.initialize_read_only + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub import PubNub + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.user_id = "my_custom_user_id" + + pubnub = PubNub(pnconfig) + # snippet.end + clients._made.append(pubnub) + a.eq(pnconfig.publish_key, None, 'no publish key') + + counter = RequestCounter() + pnconfig.origin = UNREACHABLE + with mock.patch.object(HttpxRequestHandler, 'sync_request', counter): + error = a.raises(PubNubException, pubnub.publish().channel(channel).message('x').sync, + match='Publish key not configured') + a.eq(str(error), 'Publish key not configured', 'exact error text') + a.eq(counter.calls, 0, 'no request was sent') + + # It still reads: the same config with the real subscribe key receives a message another client published + pnconfig.origin = world.config('open').origin + pnconfig.subscribe_key = world.config('open').subscribe_key + pnconfig.daemon = True + seen = listen(pubnub, channel) + clients.sync('open', 'writer').publish().channel(channel).message('read-me').sync() + eventually(lambda: a.eq([m.message for m in seen.messages], ['read-me'], 'the read-only client received')) + + +@step('initialize_access_manager', role='open') +def test_initialize_access_manager(clients): + # snippet.initialize_access_manager + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub import PubNub + + pnconfig = PNConfiguration() + pnconfig.subscribe_key = "my_subkey" + pnconfig.publish_key = "my_pubkey" + pnconfig.secret_key = "my_secretkey" + pnconfig.user_id = "my_custom_user_id" + pnconfig.ssl = True + + pubnub = PubNub(pnconfig) + # snippet.end + a.eq(pnconfig.secret_key, 'my_secretkey') + clients._made.append(pubnub) + + counter = RequestCounter() + pnconfig.origin = UNREACHABLE + with mock.patch.object(HttpxRequestHandler, 'sync_request', counter): + a.raises(RuntimeError, pubnub.grant().channels('x').read(True).sync, match='a request was sent') + a.eq(counter.calls, 1, 'the grant call reaches the handler once the secret key is configured') + keyless = PubNub(world.config('open', 'no_secret')) + clients._made.append(keyless) + a.raises(PubNubException, keyless.grant().channels('x').read(True).sync, match='Secret key is not configured') + a.eq(counter.calls, 1, 'without a secret key no request is built') + + +@step('set_filter_expression', 'get_filter_expression', role='open') +def test_filter_expression(clients): + channel = names.with_suffix('my_channel') + + # snippet.set_filter_expression + from pubnub.pnconfiguration import PNConfiguration + + pnconfig = PNConfiguration() + pnconfig.filter_expression = "such == 'wow'" + # snippet.end + real_keys(pnconfig) + pnconfig.user_id = names.with_suffix('filtered') + pnconfig.daemon = True + filtered = PubNub(pnconfig) + clients._made.append(filtered) + seen = listen(filtered, channel) + publisher = clients.sync('open', 'filter_publisher') + publisher.publish().channel(channel).message('no-meta').sync() + publisher.publish().channel(channel).message('other').meta({'such': 'meh'}).sync() + publisher.publish().channel(channel).message('matching').meta({'such': 'wow'}).sync() + eventually(lambda: a.eq([m.message for m in seen.messages], ['matching'], 'only the matching message arrived')) + + # snippet.get_filter_expression + filter = pnconfig.filter_expression + # snippet.end + a.eq(filter, "such == 'wow'") + a.eq(world.config('open').filter_expression, None, 'the default is no filter') diff --git a/docs-snippets/sync/encryption.py b/docs-snippets/sync/encryption.py new file mode 100644 index 00000000..cfa869c0 --- /dev/null +++ b/docs-snippets/sync/encryption.py @@ -0,0 +1,192 @@ +"""Docs page: encryption (sync only, the asyncio docs have no encryption page). + +Source: docs/sdks/python/api-reference/encryption.md. The client of each docs sample is built by the sample itself, +so the page snippets show the crypto module lines on a PNConfiguration that the step created. +""" +import os +import tempfile +from pathlib import Path + +from Cryptodome.Cipher import AES +from pubnub.crypto import AesCbcCryptoModule, LegacyCryptoModule, PubNubAesCbcCryptor, PubNubCryptoModule +from pubnub.enums import PNStatusCategory +from pubnub.exceptions import PubNubException +from pubnub.pubnub import PubNub + +from harness import assertions as a +from harness import names, world +from harness.listeners import listen +from harness.step import step +from harness.wait import eventually + +FIXTURE_BYTES = (Path(__file__).resolve().parent.parent / 'fixtures' / 'sample.gif').read_bytes() +CBC_HEADER_B64 = 'UE5FRA' # base64 of the "PNED" sentinel that starts every 256-bit AES-CBC payload + + +def reader(clients, user, module=None, key='my_cipher_key'): + """A client that reads with the given crypto module, or with none.""" + config = world.config('open', user) + config.cipher_key = key + config.cipher_mode = AES.MODE_GCM + config.fallback_cipher_mode = AES.MODE_CBC + if module is None: + config.cipher_key = None + config.cipher_mode = AES.MODE_CBC + config.fallback_cipher_mode = None + else: + config.crypto_module = module(config) + client = PubNub(config) + clients._made.append(client) + return client + + +def only_message(seen): + eventually(lambda: a.eq(len(seen.messages), 1, 'messages received')) + return seen.messages[0].message + + +@step('crypto_module_aes_cbc', role='open') +def test_aes_cbc_crypto_module(clients): + channel = names.with_suffix('my_channel') + secret = {'secret': 'knights who say ni'} + config = world.config('open', 'encrypting_publisher') + + # snippet.crypto_module_aes_cbc + # encrypts using 256-bit AES-CBC cipher (recommended) + # decrypts data encrypted with the legacy (AES and GCM) and the 256-bit AES-CBC ciphers + config.cipher_key = "my_cipher_key" + config.cipher_mode = AES.MODE_GCM # optional, used for the legacy module only + config.fallback_cipher_mode = AES.MODE_CBC # optional, used for the legacy module only + config.crypto_module = AesCbcCryptoModule(config) + pubnub = PubNub(config) + # snippet.end + clients._made.append(pubnub) + same_key = listen(reader(clients, 'same_key', AesCbcCryptoModule), channel) + no_key = listen(reader(clients, 'no_key'), channel) + wrong_key = listen(reader(clients, 'wrong_key', AesCbcCryptoModule, key='another_cipher_key'), channel) + + envelope = pubnub.publish().channel(channel).message(secret).sync() + + a.timetoken(envelope.result.timetoken) + a.eq(only_message(same_key), secret, 'a subscriber with the same key reads the message') + unreadable = only_message(no_key) + a.ok(isinstance(unreadable, str) and unreadable.startswith(CBC_HEADER_B64), 'without a key the 256-bit payload') + a.ok('ni' not in unreadable, 'the plain text is not in the payload') + a.ok(isinstance(only_message(wrong_key), str), 'a wrong key leaves the payload encrypted') + a.ok(only_message(wrong_key) != secret, 'a wrong key does not read the message') + a.ok(wrong_key.messages[0].error is not None, 'the wrong key is reported as an error on the message') + a.contains(wrong_key.categories, PNStatusCategory.PNDecryptionErrorCategory, 'decryption error status') + + +@step('crypto_module_legacy', role='open') +def test_legacy_crypto_module(clients): + channel = names.with_suffix('my_channel') + secret = {'secret': 'legacy knights'} + config = world.config('open', 'legacy_publisher') + + # snippet.crypto_module_legacy + # encrypts with 128-bit cipher key entropy (legacy) with GCM + # decrypts data encrypted with the legacy (AES and GCM) and the 256-bit AES-CBC ciphers + config.cipher_key = "my_cipher_key" + config.cipher_mode = AES.MODE_GCM # optional, used for the legacy module only + config.fallback_cipher_mode = AES.MODE_CBC # optional, used for the legacy module only + config.crypto_module = LegacyCryptoModule(config) + pubnub = PubNub(config) + # snippet.end + clients._made.append(pubnub) + legacy_reader = listen(reader(clients, 'legacy_reader', LegacyCryptoModule), channel) + cbc_reader = listen(reader(clients, 'cbc_reader', AesCbcCryptoModule), channel) + no_key = listen(reader(clients, 'no_key'), channel) + + pubnub.publish().channel(channel).message(secret).sync() + + a.eq(only_message(legacy_reader), secret, 'the legacy module reads its own payload') + a.eq(only_message(cbc_reader), secret, 'the 256-bit module reads the legacy payload') + unreadable = only_message(no_key) + a.ok(isinstance(unreadable, str), 'without a key the payload is a string') + a.ok(not unreadable.startswith(CBC_HEADER_B64), 'the legacy payload has no 256-bit header') + a.ok(unreadable != secret and 'legacy' not in unreadable, 'the plain text is not in the payload') + + # The other direction: a message from a 256-bit client is readable by the legacy module + cbc_config = world.config('open', 'cbc_publisher') + cbc_config.cipher_key = 'my_cipher_key' + cbc_config.cipher_mode = AES.MODE_GCM + cbc_config.fallback_cipher_mode = AES.MODE_CBC + cbc_config.crypto_module = AesCbcCryptoModule(cbc_config) + cbc_publisher = PubNub(cbc_config) + clients._made.append(cbc_publisher) + cbc_publisher.publish().channel(channel).message('from-cbc').sync() + eventually(lambda: a.contains([m.message for m in legacy_reader.messages], 'from-cbc', 'legacy reads 256-bit')) + eventually(lambda: a.eq(len(no_key.messages), 2, 'second message arrived')) + a.ok(no_key.messages[1].message.startswith(CBC_HEADER_B64), 'the 256-bit payload carries its header') + + +@step('crypto_module_partial', role='open') +def test_partial_crypto_module(clients): + config = world.config('open', 'partial_crypto') + pubnub = PubNub(config) # pubnub instance without encryption + clients._made.append(pubnub) + other = PubNub(world.config('open', 'other_key')) + clients._made.append(other) + other.crypto = PubNubCryptoModule({ + PubNubAesCbcCryptor.CRYPTOR_ID: PubNubAesCbcCryptor('anotherCipherKey') + }, PubNubAesCbcCryptor) + a.eq(pubnub.crypto, None, 'no crypto module before the sample') + + # snippet.crypto_module_partial + pubnub.crypto = PubNubCryptoModule({ + PubNubAesCbcCryptor.CRYPTOR_ID: PubNubAesCbcCryptor('myCipherKey') + }, PubNubAesCbcCryptor) + encrypted = pubnub.crypto.encrypt('My Secret Text') # encrypted wih AES cryptor and `myCipherKey` cipher key + decrypted = pubnub.crypto.decrypt(encrypted) + # snippet.end + a.eq(decrypted, 'My Secret Text', 'round trip') + a.ok(encrypted != 'My Secret Text' and 'Secret' not in encrypted, 'the payload is encrypted') + a.ok(encrypted.startswith(CBC_HEADER_B64), 'the payload is the 256-bit format') + a.eq(pubnub.crypto.decrypt(pubnub.crypto.encrypt('{"a": 1}')), {'a': 1}, 'json text decodes to the object') + a.raises(PubNubException, other.crypto.decrypt, encrypted, match='decryption error') + + +@step('encrypt_file', 'decrypt_file', role='open') +def test_encrypt_and_decrypt_file(clients): + pubnub = reader(clients, 'file_crypto', AesCbcCryptoModule) + plain = reader(clients, 'file_plain') + channel = names.with_suffix('ch1') + payload_to_encrypt = FIXTURE_BYTES + + # snippet.encrypt_file + encrypted_payload = pubnub.crypto.encrypt_file(payload_to_encrypt) + # snippet.end + a.ok(isinstance(encrypted_payload, bytes), 'bytes come back') + a.ok(encrypted_payload != FIXTURE_BYTES and encrypted_payload.startswith(b'PNED'), '256-bit encrypted bytes') + payload_to_decrypt_in_bytes = encrypted_payload + + # snippet.decrypt_file + decrypted_payload = pubnub.crypto.decrypt_file(payload_to_decrypt_in_bytes) + # snippet.end + a.eq(decrypted_payload, FIXTURE_BYTES, 'round trip') + + # The same module on the file API: what is stored is encrypted, what the same client downloads is plain + target = tempfile.mkdtemp(prefix='docs-snippets-') + path = os.path.join(target, 'sample.gif') + with open(path, 'wb') as f: + f.write(FIXTURE_BYTES) + uploaded = None + try: + with open(path, 'rb') as sample_file: + uploaded = pubnub.send_file().channel(channel).file_name('sample.gif') \ + .file_object(sample_file).sync().result + downloaded = pubnub.download_file().channel(channel).file_id(uploaded.file_id).file_name('sample.gif').sync() + a.eq(downloaded.result.data, FIXTURE_BYTES, 'the encrypting client downloads the original') + stored = plain.download_file().channel(channel).file_id(uploaded.file_id).file_name('sample.gif').sync() + a.ok(stored.result.data != FIXTURE_BYTES, 'the stored file is not the original') + a.ok(stored.result.data.startswith(b'PNED'), 'the stored file is in the 256-bit format') + a.eq(pubnub.crypto.decrypt_file(stored.result.data), FIXTURE_BYTES, 'the stored bytes decrypt to the original') + finally: + if uploaded is not None: + try: + pubnub.delete_file().channel(channel).file_id(uploaded.file_id).file_name('sample.gif').sync() + except Exception as e: + print('cleanup: could not delete the file (' + type(e).__name__ + ')') + os.remove(path) + os.rmdir(target) diff --git a/docs-snippets/sync/file_sharing.py b/docs-snippets/sync/file_sharing.py new file mode 100644 index 00000000..0f9df167 --- /dev/null +++ b/docs-snippets/sync/file_sharing.py @@ -0,0 +1,297 @@ +"""Docs page: file sharing (sync). Source: docs/sdks/python/api-reference/files.md. + +Every step uploads its own fixture file and deletes it in a finally block. The fixture is 134 bytes, so a download can +be compared byte for byte. +""" +import os +import tempfile +import urllib.request +from pathlib import Path + +from harness import assertions as a +from harness import names +from harness.step import step +from harness.wait import eventually + +FIXTURE = Path(__file__).resolve().parent.parent / 'fixtures' / 'sample.gif' +FIXTURE_BYTES = FIXTURE.read_bytes() + + +def channel_name(): + return names.with_suffix('ch1') + + +def upload(pubnub, channel=None, store=True): + """Seed a file outside the snippet markers. Returns the PNSendFileResult.""" + with open(FIXTURE, 'rb') as sample_file: + return pubnub.send_file().channel(channel or channel_name()).file_name('sample.gif') \ + .file_object(sample_file).should_store(store).sync().result + + +def remove(pubnub, *uploaded): + for item in uploaded: + try: + pubnub.delete_file().channel(channel_name()).file_id(item.file_id).file_name(item.name).sync() + except Exception as e: # cleanup must never hide the real failure + print('cleanup: could not delete a file (' + type(e).__name__ + ')') + + +def stored_file_messages(pubnub, *file_ids): + """File events read back from Message Persistence. A subscriber callback cannot be used, see the report: the SDK + raises AttributeError in models/subscription.py for every file event and the worker thread dies.""" + envelope = pubnub.fetch_messages().channels(channel_name()).include_meta(True).sync() + items = envelope.result.channels.get(channel_name(), []) + files = [item for item in items if isinstance(item.message, dict) and 'file' in item.message] + return [item for item in files if not file_ids or item.message['file']['id'] in file_ids] + + +def listed_ids(pubnub): + envelope = pubnub.list_files().channel(channel_name()).sync() + return [item['id'] for item in envelope.result.data] + + +@step('send_file_builder', 'send_file_custom_message_type', role='open') +def test_send_file(clients): + pubnub = clients.sync('open', 'file_sender') + file_path = str(FIXTURE) + sent = [] + try: + # snippet.send_file_builder + with open(file_path, 'rb') as sample_file: + response = pubnub.send_file() \ + .channel("ch1") \ + .file_name("sample.gif") \ + .message({"test_message": "test"}) \ + .should_store(True) \ + .file_object(sample_file) \ + .sync() + print("File sent successfully. File ID:", response.result.file_id) + # snippet.end + sent.append(response.result) + a.ok(response.result.file_id, 'file id returned') + a.eq(response.result.name, 'sample.gif') + a.timetoken(response.result.timestamp) + a.ok(not response.status.is_error(), 'status is not an error') + + # snippet.send_file_custom_message_type + with open(file_path, 'rb') as sample_file: + response_2 = pubnub.send_file() \ + .channel("ch1") \ + .file_name("sample.gif") \ + .message("Hey, it's gif not jif") \ + .custom_message_type("file-message") \ + .should_store(True) \ + .file_object(sample_file) \ + .sync() + print("File sent successfully. File ID:", response_2.result.file_id) + # snippet.end + sent.append(response_2.result) + + a.ok(response_2.result.file_id != response.result.file_id, 'every upload gets its own id') + both = (response.result.file_id, response_2.result.file_id) + eventually(lambda: a.eq(len(stored_file_messages(pubnub, *both)), 2, 'file events stored')) + by_id = {m.message['file']['id']: m for m in stored_file_messages(pubnub, *both)} + first_event = by_id[response.result.file_id] + a.eq(first_event.message['message'], {'test_message': 'test'}) + a.eq(first_event.message['file']['name'], 'sample.gif') + a.eq(str(first_event.timetoken), str(response.result.timestamp), 'the file message carries the timestamp') + a.eq(by_id[response_2.result.file_id].message['message'], "Hey, it's gif not jif") + eventually(lambda: a.contains(listed_ids(pubnub), response.result.file_id, 'listed after upload')) + listing = pubnub.list_files().channel(channel_name()).sync().result.data + entry = [item for item in listing if item['id'] == response.result.file_id][0] + a.eq(entry['name'], 'sample.gif') + a.eq(entry['size'], len(FIXTURE_BYTES), 'listed size is the fixture size') + finally: + remove(pubnub, *sent) + + +@step('list_files_builder', 'list_files_named', role='open') +def test_list_files(clients): + pubnub = clients.sync('open', 'file_lister') + first, second = upload(pubnub), upload(pubnub) + try: + eventually(lambda: a.contains(listed_ids(pubnub), second.file_id, 'second upload listed')) + + # snippet.list_files_builder + file_list_response = pubnub.list_files().channel("ch1").sync() + print(f"Found {len(file_list_response.result.data)} files:") + + for file_data in file_list_response.result.data: + print(f" {file_data['name']} with id: {file_data['id']}") + # snippet.end + result = file_list_response.result + a.contains([item['id'] for item in result.data], first.file_id) + a.contains([item['id'] for item in result.data], second.file_id) + a.eq(result.count, len(result.data), 'count matches the page') + a.ok({'id', 'name', 'size', 'created'} <= set(result.data[0]), 'file entry keys') + + # snippet.list_files_named + page_1 = pubnub.list_files(channel="ch1", limit=1).sync() + page_2 = pubnub.list_files(channel="ch1", limit=1, next=page_1.result.next).sync() + # snippet.end + a.eq(len(page_1.result.data), 1, 'first page has the limit') + a.eq(len(page_2.result.data), 1, 'second page has the limit') + a.ok(page_1.result.data[0]['id'] != page_2.result.data[0]['id'], 'the next token moves forward') + a.eq({page_1.result.data[0]['id'], page_2.result.data[0]['id']}, {first.file_id, second.file_id}) + finally: + remove(pubnub, first, second) + + +@step('get_file_url_builder', 'get_file_url_named', role='open') +def test_get_file_url(clients): + pubnub = clients.sync('open', 'file_url_reader') + uploaded = upload(pubnub) + file_data = {'id': uploaded.file_id, 'name': uploaded.name} + try: + # snippet.get_file_url_builder + download_url = pubnub.get_file_url() \ + .channel("ch1") \ + .file_id(file_data['id']) \ + .file_name(file_data['name']) \ + .sync() + + print(f' Download url: {download_url.result.file_url}') + # snippet.end + url_builder = download_url.result.file_url + + # snippet.get_file_url_named + download_url = pubnub.get_file_url(channel="ch1", + file_id=file_data['id'], + file_name=file_data['name']) \ + .sync() + + print(f' Download url: {download_url.result.file_url}') + # snippet.end + url_named = download_url.result.file_url + + for url in (url_builder, url_named): + a.ok(url.startswith('https://'), 'url scheme') + a.contains(url, uploaded.file_id) + with urllib.request.urlopen(url, timeout=30) as served: + a.eq(served.status, 200, 'the url serves the file') + a.eq(served.read(), FIXTURE_BYTES, 'the url serves the same bytes') + finally: + remove(pubnub, uploaded) + + +@step('download_file_builder', role='open') +def test_download_file(clients): + pubnub = clients.sync('open', 'file_downloader') + uploaded = upload(pubnub) + file_data = {'id': uploaded.file_id, 'name': uploaded.name} + target = tempfile.mkdtemp(prefix='docs-snippets-') + save_path = os.path.join(target, file_data['name']) + try: + # snippet.download_file_builder + download_file = pubnub.download_file() \ + .channel("ch1") \ + .file_id(file_data['id']) \ + .file_name(file_data['name']) \ + .sync() + + with open(save_path, 'wb') as fw: + fw.write(download_file.result.data) + + print(f"File saved as {save_path}") + # snippet.end + a.eq(download_file.result.data, FIXTURE_BYTES, 'downloaded bytes equal the upload') + with open(save_path, 'rb') as saved: + a.eq(saved.read(), FIXTURE_BYTES, 'saved bytes equal the upload') + finally: + remove(pubnub, uploaded) + if os.path.exists(save_path): + os.remove(save_path) + os.rmdir(target) + + +@step('delete_file_builder', 'delete_file_named', role='open') +def test_delete_file(clients): + pubnub = clients.sync('open', 'file_deleter') + one, two = upload(pubnub), upload(pubnub) + remaining = [one, two] + try: + eventually(lambda: a.contains(listed_ids(pubnub), two.file_id, 'uploads listed')) + file_data = {'id': one.file_id, 'name': one.name} + + # snippet.delete_file_builder + envelope = pubnub.delete_file() \ + .channel("ch1") \ + .file_id(file_data['id']) \ + .file_name(file_data['name']) \ + .sync() + + print("File deleted") + # snippet.end + remaining.remove(one) + a.eq(envelope.result.status, 200, 'delete status') + eventually(lambda: a.not_contains(listed_ids(pubnub), one.file_id, 'deleted file still listed')) + a.contains(listed_ids(pubnub), two.file_id, 'the other file is untouched') + + file_data = {'id': two.file_id, 'name': two.name} + + # snippet.delete_file_named + envelope = pubnub.delete_file(channel="ch1", + file_id=file_data['id'], + file_name=file_data['name']) \ + .sync() + + print("File deleted") + # snippet.end + remaining.remove(two) + a.eq(envelope.result.status, 200, 'delete status') + eventually(lambda: a.not_contains(listed_ids(pubnub), two.file_id, 'deleted file still listed')) + a.raises(Exception, lambda: pubnub.download_file().channel(channel_name()).file_id(one.file_id) + .file_name(one.name).sync()) + finally: + remove(pubnub, *remaining) + + +@step('publish_file_message_builder', 'publish_file_message_callback', role='open') +def test_publish_file_message(clients): + pubnub = clients.sync('open', 'file_publisher') + uploaded = upload(pubnub) + results = [] + try: + eventually(lambda: a.eq(len(stored_file_messages(pubnub, uploaded.file_id)), 1, 'upload stored a message')) + file_id, file_name = uploaded.file_id, uploaded.name + + # snippet.publish_file_message_builder + envelope = pubnub.publish_file_message() \ + .channel("ch1") \ + .meta({"test": "test"}) \ + .message({"test_message": "test"}) \ + .custom_message_type("file-message") \ + .file_id(file_id) \ + .file_name(file_name) \ + .should_store(True) \ + .ttl(22) \ + .sync() + # snippet.end + a.timetoken(envelope.result.timestamp) + eventually(lambda: a.eq(len(stored_file_messages(pubnub, file_id)), 2, 'file message stored')) + stored = stored_file_messages(pubnub, file_id) + event = [m for m in stored if str(m.timetoken) == str(envelope.result.timestamp)][0] + a.eq(event.message['file'], {'id': file_id, 'name': file_name}) + a.eq(event.message['message'], {'test_message': 'test'}) + a.eq(event.meta, {'test': 'test'}) + + # snippet.publish_file_message_callback + def callback(response, status): + results.append((response, status)) + + pubnub.publish_file_message() \ + .channel("ch1") \ + .message("Bring me a shrubbery") \ + .file_id(file_id) \ + .file_name(file_name) \ + .should_store(True) \ + .pn_async(callback) + # snippet.end + eventually(lambda: a.eq(len(results), 1, 'callback called')) + response, status = results[0] + a.ok(not status.is_error(), 'callback status is not an error') + a.timetoken(response.timestamp) + eventually(lambda: a.contains([m.message['message'] for m in stored_file_messages(pubnub, file_id)], + 'Bring me a shrubbery', 'callback file message stored')) + finally: + remove(pubnub, uploaded) diff --git a/docs-snippets/sync/message_actions.py b/docs-snippets/sync/message_actions.py new file mode 100644 index 00000000..eaab3d62 --- /dev/null +++ b/docs-snippets/sync/message_actions.py @@ -0,0 +1,192 @@ +"""Docs page: message actions (sync). Add, remove and get message actions. + +A message action needs a stored message, so every step waits until the published message is readable from history. +""" +from pubnub.models.consumer.message_actions import PNMessageAction + +from harness import assertions as a +from harness import names +from harness.listeners import listen +from harness.step import step +from harness.wait import eventually + +REGISTERED_NAMES = ['chats.room1'] + + +def publish_stored(client, channel, count=1): + """Publish count messages and wait until history returns them. Returns their timetokens, oldest first.""" + tokens = [int(client.publish().channel(channel).message({'n': i}).sync().result.timetoken) for i in range(count)] + + def stored(): + got = client.fetch_messages(channels=[channel], start=tokens[-1] + 1, end=tokens[0]).sync() + a.eq([int(i.timetoken) for i in got.result.channels.get(channel, [])], tokens, 'stored timetokens') + + eventually(stored, timeout=30, what='published messages stored') + return tokens + + +def add_action(client, channel, message_timetoken, value, action_type='reaction'): + action = PNMessageAction() + action.type = action_type + action.value = value + action.message_timetoken = message_timetoken + return client.add_message_action().channel(channel).message_action(action).sync().result + + +@step('add_message_action_builder', 'add_message_action_named', role='open') +def test_add_message_action(clients): + pubnub = clients.sync('open', 'reactor') + channel = names.with_suffix('chats.room1') + seen = listen(clients.sync('open', 'watcher'), channel) + message_timetoken = publish_stored(pubnub, channel)[0] + + # snippet.add_message_action_builder + # snippet.hide + from pubnub.models.consumer.message_actions import PNMessageAction + # snippet.show + msg_action = PNMessageAction() + msg_action.type = "reaction" + msg_action.value = "smiley_face" + msg_action.message_timetoken = message_timetoken + + envelope = pubnub.add_message_action() \ + .channel("chats.room1") \ + .message_action(msg_action) \ + .sync() + + print(f"Response: {envelope.result.__dict__}") + # snippet.end + + added = envelope.result + a.timetoken(added.action_timetoken) + a.eq((added.type, added.value, added.uuid), ('reaction', 'smiley_face', pubnub.uuid)) + a.eq(int(added.message_timetoken), message_timetoken) + eventually(lambda: a.eq(len(seen.message_actions), 1, 'action events received')) + event = seen.message_actions[0] + a.eq((event.type, event.value, event.uuid), ('reaction', 'smiley_face', pubnub.uuid)) + a.eq(event.action_timetoken, added.action_timetoken) + a.eq(int(event.message_timetoken), message_timetoken) + a.eq(event.channel, channel) + + # snippet.add_message_action_named + msg_action = PNMessageAction() + msg_action.type = "reaction" + msg_action.value = "thumbs_up" + msg_action.message_timetoken = message_timetoken + + envelope = pubnub.add_message_action( + channel="chats.room1", + message_action=msg_action + ).sync() + + print(f"Response: {envelope.result.__dict__}") + # snippet.end + a.eq(envelope.result.value, 'thumbs_up') + + def read_back(): + got = pubnub.fetch_messages(channels=[channel], count=1, include_message_actions=True).sync() + actions = got.result.channels[channel][0].actions + a.eq(sorted(actions['reaction']), ['smiley_face', 'thumbs_up'], 'fetch shows both actions') + a.eq(actions['reaction']['smiley_face'][0]['actionTimetoken'], added.action_timetoken) + a.eq(actions['reaction']['smiley_face'][0]['uuid'], pubnub.uuid) + + eventually(read_back, timeout=30, what='actions visible through fetch_messages') + eventually(lambda: a.eq(len(seen.message_actions), 2, 'second action event received')) + for action_timetoken in (added.action_timetoken, envelope.result.action_timetoken): + pubnub.remove_message_action(channel=channel, message_timetoken=message_timetoken, + action_timetoken=int(action_timetoken)).sync() + + +@step('get_message_actions_builder', 'get_message_actions_named', role='open') +def test_get_message_actions(clients): + pubnub = clients.sync('open', 'reader') + channel = names.with_suffix('chats.room1') + tokens = publish_stored(pubnub, channel, 3) + first, second, third = [add_action(pubnub, channel, tt, 'smiley_face').action_timetoken for tt in tokens] + a.ok(first < second < third, 'action timetokens grow') + + def all_visible(): + got = pubnub.get_message_actions(channel=channel).sync().result.actions + mine = [x.action_timetoken for x in got if int(x.message_timetoken) in tokens] + a.eq(mine, [first, second, third], 'ascending by action timetoken') + + eventually(all_visible, timeout=30, what='actions listed') + start, end = str(third), str(second) + + # snippet.get_message_actions_builder + envelope = pubnub.get_message_actions() \ + .channel("chats.room1") \ + .start(start) \ + .end(end) \ + .limit("50") \ + .sync() + + for action in envelope.result.actions: + print(f"Message Action type: {action.type}") + print(f"Message Action value: {action.value}") + print(f"Message Action timetoken: {action.message_timetoken}") + print(f"Message Action uuid: {action.uuid}") + print(f"Message Action timetoken: {action.action_timetoken}") + # snippet.end + + built = envelope.result.actions + a.eq(len(built) > 0 and built[-1].uuid, pubnub.uuid) + + # snippet.get_message_actions_named + envelope = pubnub.get_message_actions(channel="chats.room1", start=start, end=end, limit="50").sync() + # snippet.end + named = envelope.result.actions + limited = pubnub.get_message_actions(channel=channel, limit="2").sync().result.actions + a.eq(len(limited), 2, 'limit caps the list') + for token, action_timetoken in zip(tokens, (first, second, third)): + pubnub.remove_message_action(channel=channel, message_timetoken=token, + action_timetoken=int(action_timetoken)).sync() + # Last on purpose: the SDK sends no end boundary when a start boundary is set (get_message_actions.py:50). + a.eq([x.action_timetoken for x in built], [second], 'builder: start excluded, end included') + a.eq(int(built[0].message_timetoken), tokens[1]) + a.eq([x.action_timetoken for x in named], [second], 'named arguments: same range') + + +@step('remove_message_action_builder', 'remove_message_action_named', role='open') +def test_remove_message_action(clients): + pubnub = clients.sync('open', 'remover') + channel = names.with_suffix('chats.room1') + seen = listen(clients.sync('open', 'watcher'), channel) + message_timetoken = publish_stored(pubnub, channel)[0] + first = add_action(pubnub, channel, message_timetoken, 'smiley_face').action_timetoken + second = add_action(pubnub, channel, message_timetoken, 'thumbs_up').action_timetoken + eventually(lambda: a.eq(len(seen.message_actions), 2, 'added events received')) + + def listed(expected): + got = pubnub.get_message_actions(channel=channel).sync().result.actions + a.eq([x.action_timetoken for x in got if int(x.message_timetoken) == message_timetoken], expected) + + eventually(lambda: listed([first, second]), timeout=30, what='both actions listed') + action_timetoken = int(first) + + # snippet.remove_message_action_builder + envelope = pubnub.remove_message_action() \ + .channel("chats.room1") \ + .action_timetoken(action_timetoken) \ + .message_timetoken(message_timetoken) \ + .sync() + # snippet.end + a.eq(envelope.result.__class__.__name__, 'dict', 'the result is an empty dictionary') + a.eq(envelope.result, {}) + eventually(lambda: listed([second]), timeout=30, what='first action removed') + eventually(lambda: a.eq(len(seen.message_actions), 3, 'removed event received')) + a.eq(seen.message_actions[2].action_timetoken, first) + + action_timetoken = int(second) + # snippet.remove_message_action_named + envelope = pubnub.remove_message_action(channel="chats.room1", + action_timetoken=action_timetoken, + message_timetoken=message_timetoken) \ + .sync() + # snippet.end + a.eq(envelope.result, {}) + eventually(lambda: listed([]), timeout=30, what='second action removed') + eventually(lambda: a.eq(len(seen.message_actions), 4, 'second removed event received')) + a.eq(seen.message_actions[3].action_timetoken, second) + got = pubnub.fetch_messages(channels=[channel], count=1, include_message_actions=True).sync() + a.eq(got.result.channels[channel][0].actions, {}, 'history shows no actions') diff --git a/docs-snippets/sync/message_persistence.py b/docs-snippets/sync/message_persistence.py new file mode 100644 index 00000000..f77149b2 --- /dev/null +++ b/docs-snippets/sync/message_persistence.py @@ -0,0 +1,403 @@ +"""Docs page: message persistence (sync). Fetch history, delete messages, message counts and the deprecated history(). + +Each step seeds its own messages and reads them back through bounds made of the seeded timetokens, so leftovers on a +shared channel name from an earlier run of this page (for example the asyncio one) never change a result. +""" +import contextlib +import io +from concurrent.futures import ThreadPoolExecutor + +from pubnub.exceptions import PubNubException + +from harness import assertions as a +from harness import names +from harness.step import step +from harness.wait import eventually + +REGISTERED_NAMES = ['history_channel', 'history_paging_channel'] + + +def seed(client, channel, count, **extra): + """Publish count messages one after the other and return [(message, timetoken)] oldest first.""" + seeded = [] + for index in range(count): + message = {'n': index, 'channel': channel} + builder = client.publish().channel(channel).message(message) + if extra.get('meta'): + builder = builder.meta(extra['meta']) + if extra.get('custom_message_type'): + builder = builder.custom_message_type(extra['custom_message_type']) + seeded.append((message, int(builder.sync().result.timetoken))) + return seeded + + +def seed_parallel(client, channel, count): + """Publish count messages from several threads. Only the timetokens matter, they come back sorted.""" + def one(index): + return int(client.publish().channel(channel).message({'n': index}).sync().result.timetoken) + + with ThreadPoolExecutor(8) as pool: + return sorted(pool.map(one, range(count))) + + +def window(client, channel, newest, oldest, **kwargs): + """Fetch the seeded range: start is exclusive, so one above the newest. Returns the items oldest first.""" + envelope = client.fetch_messages(channels=[channel], start=newest + 1, end=oldest, **kwargs).sync() + return envelope.result.channels.get(channel, []) + + +def wait_stored(client, channel, timetokens): + def check(): + stored = [int(item.timetoken) for item in window(client, channel, max(timetokens), min(timetokens))] + a.eq(stored, sorted(timetokens), 'stored timetokens') + + eventually(check, timeout=30, what='seeded messages stored') + + +@step('fetch_history_builder', 'fetch_history_named', role='open') +def test_fetch_history(clients): + pubnub = clients.sync('open', 'fetcher') + channel = names.with_suffix('my_channel') + seeded = seed(pubnub, channel, 3, meta={'origin': 'docs'}, custom_message_type='text-message') + wait_stored(pubnub, channel, [tt for _, tt in seeded]) + last_message, last_timetoken = seeded[-1] + + # snippet.fetch_history_builder + envelope = pubnub.fetch_messages() \ + .channels(["my_channel"]) \ + .maximum_per_channel(1) \ + .include_message_actions(True) \ + .include_meta(True) \ + .include_message_type(True) \ + .include_uuid(True) \ + .sync() + + for message in envelope.result.channels["my_channel"]: + print("Message: %s" % message.message) + print("Meta: %s" % message.meta) + print("Timetoken: %s" % message.timetoken) + + for action_type in message.actions: + print("Message Action type: " + action_type) + # snippet.end + + def check(items): + a.eq(len(items), 1, 'maximum_per_channel(1) returns one message') + a.eq(items[0].message, last_message) + a.eq(int(items[0].timetoken), last_timetoken) + a.eq(items[0].meta, {'origin': 'docs'}) + a.eq(items[0].uuid, pubnub.uuid) + a.ok(hasattr(items[0], 'message_type'), 'include_message_type adds message_type') + a.eq(items[0].actions, {}, 'no actions yet') + + check(envelope.result.channels[channel]) + + # snippet.fetch_history_named + envelope = pubnub.fetch_messages(channels=["my_channel"], count=1, include_message_actions=True, + include_meta=True, include_message_type=True, include_uuid=True).sync() + + for message in envelope.result.channels["my_channel"]: + print(f"Message: {message.message}") + print(f"Meta: {message.meta}") + print(f"Timetoken: {message.timetoken}") + # snippet.end + check(envelope.result.channels[channel]) + a.eq(sorted(envelope.result.channels), [channel]) + + +@step('fetch_history_range', role='open') +def test_fetch_history_range(clients): + pubnub = clients.sync('open', 'ranger') + channel = names.with_suffix('my_channel') + seeded = seed(pubnub, channel, 5) + tokens = [tt for _, tt in seeded] + messages = [m for m, _ in seeded] + wait_stored(pubnub, channel, tokens) + newer_timetoken, older_timetoken = tokens[3], tokens[1] + + # snippet.fetch_history_range + envelope = pubnub.fetch_messages() \ + .channels(["my_channel"]) \ + .start(newer_timetoken) \ + .end(older_timetoken) \ + .sync() + # snippet.end + + got = envelope.result.channels[channel] + a.eq([int(i.timetoken) for i in got], [tokens[1], tokens[2]], 'start excluded, end included, oldest first') + a.eq([i.message for i in got], messages[1:3]) + only_start = pubnub.fetch_messages(channels=[channel], start=tokens[3], end=tokens[0]).sync() + a.eq([int(i.timetoken) for i in only_start.result.channels[channel]], tokens[0:3], 'older than start') + only_end = pubnub.fetch_messages(channels=[channel], end=tokens[2]).sync().result.channels[channel] + a.eq([int(i.timetoken) for i in only_end][:3], tokens[2:5], 'end only: newer than or equal to end') + a.eq(int(only_end[-1].timetoken), tokens[4], 'end only: includes the newest message') + + +@step('fetch_history_multiple', role='open', timeout=300) +def test_fetch_history_caps(clients): + pubnub = clients.sync('open', 'capper') + channel = names.with_suffix('my_channel') + first = names.with_suffix('ch1') + second = names.with_suffix('ch2') + single = seed_parallel(pubnub, channel, 101) + on_first = seed_parallel(pubnub, first, 26) + on_second = seed_parallel(pubnub, second, 26) + + def single_stored(): + got = pubnub.fetch_messages(channels=[channel], count=150).sync().result.channels[channel] + a.eq([int(i.timetoken) for i in got], single[1:], 'newest 100 of 101') + + eventually(single_stored, timeout=30, what='seeded messages stored') + + def newest_stored(): + got = pubnub.fetch_messages(channels=[first, second], count=30).sync().result.channels + a.eq([int(i.timetoken) for i in got[first]], on_first[1:], 'newest 25 of ch1') + a.eq([int(i.timetoken) for i in got[second]], on_second[1:], 'newest 25 of ch2') + + eventually(newest_stored, timeout=30, what='seeded messages on both channels stored') + + # snippet.fetch_history_multiple + envelope = pubnub.fetch_messages() \ + .channels(["ch1", "ch2"]) \ + .maximum_per_channel(30) \ + .sync() + + for channel_name, items in envelope.result.channels.items(): + print(channel_name, len(items)) + # snippet.end + + for name in (first, second): + a.eq(len(envelope.result.channels[name]), 25, 'several channels: at most 25 per channel') + one = pubnub.fetch_messages(channels=[channel], count=150).sync().result.channels[channel] + a.eq(len(one), 100, 'one channel: at most 100 messages') + a.eq([int(i.timetoken) for i in one], single[1:], 'the newest 100, oldest first') + + +@step('message_counts_builder', 'message_counts_named', 'message_counts_multiple_builder', + 'message_counts_multiple_named', role='open') +def test_message_counts(clients): + pubnub = clients.sync('open', 'counter') + channel = names.with_suffix('my_channel') + other = names.with_suffix('my_channel_2') + seeded = seed(pubnub, channel, 4) + tokens = [tt for _, tt in seeded] + on_other = [tt for _, tt in seed(pubnub, other, 2)] + wait_stored(pubnub, channel, tokens) + wait_stored(pubnub, other, on_other) + timetoken = tokens[0] + other_timetoken = on_other[0] + + def counted(envelope, expected): + a.eq(envelope.result.channels, expected) + + def counted_eventually(build, expected): + eventually(lambda: counted(build(), expected), timeout=30, what='message counts') + + # The service counts messages newer than the timetoken, the message at the timetoken itself is not counted. + counted_eventually(lambda: pubnub.message_counts(channels=channel, channels_timetoken=[timetoken - 1]).sync(), + {channel: 4}) + counted_eventually(lambda: pubnub.message_counts(channels=channel, channels_timetoken=[timetoken]).sync(), + {channel: 3}) + + # snippet.message_counts_builder + envelope = pubnub.message_counts() \ + .channel("my_channel") \ + .channel_timetokens([timetoken]) \ + .sync() + print(envelope.result.channels["my_channel"]) + # snippet.end + counted(envelope, {channel: 3}) + + # snippet.message_counts_named + envelope = pubnub.message_counts(channels="my_channel", channels_timetoken=[tokens[2]]).sync() + # snippet.end + counted(envelope, {channel: 1}) + counted_eventually(lambda: pubnub.message_counts(channels=channel, channels_timetoken=[tokens[3]]).sync(), + {channel: 0}) + + # snippet.message_counts_multiple_builder + envelope = pubnub.message_counts() \ + .channel(["my_channel", "my_channel_2"]) \ + .channel_timetokens([timetoken, other_timetoken]) \ + .sync() + print(envelope.result.channels) + # snippet.end + counted(envelope, {channel: 3, other: 1}) + + # snippet.message_counts_multiple_named + envelope = pubnub.message_counts(channels=["my_channel", "my_channel_2"], + channels_timetoken=[tokens[1], other_timetoken]).sync() + # snippet.end + counted(envelope, {channel: 2, other: 1}) + + +@step('delete_messages_builder', 'delete_messages_named', role='open') +def test_delete_messages(clients): + pubnub = clients.sync('open', 'deleter') + channel = names.with_suffix('my_channel') + seeded = seed(pubnub, channel, 4) + tokens = [tt for _, tt in seeded] + wait_stored(pubnub, channel, tokens) + newer_timetoken, older_timetoken = tokens[2], tokens[1] + + # snippet.delete_messages_builder + envelope = pubnub.delete_messages() \ + .channel("my_channel") \ + .start(newer_timetoken) \ + .end(older_timetoken) \ + .sync() + # snippet.end + a.eq(envelope.status.is_error(), False) + + def left(expected): + got = [int(i.timetoken) for i in window(pubnub, channel, tokens[3], tokens[0])] + a.eq(got, expected) + + eventually(lambda: left([tokens[0], tokens[2], tokens[3]]), timeout=30, what='only the message at end is deleted') + + # snippet.delete_messages_named + envelope = pubnub.delete_messages(channel="my_channel", start=tokens[3], end=tokens[2]).sync() + # snippet.end + a.eq(envelope.status.is_error(), False) + eventually(lambda: left([tokens[0], tokens[3]]), timeout=30, what='the second message is deleted') + + +@step('delete_message_single_builder', 'delete_message_single_named', role='open') +def test_delete_single_message(clients): + pubnub = clients.sync('open', 'single_deleter') + channel = names.with_suffix('my_channel') + seeded = seed(pubnub, channel, 3) + tokens = [tt for _, tt in seeded] + wait_stored(pubnub, channel, tokens) + publish_timetoken = tokens[1] + + # snippet.delete_message_single_builder + envelope = pubnub.delete_messages() \ + .channel("my_channel") \ + .start(publish_timetoken - 1) \ + .end(publish_timetoken) \ + .sync() + # snippet.end + a.eq(envelope.status.is_error(), False) + + def left(expected): + got = [int(i.timetoken) for i in window(pubnub, channel, tokens[2], tokens[0])] + a.eq(got, expected) + + eventually(lambda: left([tokens[0], tokens[2]]), timeout=30, what='exactly the published message is deleted') + + publish_timetoken = tokens[2] + # snippet.delete_message_single_named + envelope = pubnub.delete_messages(channel="my_channel", start=publish_timetoken - 1, end=publish_timetoken).sync() + # snippet.end + a.eq(envelope.status.is_error(), False) + eventually(lambda: left([tokens[0]]), timeout=30, what='the last message is deleted') + + +@step('history_basic', 'history_reverse', 'history_start_reverse', 'history_end', + 'history_include_timetoken', role='open') +def test_history(clients): + pubnub = clients.sync('open', 'historian') + channel = names.with_suffix('history_channel') + seeded = seed(pubnub, channel, 5) + tokens = [tt for _, tt in seeded] + messages = [m for m, _ in seeded] + + def stored(): + a.eq(len(pubnub.history().channel(channel).count(100).sync().result.messages), 5) + + eventually(stored, timeout=30, what='seeded messages stored') + + # snippet.history_basic + envelope = pubnub.history() \ + .channel("history_channel") \ + .count(100) \ + .sync() + # snippet.end + a.eq([m.entry for m in envelope.result.messages], messages, 'oldest first, newest last') + a.eq(envelope.result.start_timetoken, tokens[0]) + a.eq(envelope.result.end_timetoken, tokens[4]) + + # snippet.history_reverse + envelope = pubnub.history() \ + .channel("history_channel") \ + .count(3) \ + .reverse(True) \ + .sync() + # snippet.end + a.eq([m.entry for m in envelope.result.messages], messages[0:3], 'the three oldest') + + # snippet.history_start_reverse + envelope = pubnub.history() \ + .channel("history_channel") \ + .start(tokens[1]) \ + .reverse(True) \ + .sync() + # snippet.end + a.eq([m.entry for m in envelope.result.messages], messages[2:], 'newer than start, start excluded') + + # snippet.history_end + envelope = pubnub.history() \ + .channel("history_channel") \ + .count(100) \ + .end(tokens[2]) \ + .sync() + # snippet.end + a.eq([m.entry for m in envelope.result.messages], messages[2:], 'end included, newest last') + # The docs sample adds start(-1) and reverse(True). The service refuses start(-1) and with reverse(True) the end + # timetoken becomes the newer boundary, so the result is the other half of the timeline. + a.raises(PubNubException, pubnub.history().channel(channel).count(100).start(-1).end(tokens[2]).sync, + match='400') + flipped = pubnub.history().channel(channel).count(100).end(tokens[2]).reverse(True).sync() + a.eq([m.entry for m in flipped.result.messages], messages[:3], 'reverse with end returns the older half') + + # snippet.history_include_timetoken + envelope = pubnub.history() \ + .channel("history_channel") \ + .count(100) \ + .include_timetoken(True) \ + .sync() + # snippet.end + a.eq([m.timetoken for m in envelope.result.messages], tokens) + a.eq([m.entry for m in envelope.result.messages], messages) + + +@step('history_paging', role='open', timeout=300) +def test_history_paging(clients): + pubnub = clients.sync('open', 'pager') + channel = names.with_suffix('history_paging_channel') + tokens = seed_parallel(pubnub, channel, 101) + + def stored(): + a.eq(len(pubnub.history().channel(channel).count(100).sync().result.messages), 100) + + eventually(stored, timeout=30, what='seeded messages stored') + start_timetoken = tokens[-1] + 1 + + # snippet.history_paging + def get_all_messages(start_tt): + envelope = pubnub.history() \ + .channel("history_paging_channel") \ + .count(100) \ + .start(start_tt) \ + .sync() + msgs = envelope.result.messages + start = envelope.result.start_timetoken + end = envelope.result.end_timetoken + count = len(msgs) + + if count > 0: + print("%d" % count) + print("start %d" % start) + print("end %d" % end) + + if count == 100: + get_all_messages(start) + # snippet.end + + output = io.StringIO() + with contextlib.redirect_stdout(output): + get_all_messages(start_timetoken) + lines = output.getvalue().split('\n') + a.eq(lines[0:3], ['100', 'start %d' % tokens[1], 'end %d' % tokens[-1]], 'first page: the newest 100') + a.eq(lines[3:6], ['1', 'start %d' % tokens[0], 'end %d' % tokens[0]], 'second page: the one older message') diff --git a/docs-snippets/sync/misc.py b/docs-snippets/sync/misc.py new file mode 100644 index 00000000..d5602f3a --- /dev/null +++ b/docs-snippets/sync/misc.py @@ -0,0 +1,90 @@ +"""Docs page: miscellaneous utility methods (sync). Source: docs/sdks/python/api-reference/misc.md. + +The sync page has no Time section, the asyncio page has. The time_get step covers the SDK method for both pages. +""" +import time + +from harness import assertions as a +from harness import names, world +from harness.listeners import listen +from harness.step import compile_only +from harness.step import step +from harness.wait import eventually + + +@step('disconnect_method', 'reconnect_method', role='open') +def test_disconnect_and_reconnect(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + channel = names.with_suffix('my_channel') + seen = listen(pubnub, channel) + publisher.publish().channel(channel).message('before').sync() + eventually(lambda: a.eq([m.message for m in seen.messages], ['before'], 'live before disconnect')) + manager = pubnub._subscription_manager + + # snippet.disconnect_method + pubnub._subscription_manager.disconnect() + print("Disconnected from PubNub.") + # snippet.end + a.ok(manager._should_stop, 'the manager is stopped') + publisher.publish().channel(channel).message('while-away').sync() + + # snippet.reconnect_method + pubnub._subscription_manager.reconnect() + # snippet.end + a.ok(not manager._should_stop, 'the manager runs again') + eventually(lambda: a.eq([m.message for m in seen.messages], ['before', 'while-away'], 'caught up after reconnect')) + publisher.publish().channel(channel).message('after').sync() + eventually(lambda: a.eq([m.message for m in seen.messages], ['before', 'while-away', 'after'], 'live again')) + + +@compile_only('reconnect_documented', role='open', + reason='pubnub.reconnect() raises AttributeError: PNSubscriptionRegistry has no reconnect (SDK bug)') +def reconnect_documented(pubnub): + # snippet.reconnect_documented + pubnub.reconnect() + # snippet.end + return None + + +@step('get_subscribed_channels', 'get_subscribed_channel_groups', role='open') +def test_get_subscribed(clients): + pubnub = clients.sync('open', 'subscriber') + admin = clients.sync('open', 'group_admin') + channel_a, channel_b = names.with_suffix('ch1'), names.with_suffix('ch2') + group = names.with_suffix('cg1') + a.eq(pubnub.get_subscribed_channels(), [], 'nothing subscribed yet') + world.record('channel_group', group) + try: + admin.add_channel_to_channel_group().channels([channel_b]).channel_group(group).sync() + listen(pubnub, channel_a, groups=[group]) + + # snippet.get_subscribed_channels + channels = pubnub.get_subscribed_channels() + # snippet.end + a.eq(channels, [channel_a]) + + # snippet.get_subscribed_channel_groups + channel_groups = pubnub.get_subscribed_channel_groups() + # snippet.end + a.eq(channel_groups, [group]) + pubnub.unsubscribe().channels(channel_a).channel_groups(group).execute() + eventually(lambda: a.eq(pubnub.get_subscribed_channels(), [], 'channels after unsubscribe')) + a.eq(pubnub.get_subscribed_channel_groups(), [], 'groups after unsubscribe') + finally: + admin.remove_channel_group().channel_group(group).sync() + + +@step('time_get', role='open') +def test_time(clients): + pubnub = clients.sync('open', 'timekeeper') + + # snippet.time_get + envelope = pubnub.time().sync() + print(f"Current time: {envelope.result}") + # snippet.end + a.timetoken(str(envelope.result)) + a.eq(int(envelope.result), envelope.result.value_as_int) + a.ok(abs(int(envelope.result) / 10000000 - time.time()) < 120, 'server time is within two minutes of this host') + a.ok(abs(pubnub.timestamp() - int(time.time())) <= 1, 'timestamp() is int(time.time())') + a.eq(envelope.result.date_time().year, time.gmtime().tm_year) diff --git a/docs-snippets/sync/mobile_push.py b/docs-snippets/sync/mobile_push.py new file mode 100644 index 00000000..56c90b97 --- /dev/null +++ b/docs-snippets/sync/mobile_push.py @@ -0,0 +1,262 @@ +"""Docs page: mobile push (sync). Source: docs/sdks/python/api-reference/mobile-push.md. + +Only the registration side is testable. A push is delivered by FCM or APNs to a real device, so delivery is not tested. +The device tokens are made up and unique per run, and every gateway is cleaned in a finally block. +""" +import secrets + +from pubnub.enums import PNPushEnvironment, PNPushType + +from harness import assertions as a +from harness import names +from harness.step import step +from harness.wait import eventually + +REGISTERED_NAMES = ['deviceId'] +TOPIC = 'myapptopic' +# An APNs token is hex. The docs example uses 32 hex characters, a real device token has 64. +APNS2_DEVICE = secrets.token_hex(32) +CHANNELS = ['ch1', 'ch2', 'ch3'] + + +def suffixed(channels): + return sorted(names.with_suffix(c) for c in channels) + + +def fcm_device(): + return names.with_suffix('deviceId') + + +def registered(pubnub, push_type): + """Channels the service holds for the device on one gateway.""" + if push_type == PNPushType.APNS2: + envelope = pubnub.list_push_channels().push_type(push_type).device_id(APNS2_DEVICE) \ + .topic(TOPIC).environment(PNPushEnvironment.DEVELOPMENT).sync() + else: + envelope = pubnub.list_push_channels().push_type(push_type).device_id(fcm_device()).sync() + return sorted(envelope.result.channels) + + +def seed(pubnub, channels): + channels = suffixed(channels) + pubnub.add_channels_to_push().push_type(PNPushType.FCM).channels(channels).device_id(fcm_device()).sync() + pubnub.add_channels_to_push().push_type(PNPushType.APNS2).channels(channels).device_id(APNS2_DEVICE) \ + .topic(TOPIC).environment(PNPushEnvironment.DEVELOPMENT).sync() + + +def clear(pubnub): + """Remove the test tokens on every gateway and environment so that nothing outlives the run.""" + for push_type in (PNPushType.FCM, PNPushType.GCM): + try: + pubnub.remove_device_from_push().push_type(push_type).device_id(fcm_device()).sync() + except Exception as e: # cleanup must never hide the real failure + print('cleanup: could not clear a gateway (' + type(e).__name__ + ')') + for push_type in (PNPushType.APNS, PNPushType.APNS2): + for environment in (PNPushEnvironment.DEVELOPMENT, PNPushEnvironment.PRODUCTION): + try: + pubnub.remove_device_from_push().push_type(push_type).device_id(APNS2_DEVICE) \ + .topic(TOPIC).environment(environment).sync() + except Exception as e: + print('cleanup: could not clear a gateway (' + type(e).__name__ + ')') + + +def expect(pubnub, fcm, apns2, what): + eventually(lambda: a.eq(registered(pubnub, PNPushType.FCM), fcm, 'fcm ' + what)) + eventually(lambda: a.eq(registered(pubnub, PNPushType.APNS2), apns2, 'apns2 ' + what)) + + +@step('add_channels_to_push_builder', 'add_channels_to_push_named', role='open') +def test_add_channels_to_push(clients): + pubnub = clients.sync('open', 'push_registrar') + try: + a.ok(len(APNS2_DEVICE) == 64 and set(APNS2_DEVICE) <= set('0123456789abcdef'), 'apns2 token is 64 hex chars') + + # snippet.add_channels_to_push_builder + # For FCM + pubnub.add_channels_to_push() \ + .push_type(PNPushType.FCM) \ + .channels(["ch1", "ch2", "ch3"]) \ + .device_id("deviceId") \ + .sync() + + # For APNS2 device token should be hex + pubnub.add_channels_to_push() \ + .push_type(PNPushType.APNS2) \ + .channels(["ch1", "ch2", "ch3"]) \ + .device_id(APNS2_DEVICE) \ + .topic("myapptopic") \ + .environment(PNPushEnvironment.DEVELOPMENT) \ + .sync() + # snippet.end + expect(pubnub, suffixed(CHANNELS), suffixed(CHANNELS), 'after add (builder)') + clear(pubnub) + expect(pubnub, [], [], 'after clearing') + + # snippet.add_channels_to_push_named + # For FCM + pubnub.add_channels_to_push( + push_type=PNPushType.FCM, + channels=["ch1", "ch2", "ch3"], + device_id="deviceId" + ).sync() + + # For APNS2 + pubnub.add_channels_to_push( + push_type=PNPushType.APNS2, + channels=["ch1", "ch2", "ch3"], + device_id=APNS2_DEVICE, + topic="myapptopic", + environment=PNPushEnvironment.DEVELOPMENT + ).sync() + # snippet.end + expect(pubnub, suffixed(CHANNELS), suffixed(CHANNELS), 'after add (named)') + finally: + clear(pubnub) + + +@step('list_push_channels_builder', 'list_push_channels_named', role='open') +def test_list_push_channels(clients): + pubnub = clients.sync('open', 'push_lister') + try: + seed(pubnub, CHANNELS) + expect(pubnub, suffixed(CHANNELS), suffixed(CHANNELS), 'after seeding') + + # snippet.list_push_channels_builder + # For FCM + envelope = pubnub.list_push_channels() \ + .push_type(PNPushType.FCM) \ + .device_id("deviceId") \ + .sync() + fcm_channels = envelope.result.channels + + # For APNS2 + envelope = pubnub.list_push_channels() \ + .push_type(PNPushType.APNS2) \ + .device_id(APNS2_DEVICE) \ + .topic("myapptopic") \ + .environment(PNPushEnvironment.DEVELOPMENT) \ + .sync() + apns2_channels = envelope.result.channels + # snippet.end + a.eq(sorted(fcm_channels), suffixed(CHANNELS), 'fcm channels listed') + a.eq(sorted(apns2_channels), suffixed(CHANNELS), 'apns2 channels listed') + a.ok(not envelope.status.is_error(), 'status is not an error') + + # snippet.list_push_channels_named + # For FCM + envelope = pubnub.list_push_channels(push_type=PNPushType.FCM, device_id="deviceId").sync() + fcm_channels = envelope.result.channels + + # For APNS2 + envelope = pubnub.list_push_channels(push_type=PNPushType.APNS2, + device_id=APNS2_DEVICE, + topic="myapptopic", + environment=PNPushEnvironment.DEVELOPMENT) \ + .sync() + apns2_channels = envelope.result.channels + # snippet.end + a.eq(sorted(fcm_channels), suffixed(CHANNELS), 'fcm channels listed (named)') + a.eq(sorted(apns2_channels), suffixed(CHANNELS), 'apns2 channels listed (named)') + + # A device nothing was registered for lists no channels + other = pubnub.list_push_channels().push_type(PNPushType.FCM).device_id(names.with_suffix('unknown')).sync() + a.eq(other.result.channels, [], 'unknown device') + finally: + clear(pubnub) + + +@step('remove_channels_from_push_builder', 'remove_channels_from_push_named', role='open') +def test_remove_channels_from_push(clients): + pubnub = clients.sync('open', 'push_remover') + keep = 'keep_me' + try: + seed(pubnub, CHANNELS + [keep]) + expect(pubnub, suffixed(CHANNELS + [keep]), suffixed(CHANNELS + [keep]), 'after seeding') + + # snippet.remove_channels_from_push_builder + # For FCM + envelope = pubnub.remove_channels_from_push() \ + .push_type(PNPushType.FCM) \ + .channels(["ch1", "ch2", "ch3"]) \ + .device_id("deviceId") \ + .sync() + + # For APNS2 + envelope = pubnub.remove_channels_from_push() \ + .push_type(PNPushType.APNS2) \ + .channels(["ch1", "ch2", "ch3"]) \ + .device_id(APNS2_DEVICE) \ + .topic("myapptopic") \ + .environment(PNPushEnvironment.DEVELOPMENT) \ + .sync() + # snippet.end + a.ok(not envelope.status.is_error(), 'status is not an error') + expect(pubnub, suffixed([keep]), suffixed([keep]), 'after remove (builder), only the other channel is left') + + seed(pubnub, CHANNELS) + expect(pubnub, suffixed(CHANNELS + [keep]), suffixed(CHANNELS + [keep]), 'after seeding again') + + # snippet.remove_channels_from_push_named + # For FCM + envelope = pubnub.remove_channels_from_push(channels=["ch1", "ch2", "ch3"], + push_type=PNPushType.FCM, + device_id="deviceId") \ + .sync() + + # For APNS2 + envelope = pubnub.remove_channels_from_push(channels=["ch1", "ch2", "ch3"], + push_type=PNPushType.APNS2, + device_id=APNS2_DEVICE, + topic="myapptopic", + environment=PNPushEnvironment.DEVELOPMENT) \ + .sync() + # snippet.end + a.ok(not envelope.status.is_error(), 'status is not an error') + expect(pubnub, suffixed([keep]), suffixed([keep]), 'after remove (named), only the other channel is left') + finally: + clear(pubnub) + + +@step('remove_device_from_push_builder', 'remove_device_from_push_named', role='open') +def test_remove_device_from_push(clients): + pubnub = clients.sync('open', 'push_device_remover') + try: + seed(pubnub, CHANNELS) + expect(pubnub, suffixed(CHANNELS), suffixed(CHANNELS), 'after seeding') + + # snippet.remove_device_from_push_builder + # For FCM + envelope = pubnub.remove_device_from_push() \ + .push_type(PNPushType.FCM) \ + .device_id("deviceId") \ + .sync() + + # For APNS2 + envelope = pubnub.remove_device_from_push() \ + .push_type(PNPushType.APNS2) \ + .device_id(APNS2_DEVICE) \ + .topic("myapptopic") \ + .environment(PNPushEnvironment.DEVELOPMENT) \ + .sync() + # snippet.end + a.ok(not envelope.status.is_error(), 'status is not an error') + expect(pubnub, [], [], 'after remove all (builder)') + + seed(pubnub, CHANNELS) + expect(pubnub, suffixed(CHANNELS), suffixed(CHANNELS), 'after seeding again') + + # snippet.remove_device_from_push_named + # For FCM + envelope = pubnub.remove_device_from_push(device_id="deviceId", push_type=PNPushType.FCM).sync() + + # For APNS2 + envelope = pubnub.remove_device_from_push(device_id=APNS2_DEVICE, + push_type=PNPushType.APNS2, + topic="myapptopic", + environment=PNPushEnvironment.DEVELOPMENT) \ + .sync() + # snippet.end + a.ok(not envelope.status.is_error(), 'status is not an error') + expect(pubnub, [], [], 'after remove all (named)') + finally: + clear(pubnub) diff --git a/docs-snippets/sync/objects.py b/docs-snippets/sync/objects.py new file mode 100644 index 00000000..ab4c5473 --- /dev/null +++ b/docs-snippets/sync/objects.py @@ -0,0 +1,579 @@ +"""Docs page: App Context (sync). User and channel metadata. + +Memberships, members and the two helper classes are in objects_membership.py. +Every object a step creates is removed in a finally block. The janitor also removes leftovers that carry the run +marker and are older than six hours. +""" +from pubnub.callbacks import SubscribeCallback +from pubnub.enums import PNStatusCategory +from pubnub.exceptions import PubNubException +from pubnub.models.consumer.objects_v2.memberships import PNChannelMembership +from pubnub.models.consumer.objects_v2.page import Next, Previous + +from harness import assertions as a +from harness import names +from harness import world +from harness.step import step +from harness.wait import eventually + +REGISTERED_NAMES = [ + 'some_uuid', 'some_uuid_named', 'channel_id', 'channel_id_named', 'doc_type', +] + +UUID_FIELDS = { + 'name': 'Some Name', + 'status': 'Active', + 'type': 'User', + 'email': 'test@example.com', + 'profileUrl': 'http://example.com', + 'externalId': '1234567890', +} + + +class AppContextEvents(SubscribeCallback): + """Collects App Context events. + + The sync SDK stops its message worker with an AttributeError right after it hands the first App Context event + to the listeners (models/subscription.py:314-324). A watcher therefore sees exactly one event, so a step + creates one watcher per change it wants to observe. + """ + + def __init__(self): + self.categories = [] + self.items = [] + + def status(self, pubnub, status): + self.categories.append(status.category) + + def message(self, pubnub, message): + pass + + def presence(self, pubnub, presence): + pass + + def uuid(self, pubnub, uuid): + self.items.append(('uuid', uuid.event, uuid.data)) + + def channel(self, pubnub, channel): + self.items.append(('channel', channel.event, channel.data)) + + def membership(self, pubnub, membership): + self.items.append(('membership', membership.event, membership.data)) + + +def watch(clients, *channels): + """A second client that listens on the given channels. Returns after it is connected.""" + client = clients.sync('open', 'watcher') + seen = AppContextEvents() + client.subscribed_by_suite = True + client.add_listener(seen) + client.subscribe().channels(list(channels)).execute() + eventually(lambda: a.ok(PNStatusCategory.PNConnectedCategory in seen.categories, 'not connected yet'), + timeout=15, what='watcher connected') + return seen + + +def first_event(seen): + eventually(lambda: a.ok(len(seen.items) >= 1, 'no App Context event yet'), timeout=20, what='App Context event') + return seen.items[0] + + +def quiet_404(call): + """Run a cleanup call. A missing object is fine, any other error is not.""" + try: + return call() + except PubNubException as e: + if '404' not in str(e): + raise + return None + + +def cleanup(pubnub, uuids=(), channels=()): + """Remove memberships first, then the metadata objects.""" + for uid in uuids: + found = quiet_404(lambda: pubnub.get_memberships(uuid=uid, limit=100).sync()) + if found and found.result.data: + leftovers = [PNChannelMembership.channel(m['channel']['id']) for m in found.result.data] + quiet_404(lambda: pubnub.remove_memberships(uuid=uid, channel_memberships=leftovers).sync()) + for channel in channels: + quiet_404(lambda: pubnub.remove_channel_metadata(channel=channel).sync()) + for uid in uuids: + quiet_404(lambda: pubnub.remove_uuid_metadata(uuid=uid).sync()) + + +def record(uuids=(), channels=()): + for uid in uuids: + world.record('uuid_metadata', uid) + for channel in channels: + world.record('channel_metadata', channel) + + +def ids(items): + return [item['id'] for item in items] + + +def gone(read): + """The read raises a 404 once the object is removed.""" + eventually(lambda: a.raises(PubNubException, read, match='404'), what='object reports 404') + + +@step('get_all_uuid_metadata_builder', 'get_all_uuid_metadata_named', role='open') +def test_get_all_uuid_metadata(clients): + pubnub = clients.sync('open', 'lister') + kind = names.with_suffix('doc_type') + users = [names.with_suffix('doc_user_' + x) for x in 'abc'] + record(uuids=users) + try: + for rank, uid in enumerate(users): + pubnub.set_uuid_metadata(uuid=uid, name=kind, type='doc', custom={'rank': rank}).sync() + + def listed(page=None, **kwargs): + # the named arguments of get_all_uuid_metadata have no page, only the builder does + builder = pubnub.get_all_uuid_metadata(filter="name == '%s'" % kind, **kwargs) + return (builder.page(page) if page else builder).sync().result + + eventually(lambda: a.eq(ids(listed().data), sorted(users), 'seeded users listed'), what='users listed') + + # snippet.get_all_uuid_metadata_builder + # snippet.hide + from pubnub.models.consumer.objects_v2.sort import PNSortKey, PNSortKeyValue + # snippet.show + result = pubnub.get_all_uuid_metadata() \ + .include_custom(True) \ + .limit(10) \ + .filter("name == 'doc_type'") \ + .include_total_count(True) \ + .sort(PNSortKey.asc(PNSortKeyValue.ID), PNSortKey.desc(PNSortKeyValue.UPDATED)) \ + .page(None) \ + .sync() + + for uuid_data in result.result.data: + print(f"UUID: {uuid_data['id']}") + print(f"Name: {uuid_data['name']}") + print(f"Custom: {uuid_data['custom']}") + # snippet.end + + built = result.result + a.eq(ids(built.data), users, 'ascending by id') + a.eq(built.total_count, 3, 'total count') + a.eq([u['custom'] for u in built.data], [{'rank': 0}, {'rank': 1}, {'rank': 2}]) + a.eq([u['name'] for u in built.data], [kind] * 3) + + # snippet.get_all_uuid_metadata_named + metadata = pubnub.get_all_uuid_metadata( + limit=10, + include_custom=True, + filter="name == 'doc_type'", + include_total_count=True, + sort_keys=[PNSortKey.asc(PNSortKeyValue.ID), PNSortKey.desc(PNSortKeyValue.UPDATED)] + ).sync() + + for uuid_data in metadata.result.data: + print(f"UUID: {uuid_data['id']}") + print(f"Name: {uuid_data['name']}") + print(f"Custom: {uuid_data['custom']}") + # snippet.end + + a.eq(ids(metadata.result.data), users, 'named arguments: same list') + a.eq(metadata.result.total_count, 3) + a.eq([u['custom'] for u in metadata.result.data], [{'rank': 0}, {'rank': 1}, {'rank': 2}]) + + # Defaults: custom and the total count are omitted, status and type are returned. + plain = listed() + a.not_contains(plain.data[0], 'custom', 'custom only when included') + a.eq(plain.data[0]['type'], 'doc') + a.contains(plain.data[0], 'status') + a.eq(plain.total_count, None, 'total count only when requested') + a.not_contains(listed(include_type=False).data[0], 'type', 'type can be left out') + a.not_contains(listed(include_status=False).data[0], 'status', 'status can be left out') + a.eq(ids(listed(sort_keys=[PNSortKey.desc(PNSortKeyValue.ID)]).data), list(reversed(users)), 'descending') + only = pubnub.get_all_uuid_metadata(filter="name == '%s' && id == '%s'" % (kind, users[1])).sync() + a.eq(ids(only.result.data), [users[1]], 'filter on name and id') + + # Paging with next and prev + ordered = [PNSortKey.asc(PNSortKeyValue.ID)] + first = listed(limit=2, sort_keys=ordered) + a.eq(ids(first.data), users[:2]) + a.ok(isinstance(first.next, Next), 'next page token') + a.eq(first.prev, None) + second = listed(limit=2, sort_keys=ordered, page=first.next) + a.eq(ids(second.data), users[2:]) + a.ok(isinstance(second.prev, Previous), 'prev page token') + back = listed(limit=2, sort_keys=ordered, page=second.prev) + a.eq(ids(back.data), users[:2], 'prev returns the first page') + finally: + cleanup(pubnub, users) + + +@step('get_uuid_metadata_builder', 'get_uuid_metadata_named', role='open') +def test_get_uuid_metadata(clients): + pubnub = clients.sync('open', 'profile') + me = pubnub.uuid + other = names.with_suffix('doc_other') + record(uuids=[me, other]) + try: + pubnub.set_uuid_metadata(uuid=me, name='Doc profile', status='Active', type='doc', + custom={'team': 'docs'}).sync() + pubnub.set_uuid_metadata(uuid=other, name='Doc other', custom={'team': 'other'}).sync() + + # snippet.get_uuid_metadata_builder + envelope = pubnub.get_uuid_metadata() \ + .include_custom(True) \ + .sync() + # snippet.end + + data = envelope.result.data + a.eq(data['id'], me, 'defaults to the configured user id') + a.eq((data['name'], data['status'], data['type']), ('Doc profile', 'Active', 'doc')) + a.eq(data['custom'], {'team': 'docs'}) + a.eq(envelope.result.status, 200) + + # snippet.get_uuid_metadata_named + metadata = pubnub.get_uuid_metadata(include_custom=True).sync() + # snippet.end + + a.eq(metadata.result.data, data, 'named arguments: same object') + + plain = pubnub.get_uuid_metadata().sync().result.data + a.not_contains(plain, 'custom', 'custom only when included') + a.not_contains(pubnub.get_uuid_metadata(include_type=False).sync().result.data, 'type') + a.not_contains(pubnub.get_uuid_metadata(include_status=False).sync().result.data, 'status') + explicit = pubnub.get_uuid_metadata(uuid=other, include_custom=True).sync().result.data + a.eq((explicit['id'], explicit['custom']), (other, {'team': 'other'}), 'explicit uuid') + missing = names.with_suffix('doc_missing') + e = a.raises(PubNubException, lambda: pubnub.get_uuid_metadata(uuid=missing).sync(), match='404') + a.contains(str(e), 'not found') + finally: + cleanup(pubnub, [me, other]) + + +@step('set_uuid_metadata_builder', 'set_uuid_metadata_named', role='open', timeout=180) +def test_set_uuid_metadata(clients): + pubnub = clients.sync('open', 'setter') + first = names.with_suffix('some_uuid') + second = names.with_suffix('some_uuid_named') + record(uuids=[first, second]) + try: + seen = watch(clients, first) + + # snippet.set_uuid_metadata_builder + envelope = pubnub.set_uuid_metadata() \ + .include_custom(True) \ + .uuid("some_uuid") \ + .set_name("Some Name") \ + .set_status("Active") \ + .set_type("User") \ + .email("test@example.com") \ + .profile_url("http://example.com") \ + .external_id("1234567890") \ + .custom({"key1": "val1", "key2": "val2"}) \ + .sync() + # snippet.end + + data = envelope.result.data + a.eq({k: data[k] for k in UUID_FIELDS}, UUID_FIELDS) + a.eq((data['id'], data['custom']), (first, {'key1': 'val1', 'key2': 'val2'})) + a.ok(data['eTag'] and data['updated'], 'eTag and updated are returned') + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['id'], payload['name']), ('uuid', 'set', first, 'Some Name')) + a.eq(payload['custom'], {'key1': 'val1', 'key2': 'val2'}) + a.eq(payload['eTag'], data['eTag']) + read = pubnub.get_uuid_metadata(uuid=first, include_custom=True).sync().result.data + a.eq(read, data, 'read back equals the set response') + + seen = watch(clients, second) + + # snippet.set_uuid_metadata_named + envelope = pubnub.set_uuid_metadata(uuid="some_uuid_named", + name="Some Name", + status="Active", type="User", + email="test@example.com", + profile_url="http://example.com", + external_id="1234567890", + custom={"key1": "val1", "key2": "val2"}, + include_custom=True) \ + .sync() + # snippet.end + + named = envelope.result.data + a.eq({k: named[k] for k in UUID_FIELDS}, UUID_FIELDS) + a.eq((named['id'], named['custom']), (second, {'key1': 'val1', 'key2': 'val2'})) + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['id']), ('uuid', 'set', second)) + + # Custom is replaced as a whole, never merged + replaced = pubnub.set_uuid_metadata(uuid=first, custom={'only': 'this'}, include_custom=True).sync().result.data + a.eq(replaced['custom'], {'only': 'this'}) + read = pubnub.get_uuid_metadata(uuid=first, include_custom=True).sync().result.data + a.eq(read['custom'], {'only': 'this'}, 'custom is overwritten') + + # Defaults of the include flags: status and type come back unless they are switched off + light = pubnub.set_uuid_metadata(uuid=first, name='Some Name', include_status=False, include_type=False) + light_data = light.sync().result.data + a.not_contains(light_data, 'status') + a.not_contains(light_data, 'type') + a.not_contains(light_data, 'custom', 'custom only when included') + + # Setting the same data again triggers no event. The first event the watcher sees is the later change. + fields = dict(name='Some Name', status='Active', type='User', custom={'only': 'this'}) + pubnub.set_uuid_metadata(uuid=first, **fields).sync() + same = watch(clients, first) + pubnub.set_uuid_metadata(uuid=first, **fields).sync() + fields['name'] = 'Renamed' + pubnub.set_uuid_metadata(uuid=first, **fields).sync() + kind, event, payload = first_event(same) + a.eq((event, payload['name']), ('set', 'Renamed'), 'unchanged data raised no event') + finally: + cleanup(pubnub, [first, second]) + + +@step('remove_uuid_metadata_builder', 'remove_uuid_metadata_named', role='open') +def test_remove_uuid_metadata(clients): + pubnub = clients.sync('open', 'remover') + first = names.with_suffix('some_uuid') + second = names.with_suffix('some_uuid_named') + record(uuids=[first, second]) + try: + for uid in (first, second): + pubnub.set_uuid_metadata(uuid=uid, name='To remove', custom={'k': 'v'}).sync() + a.eq(pubnub.get_uuid_metadata(uuid=first, include_custom=True).sync().result.data['custom'], {'k': 'v'}) + seen = watch(clients, first) + + # snippet.remove_uuid_metadata_builder + envelope = pubnub.remove_uuid_metadata() \ + .uuid("some_uuid").sync() + # snippet.end + + a.eq(envelope.result.status, 200) + a.eq(envelope.result.data, None, 'nothing is returned') + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['id']), ('uuid', 'delete', first)) + gone(lambda: pubnub.get_uuid_metadata(uuid=first).sync()) + a.eq(pubnub.get_uuid_metadata(uuid=second).sync().result.data['id'], second, 'other users stay') + + # snippet.remove_uuid_metadata_named + envelope = pubnub.remove_uuid_metadata(uuid="some_uuid_named").sync() + # snippet.end + + a.eq(envelope.result.status, 200) + gone(lambda: pubnub.get_uuid_metadata(uuid=second).sync()) + listed = pubnub.get_all_uuid_metadata(filter="id == '%s'" % second).sync().result.data + a.eq(listed, [], 'the list no longer has the user') + finally: + cleanup(pubnub, [first, second]) + + +@step('get_all_channel_metadata_builder', 'get_all_channel_metadata_named', role='open') +def test_get_all_channel_metadata(clients): + pubnub = clients.sync('open', 'lister') + kind = names.with_suffix('doc_type') + channels = [names.with_suffix('doc_channel_' + x) for x in 'abc'] + record(channels=channels) + try: + for rank, channel in enumerate(channels): + pubnub.set_channel_metadata(channel=channel, name=kind, type='doc', description='Channel %d' % rank, + custom={'rank': rank}).sync() + + def listed(**kwargs): + return pubnub.get_all_channel_metadata(filter="name == '%s'" % kind, **kwargs).sync().result + + eventually(lambda: a.eq(ids(listed().data), sorted(channels), 'seeded channels listed'), what='channels listed') + + # snippet.get_all_channel_metadata_builder + # snippet.hide + from pubnub.models.consumer.objects_v2.sort import PNSortKey, PNSortKeyValue + # snippet.show + envelope = pubnub.get_all_channel_metadata() \ + .include_custom(True) \ + .limit(10) \ + .filter("name == 'doc_type'") \ + .include_total_count(True) \ + .sort(PNSortKey.asc(PNSortKeyValue.ID), PNSortKey.desc(PNSortKeyValue.UPDATED)) \ + .page(None) \ + .sync() + # snippet.end + + built = envelope.result + a.eq(ids(built.data), channels, 'ascending by id') + a.eq(built.total_count, 3) + a.eq([c['custom'] for c in built.data], [{'rank': 0}, {'rank': 1}, {'rank': 2}]) + a.eq([c['description'] for c in built.data], ['Channel 0', 'Channel 1', 'Channel 2']) + + # snippet.get_all_channel_metadata_named + metadata = pubnub.get_all_channel_metadata(limit=10, + include_custom=True, + filter="name == 'doc_type'", + include_total_count=True, + sort_keys=[PNSortKey.asc(PNSortKeyValue.ID), + PNSortKey.desc(PNSortKeyValue.UPDATED)]) \ + .sync() + # snippet.end + + a.eq(ids(metadata.result.data), channels, 'named arguments: same list') + a.eq(metadata.result.total_count, 3) + + plain = listed() + a.not_contains(plain.data[0], 'custom', 'custom only when included') + a.eq(plain.data[0]['type'], 'doc') + a.eq(plain.total_count, None) + a.not_contains(listed(include_type=False).data[0], 'type') + a.not_contains(listed(include_status=False).data[0], 'status') + only = pubnub.get_all_channel_metadata(filter="name == '%s' && id == '%s'" % (kind, channels[2])).sync() + a.eq(ids(only.result.data), [channels[2]], 'filter on name and id') + + ordered = [PNSortKey.asc(PNSortKeyValue.ID)] + first = listed(limit=2, sort_keys=ordered) + a.eq(ids(first.data), channels[:2]) + a.ok(isinstance(first.next, Next), 'next page token') + second = listed(limit=2, sort_keys=ordered, page=first.next) + a.eq(ids(second.data), channels[2:]) + a.ok(isinstance(second.prev, Previous), 'prev page token') + a.eq(ids(listed(limit=2, sort_keys=ordered, page=second.prev).data), channels[:2]) + finally: + cleanup(pubnub, channels=channels) + + +@step('get_channel_metadata_builder', 'get_channel_metadata_named', role='open') +def test_get_channel_metadata(clients): + pubnub = clients.sync('open', 'reader') + channel = names.with_suffix('channel_id') + record(channels=[channel]) + try: + pubnub.set_channel_metadata(channel=channel, name='Doc channel', description='About', status='live', + type='room', custom={'topic': 'docs'}).sync() + + # snippet.get_channel_metadata_builder + envelope = pubnub.get_channel_metadata() \ + .include_custom(True) \ + .channel("channel_id") \ + .sync() + # snippet.end + + data = envelope.result.data + a.eq((data['id'], data['name'], data['description']), (channel, 'Doc channel', 'About')) + a.eq((data['status'], data['type'], data['custom']), ('live', 'room', {'topic': 'docs'})) + a.eq(envelope.result.status, 200) + + # snippet.get_channel_metadata_named + metadata = pubnub.get_channel_metadata(channel="channel_id", include_custom=True).sync() + # snippet.end + + a.eq(metadata.result.data, data, 'named arguments: same object') + a.not_contains(pubnub.get_channel_metadata(channel=channel).sync().result.data, 'custom') + a.not_contains(pubnub.get_channel_metadata(channel=channel, include_type=False).sync().result.data, 'type') + a.not_contains(pubnub.get_channel_metadata(channel=channel, include_status=False).sync().result.data, 'status') + missing = names.with_suffix('doc_missing_channel') + a.raises(PubNubException, lambda: pubnub.get_channel_metadata(channel=missing).sync(), match='404') + finally: + cleanup(pubnub, channels=[channel]) + + +@step('set_channel_metadata_builder', 'set_channel_metadata_named', role='open', timeout=180) +def test_set_channel_metadata(clients): + pubnub = clients.sync('open', 'setter') + first = names.with_suffix('channel_id') + second = names.with_suffix('channel_id_named') + record(channels=[first, second]) + fields = {'name': 'Channel Name', 'description': 'Description', 'status': 'Archived', 'type': 'Archived'} + try: + seen = watch(clients, first) + + # snippet.set_channel_metadata_builder + envelope = pubnub.set_channel_metadata() \ + .include_custom(True) \ + .channel("channel_id") \ + .set_name("Channel Name") \ + .set_status("Archived") \ + .set_type("Archived") \ + .description("Description") \ + .custom({"key1": "val1", "key2": "val2"}) \ + .sync() + # snippet.end + + data = envelope.result.data + a.eq({k: data[k] for k in fields}, fields) + a.eq((data['id'], data['custom']), (first, {'key1': 'val1', 'key2': 'val2'})) + a.ok(data['eTag'] and data['updated'], 'eTag and updated are returned') + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['id'], payload['name']), ('channel', 'set', first, 'Channel Name')) + a.eq(payload['custom'], {'key1': 'val1', 'key2': 'val2'}) + read = pubnub.get_channel_metadata(channel=first, include_custom=True).sync().result.data + a.eq(read, data, 'read back equals the set response') + + seen = watch(clients, second) + + # snippet.set_channel_metadata_named + envelope = pubnub.set_channel_metadata(channel="channel_id_named", + name="Channel Name", + status="Archived", + type="Archived", + description="Description", + custom={"key1": "val1", "key2": "val2"}, + include_custom=True) \ + .sync() + # snippet.end + + named = envelope.result.data + a.eq({k: named[k] for k in fields}, fields) + a.eq((named['id'], named['custom']), (second, {'key1': 'val1', 'key2': 'val2'})) + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['id']), ('channel', 'set', second)) + + replaced = pubnub.set_channel_metadata(channel=first, custom={'only': 'this'}, + include_custom=True).sync().result.data + a.eq(replaced['custom'], {'only': 'this'}) + read = pubnub.get_channel_metadata(channel=first, include_custom=True).sync().result.data + a.eq(read['custom'], {'only': 'this'}, 'custom is overwritten') + light = pubnub.set_channel_metadata(channel=first, name='Channel Name', include_status=False, + include_type=False).sync().result.data + a.not_contains(light, 'status') + a.not_contains(light, 'type') + a.not_contains(light, 'custom', 'custom only when included') + + keep = dict(name='Channel Name', description='Description', status='Archived', type='Archived', + custom={'only': 'this'}) + pubnub.set_channel_metadata(channel=first, **keep).sync() + same = watch(clients, first) + pubnub.set_channel_metadata(channel=first, **keep).sync() + keep['description'] = 'Changed' + pubnub.set_channel_metadata(channel=first, **keep).sync() + kind, event, payload = first_event(same) + a.eq((event, payload['description']), ('set', 'Changed'), 'unchanged data raised no event') + finally: + cleanup(pubnub, channels=[first, second]) + + +@step('remove_channel_metadata_builder', 'remove_channel_metadata_named', role='open') +def test_remove_channel_metadata(clients): + pubnub = clients.sync('open', 'remover') + first = names.with_suffix('channel_id') + second = names.with_suffix('channel_id_named') + record(channels=[first, second]) + try: + for channel in (first, second): + pubnub.set_channel_metadata(channel=channel, name='To remove').sync() + seen = watch(clients, first) + + # snippet.remove_channel_metadata_builder + envelope = pubnub.remove_channel_metadata() \ + .channel("channel_id") \ + .sync() + # snippet.end + + a.eq(envelope.result.status, 200) + a.eq(envelope.result.data, None, 'nothing is returned') + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['id']), ('channel', 'delete', first)) + gone(lambda: pubnub.get_channel_metadata(channel=first).sync()) + a.eq(pubnub.get_channel_metadata(channel=second).sync().result.data['id'], second, 'other channels stay') + + # snippet.remove_channel_metadata_named + envelope = pubnub.remove_channel_metadata(channel="channel_id_named").sync() + # snippet.end + + a.eq(envelope.result.status, 200) + gone(lambda: pubnub.get_channel_metadata(channel=second).sync()) + listed = pubnub.get_all_channel_metadata(filter="id == '%s'" % second).sync().result.data + a.eq(listed, [], 'the list no longer has the channel') + finally: + cleanup(pubnub, channels=[first, second]) diff --git a/docs-snippets/sync/objects_membership.py b/docs-snippets/sync/objects_membership.py new file mode 100644 index 00000000..825ead9f --- /dev/null +++ b/docs-snippets/sync/objects_membership.py @@ -0,0 +1,803 @@ +"""Docs page: App Context (sync). Channel memberships, channel members and the PNChannelMembership and PNUUID classes. + +User and channel metadata are in objects.py. The helpers below repeat the ones in that module because a page module +cannot import a sibling file. Every object a step creates is removed in a finally block. +""" +from pubnub.callbacks import SubscribeCallback +from pubnub.enums import PNStatusCategory +from pubnub.exceptions import PubNubException +from pubnub.models.consumer.objects_v2.channel_members import PNUUID +from pubnub.models.consumer.objects_v2.common import MemberIncludes, MembershipIncludes +from pubnub.models.consumer.objects_v2.memberships import PNChannelMembership +from pubnub.models.consumer.objects_v2.page import Next, Previous +from pubnub.models.consumer.objects_v2.sort import PNSortKey, PNSortKeyValue + +from harness import assertions as a +from harness import names +from harness import world +from harness.step import step +from harness.wait import eventually + +REGISTERED_NAMES = [ + 'some_uuid', 'some_uuid_with_custom', 'some_uuid_named', 'somechannel', 'somechannel_with_custom', + 'channel_id', 'channel_id_named', +] + +CUSTOM_1 = {'key3': 'val1', 'key4': 'val2'} +CUSTOM_2 = {'key5': 'val1', 'key6': 'val2'} + + +class AppContextEvents(SubscribeCallback): + """Collects App Context events. + + The sync SDK stops its message worker with an AttributeError right after it hands the first App Context event + to the listeners (models/subscription.py:314-324). A watcher therefore sees exactly one event, so a step + creates one watcher per change it wants to observe. + """ + + def __init__(self): + self.categories = [] + self.items = [] + + def status(self, pubnub, status): + self.categories.append(status.category) + + def message(self, pubnub, message): + pass + + def presence(self, pubnub, presence): + pass + + def uuid(self, pubnub, uuid): + self.items.append(('uuid', uuid.event, uuid.data)) + + def channel(self, pubnub, channel): + self.items.append(('channel', channel.event, channel.data)) + + def membership(self, pubnub, membership): + self.items.append(('membership', membership.event, membership.data)) + + +def watch(clients, *channels): + """A second client that listens on the given channels. Returns after it is connected.""" + client = clients.sync('open', 'watcher') + seen = AppContextEvents() + client.subscribed_by_suite = True + client.add_listener(seen) + client.subscribe().channels(list(channels)).execute() + eventually(lambda: a.ok(PNStatusCategory.PNConnectedCategory in seen.categories, 'not connected yet'), + timeout=15, what='watcher connected') + return seen + + +def first_event(seen): + eventually(lambda: a.ok(len(seen.items) >= 1, 'no App Context event yet'), timeout=20, what='App Context event') + return seen.items[0] + + +def quiet_404(call): + """Run a cleanup call. A missing object is fine, any other error is not.""" + try: + return call() + except PubNubException as e: + if '404' not in str(e): + raise + return None + + +def cleanup(pubnub, uuids=(), channels=()): + """Remove memberships first, then the metadata objects.""" + for uid in uuids: + found = quiet_404(lambda: pubnub.get_memberships(uuid=uid, limit=100).sync()) + if found and found.result.data: + leftovers = [PNChannelMembership.channel(m['channel']['id']) for m in found.result.data] + quiet_404(lambda: pubnub.remove_memberships(uuid=uid, channel_memberships=leftovers).sync()) + for channel in channels: + quiet_404(lambda: pubnub.remove_channel_metadata(channel=channel).sync()) + for uid in uuids: + quiet_404(lambda: pubnub.remove_uuid_metadata(uuid=uid).sync()) + + +def record(uuids=(), channels=()): + for uid in uuids: + world.record('uuid_metadata', uid) + for channel in channels: + world.record('channel_metadata', channel) + + +def channel_ids(items): + return [item['channel']['id'] for item in items] + + +def member_ids(items): + return [item['uuid']['id'] for item in items] + + +def by_channel(items): + return {item['channel']['id']: item for item in items} + + +def by_member(items): + return {item['uuid']['id']: item for item in items} + + +def seed_channels(pubnub, plain, with_custom): + pubnub.set_channel_metadata(channel=plain, name='some name').sync() + pubnub.set_channel_metadata(channel=with_custom, name='some name with custom', custom=CUSTOM_1).sync() + + +def seed_users(pubnub, plain, with_custom): + pubnub.set_uuid_metadata(uuid=plain, name='some name').sync() + pubnub.set_uuid_metadata(uuid=with_custom, name='some name with custom', custom=CUSTOM_1).sync() + + +def seed_memberships(pubnub, uid, plain, with_custom): + pubnub.set_memberships(uuid=uid, channel_memberships=[ + PNChannelMembership.channel(plain), PNChannelMembership.channel_with_custom(with_custom, CUSTOM_2)]).sync() + + +def seed_one_membership(pubnub, uid, with_custom): + pubnub.set_memberships(uuid=uid, channel_memberships=[ + PNChannelMembership.channel_with_custom(with_custom, CUSTOM_2)]).sync() + + +def seed_one_member(pubnub, channel, with_custom): + pubnub.set_channel_members(channel=channel, uuids=[PNUUID.uuid_with_custom(with_custom, CUSTOM_2)]).sync() + + +def seed_members(pubnub, channel, plain, with_custom): + pubnub.set_channel_members(channel=channel, uuids=[ + PNUUID.uuid(plain), PNUUID.uuid_with_custom(with_custom, CUSTOM_2)]).sync() + + +def gone(read): + """The read raises a 404 once the object is removed.""" + eventually(lambda: a.raises(PubNubException, read, match='404'), what='object reports 404') + + +def check_memberships(items, plain, with_custom): + """Both memberships with the details MembershipIncludes(custom, channel, channel_custom) asks for.""" + a.eq(sorted(channel_ids(items)), sorted([plain, with_custom])) + found = by_channel(items) + a.eq(found[plain]['custom'], None, 'plain membership has no custom') + a.eq(found[with_custom]['custom'], CUSTOM_2, 'membership custom') + a.eq(found[plain]['channel']['name'], 'some name') + a.eq(found[with_custom]['channel']['custom'], CUSTOM_1, 'channel custom') + a.eq(found[with_custom]['channel']['name'], 'some name with custom') + a.ok(found[plain]['eTag'] and found[plain]['updated'], 'eTag and updated are returned') + + +def check_members(items, plain, with_custom): + a.eq(sorted(member_ids(items)), sorted([plain, with_custom])) + found = by_member(items) + a.eq(found[plain]['custom'], None, 'plain member has no custom') + a.eq(found[with_custom]['custom'], CUSTOM_2, 'member custom') + a.eq(found[plain]['uuid']['name'], 'some name') + a.eq(found[with_custom]['uuid']['custom'], CUSTOM_1, 'user custom') + a.eq(found[with_custom]['uuid']['name'], 'some name with custom') + + +@step('get_memberships_builder', 'get_memberships_named', role='open') +def test_get_memberships(clients): + pubnub = clients.sync('open', 'reader') + uid = names.with_suffix('some_uuid') + plain = names.with_suffix('somechannel') + with_custom = names.with_suffix('somechannel_with_custom') + record([uid], [plain, with_custom]) + try: + seed_channels(pubnub, plain, with_custom) + seed_memberships(pubnub, uid, plain, with_custom) + + # snippet.get_memberships_builder + # snippet.hide + from pubnub.models.consumer.objects_v2.common import MembershipIncludes + # snippet.show + envelope = pubnub.get_memberships() \ + .include(MembershipIncludes(custom=True, channel=True, channel_custom=True)) \ + .uuid("some_uuid").sync() + # snippet.end + + check_memberships(envelope.result.data, plain, with_custom) + a.eq(envelope.result.status, 200) + + # snippet.get_memberships_named + envelope = pubnub.get_memberships(uuid="some_uuid", + include=MembershipIncludes(custom=True, channel=True, channel_custom=True)) \ + .sync() + # snippet.end + + check_memberships(envelope.result.data, plain, with_custom) + + # Details appear only when they are included + bare = pubnub.get_memberships(uuid=uid).sync().result.data + a.not_contains(bare[0], 'custom', 'custom only when included') + a.eq(bare[0]['channel'], {'id': bare[0]['channel']['id']}, 'channel is only an id by default') + only_custom = pubnub.get_memberships(uuid=uid, include=MembershipIncludes(custom=True)).sync().result.data + a.contains(only_custom[0], 'custom') + a.eq(by_channel(only_custom)[with_custom]['custom'], CUSTOM_2) + a.not_contains(bare[0], 'type') + a.not_contains(bare[0], 'status') + typed = PNChannelMembership(plain, status='active', type='doc') + pubnub.set_memberships(uuid=uid, channel_memberships=[typed]).sync() + shown = pubnub.get_memberships(uuid=uid, include=MembershipIncludes(type=True, status=True)).sync() + a.eq((by_channel(shown.result.data)[plain]['type'], by_channel(shown.result.data)[plain]['status']), + ('doc', 'active')) + + filtered = pubnub.get_memberships(uuid=uid, filter="channel.name LIKE 'some name with*'").sync().result.data + a.eq(channel_ids(filtered), [with_custom], 'filter on the channel name') + counted = pubnub.get_memberships(uuid=uid).include_total_count(True).sync().result + a.eq(counted.total_count, 2, 'total count') + + # Paging with next and prev, sorted by the update time (the only sort key the service accepts here) + newest = [PNSortKey.desc(PNSortKeyValue.UPDATED)] + first = pubnub.get_memberships(uuid=uid, limit=1, sort_keys=newest).sync().result + a.eq(len(first.data), 1) + a.ok(isinstance(first.next, Next), 'next page token') + second = pubnub.get_memberships(uuid=uid, limit=1, sort_keys=newest, page=first.next).sync().result + a.eq(sorted(channel_ids(first.data + second.data)), sorted([plain, with_custom]), 'two pages cover both') + a.ok(isinstance(second.prev, Previous), 'prev page token') + back = pubnub.get_memberships(uuid=uid, limit=1, sort_keys=newest, page=second.prev).sync().result + a.eq(channel_ids(back.data), channel_ids(first.data), 'prev returns the first page') + finally: + cleanup(pubnub, [uid], [plain, with_custom]) + + +@step('set_memberships_builder', 'set_memberships_named', role='open', timeout=180) +def test_set_memberships(clients): + pubnub = clients.sync('open', 'setter') + first = names.with_suffix('some_uuid') + second = names.with_suffix('some_uuid_named') + plain = names.with_suffix('somechannel') + with_custom = names.with_suffix('somechannel_with_custom') + record([first, second], [plain, with_custom]) + try: + seen = watch(clients, first) + + # snippet.set_memberships_builder + # snippet.hide + from pubnub.models.consumer.objects_v2.common import MembershipIncludes + from pubnub.models.consumer.objects_v2.memberships import PNChannelMembership + # snippet.show + some_channel = "somechannel" + some_channel_with_custom = "somechannel_with_custom" + + pubnub.set_channel_metadata() \ + .channel(some_channel) \ + .set_name("some name") \ + .sync() + + custom_1 = { + "key3": "val1", + "key4": "val2", + } + + pubnub.set_channel_metadata() \ + .channel(some_channel_with_custom) \ + .set_name("some name with custom") \ + .custom(custom_1) \ + .sync() + + custom_2 = { + "key5": "val1", + "key6": "val2", + } + + channel_memberships_to_set = [ + PNChannelMembership.channel(some_channel), + PNChannelMembership.channel_with_custom(some_channel_with_custom, custom_2), + ] + + envelope = pubnub.set_memberships() \ + .uuid("some_uuid") \ + .channel_memberships(channel_memberships_to_set) \ + .include(MembershipIncludes(custom=True, channel=True, channel_custom=True)) \ + .sync() + # snippet.end + + check_memberships(envelope.result.data, plain, with_custom) + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['uuid']['id']), ('membership', 'set', first)) + a.contains([plain, with_custom], payload['channel']['id']) + + seen = watch(clients, second) + + # snippet.set_memberships_named + some_channel = "somechannel" + some_channel_with_custom = "somechannel_with_custom" + + pubnub.set_channel_metadata(channel=some_channel, name="some name").sync() + + custom_1 = { + "key3": "val1", + "key4": "val2", + } + + pubnub.set_channel_metadata(channel=some_channel_with_custom, + name="some name with custom", + custom=custom_1) \ + .sync() + + custom_2 = { + "key5": "val1", + "key6": "val2", + } + + channel_memberships_to_set = [ + PNChannelMembership.channel(some_channel), + PNChannelMembership.channel_with_custom(some_channel_with_custom, custom_2), + ] + + envelope = pubnub.set_memberships(uuid="some_uuid_named", + channel_memberships=channel_memberships_to_set, + include=MembershipIncludes(custom=True, channel=True, channel_custom=True)) \ + .sync() + # snippet.end + + check_memberships(envelope.result.data, plain, with_custom) + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['uuid']['id']), ('membership', 'set', second)) + + # The channel side sees the same memberships + members = pubnub.get_channel_members(channel=plain).sync().result.data + a.eq(sorted(member_ids(members)), sorted([first, second]), 'both users are members') + + # Set adds to the stored memberships, it does not replace them + extra = names.with_suffix('somechannel_extra') + record(channels=[extra]) + try: + pubnub.set_channel_metadata(channel=extra, name='extra').sync() + added = pubnub.set_memberships(uuid=first, channel_memberships=[PNChannelMembership.channel(extra)]).sync() + a.eq(sorted(channel_ids(added.result.data)), sorted([plain, with_custom, extra]), 'set keeps the others') + finally: + cleanup(pubnub, [first], [extra]) + finally: + cleanup(pubnub, [first, second], [plain, with_custom]) + + +@step('remove_memberships_builder', 'remove_memberships_named', role='open', timeout=180) +def test_remove_memberships(clients): + pubnub = clients.sync('open', 'remover') + first = names.with_suffix('some_uuid') + second = names.with_suffix('some_uuid_named') + plain = names.with_suffix('somechannel') + with_custom = names.with_suffix('somechannel_with_custom') + record([first, second], [plain, with_custom]) + try: + seed_channels(pubnub, plain, with_custom) + for uid in (first, second): + seed_memberships(pubnub, uid, plain, with_custom) + seen = watch(clients, first) + + # snippet.remove_memberships_builder + # snippet.hide + from pubnub.endpoints.objects_v2.objects_endpoint import ChannelIncludeEndpoint + from pubnub.models.consumer.objects_v2.memberships import PNChannelMembership + # snippet.show + some_channel = "somechannel" + + envelope = pubnub.remove_memberships() \ + .uuid("some_uuid") \ + .channel_memberships([PNChannelMembership.channel(some_channel)]) \ + .include_custom(True) \ + .include_channel(ChannelIncludeEndpoint.CHANNEL_WITH_CUSTOM) \ + .sync() + # snippet.end + + left = envelope.result.data + a.eq(channel_ids(left), [with_custom], 'the response lists the remaining memberships') + a.eq(left[0]['custom'], CUSTOM_2) + a.eq(left[0]['channel']['custom'], CUSTOM_1) + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['uuid']['id'], payload['channel']['id']), ('membership', 'delete', first, plain)) + eventually(lambda: a.eq(channel_ids(pubnub.get_memberships(uuid=first).sync().result.data), [with_custom]), + what='membership removed') + a.eq(len(pubnub.get_memberships(uuid=second).sync().result.data), 2, 'other users keep theirs') + + # snippet.remove_memberships_named + envelope = pubnub.remove_memberships(uuid="some_uuid_named", + channel_memberships=[PNChannelMembership.channel(some_channel)], + include=MembershipIncludes(custom=True, channel=True, + channel_custom=True)) \ + .sync() + # snippet.end + + a.eq(channel_ids(envelope.result.data), [with_custom]) + a.eq(envelope.result.data[0]['channel']['custom'], CUSTOM_1) + eventually(lambda: a.eq(channel_ids(pubnub.get_memberships(uuid=second).sync().result.data), [with_custom]), + what='second membership removed') + a.eq(pubnub.get_channel_metadata(channel=plain).sync().result.data['id'], plain, 'the channel stays') + finally: + cleanup(pubnub, [first, second], [plain, with_custom]) + + +@step('manage_memberships_builder', 'manage_memberships_named', role='open', timeout=180) +def test_manage_memberships(clients): + pubnub = clients.sync('open', 'manager') + first = names.with_suffix('some_uuid') + second = names.with_suffix('some_uuid_named') + plain = names.with_suffix('somechannel') + with_custom = names.with_suffix('somechannel_with_custom') + record([first, second], [plain, with_custom]) + try: + seed_channels(pubnub, plain, with_custom) + for uid in (first, second): + seed_one_membership(pubnub, uid, with_custom) + eventually(lambda: a.eq(channel_ids(pubnub.get_memberships(uuid=second).sync().result.data), [with_custom]), + what='seed visible') + seen = watch(clients, first) + + # snippet.manage_memberships_builder + # snippet.hide + from pubnub.models.consumer.objects_v2.common import MembershipIncludes + from pubnub.models.consumer.objects_v2.memberships import PNChannelMembership + # snippet.show + some_channel = "somechannel" + some_channel_with_custom = "somechannel_with_custom" + + envelope = pubnub.manage_memberships() \ + .uuid("some_uuid") \ + .set([PNChannelMembership.channel(some_channel)]) \ + .remove([PNChannelMembership.channel(some_channel_with_custom)]) \ + .include(MembershipIncludes(custom=True, channel=True, channel_custom=True)) \ + .sync() + # snippet.end + + a.eq(channel_ids(envelope.result.data), [plain], 'the response lists the remaining memberships') + a.eq(envelope.result.data[0]['channel']['name'], 'some name') + kind, event, payload = first_event(seen) + a.eq(kind, 'membership') + a.contains([('set', plain), ('delete', with_custom)], (event, payload['channel']['id'])) + eventually(lambda: a.eq(channel_ids(pubnub.get_memberships(uuid=first).sync().result.data), [plain]), + what='memberships changed') + + # snippet.manage_memberships_named + envelope = pubnub.manage_memberships(uuid="some_uuid_named", + channel_memberships_to_set=[PNChannelMembership.channel(some_channel)], + channel_memberships_to_remove=[ + PNChannelMembership.channel(some_channel_with_custom)], + include=MembershipIncludes(custom=True, channel=True, + channel_custom=True)) \ + .sync() + # snippet.end + + a.eq(channel_ids(envelope.result.data), [plain]) + eventually(lambda: a.eq(channel_ids(pubnub.get_memberships(uuid=second).sync().result.data), [plain]), + what='second user changed') + finally: + cleanup(pubnub, [first, second], [plain, with_custom]) + + +@step('get_channel_members_builder', 'get_channel_members_named', role='open') +def test_get_channel_members(clients): + pubnub = clients.sync('open', 'reader') + channel = names.with_suffix('channel_id') + plain = names.with_suffix('some_uuid') + with_custom = names.with_suffix('some_uuid_with_custom') + record([plain, with_custom], [channel]) + try: + seed_users(pubnub, plain, with_custom) + pubnub.set_channel_metadata(channel=channel, name='Doc channel').sync() + seed_members(pubnub, channel, plain, with_custom) + + # snippet.get_channel_members_builder + # snippet.hide + from pubnub.models.consumer.objects_v2.common import MemberIncludes + # snippet.show + envelope = pubnub.get_channel_members() \ + .channel("channel_id") \ + .include(MemberIncludes(custom=True, user=True, user_custom=True)) \ + .sync() + # snippet.end + + check_members(envelope.result.data, plain, with_custom) + a.eq(envelope.result.status, 200) + + # snippet.get_channel_members_named + envelope = pubnub.get_channel_members(channel="channel_id", + include=MemberIncludes(custom=True, user=True, user_custom=True)) \ + .sync() + # snippet.end + + check_members(envelope.result.data, plain, with_custom) + + bare = pubnub.get_channel_members(channel=channel).sync().result.data + a.not_contains(bare[0], 'custom', 'custom only when included') + a.eq(bare[0]['uuid'], {'id': bare[0]['uuid']['id']}, 'the user is only an id by default') + only_custom = pubnub.get_channel_members(channel=channel, include=MemberIncludes(custom=True)).sync() + a.eq(by_member(only_custom.result.data)[with_custom]['custom'], CUSTOM_2) + + # Member filters address the user as uuid.name, uuid.description is not a filter property + filtered = pubnub.get_channel_members(channel=channel, filter="uuid.name LIKE 'some name with*'").sync() + a.eq(member_ids(filtered.result.data), [with_custom], 'filter on the user name') + counted = pubnub.get_channel_members(channel=channel).include_total_count(True).sync().result + a.eq(counted.total_count, 2, 'total count') + + newest = [PNSortKey.desc(PNSortKeyValue.UPDATED)] + first = pubnub.get_channel_members(channel=channel, limit=1, sort_keys=newest).sync().result + a.eq(len(first.data), 1) + a.ok(isinstance(first.next, Next), 'next page token') + second = pubnub.get_channel_members(channel=channel, limit=1, sort_keys=newest, page=first.next).sync().result + a.eq(sorted(member_ids(first.data + second.data)), sorted([plain, with_custom]), 'two pages cover both') + a.ok(isinstance(second.prev, Previous), 'prev page token') + back = pubnub.get_channel_members(channel=channel, limit=1, sort_keys=newest, page=second.prev).sync().result + a.eq(member_ids(back.data), member_ids(first.data), 'prev returns the first page') + finally: + cleanup(pubnub, [plain, with_custom], [channel]) + + +@step('set_channel_members_builder', 'set_channel_members_named', role='open', timeout=180) +def test_set_channel_members(clients): + pubnub = clients.sync('open', 'setter') + first = names.with_suffix('channel_id') + second = names.with_suffix('channel_id_named') + plain = names.with_suffix('some_uuid') + with_custom = names.with_suffix('some_uuid_with_custom') + record([plain, with_custom], [first, second]) + try: + seen = watch(clients, first) + + # snippet.set_channel_members_builder + # snippet.hide + from pubnub.models.consumer.objects_v2.channel_members import PNUUID + from pubnub.models.consumer.objects_v2.common import MemberIncludes + # snippet.show + pubnub.set_uuid_metadata() \ + .uuid("some_uuid") \ + .set_name("some name") \ + .sync() + + custom_1 = { + "key3": "val1", + "key4": "val2" + } + + pubnub.set_uuid_metadata() \ + .uuid("some_uuid_with_custom") \ + .set_name("some name with custom") \ + .custom(custom_1) \ + .sync() + + custom_2 = { + "key5": "val1", + "key6": "val2" + } + + uuids_to_set = [ + PNUUID.uuid("some_uuid"), + PNUUID.uuid_with_custom("some_uuid_with_custom", custom_2) + ] + + envelope = pubnub.set_channel_members() \ + .channel("channel_id") \ + .uuids(uuids_to_set) \ + .include(MemberIncludes(custom=True, user=True, user_custom=True)) \ + .sync() + # snippet.end + + check_members(envelope.result.data, plain, with_custom) + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['channel']['id']), ('membership', 'set', first)) + a.contains([plain, with_custom], payload['uuid']['id']) + + seen = watch(clients, second) + + # snippet.set_channel_members_named + pubnub.set_uuid_metadata(uuid="some_uuid", name="some name").sync() + + custom_1 = { + "key3": "val1", + "key4": "val2" + } + + pubnub.set_uuid_metadata(uuid="some_uuid_with_custom", + name="some name with custom", + custom=custom_1) \ + .sync() + + custom_2 = { + "key5": "val1", + "key6": "val2" + } + + uuids_to_set = [ + PNUUID.uuid("some_uuid"), + PNUUID.uuid_with_custom("some_uuid_with_custom", custom_2) + ] + + envelope = pubnub.set_channel_members(channel="channel_id_named", + uuids=uuids_to_set, + include=MemberIncludes(custom=True, user=True, user_custom=True)) \ + .sync() + # snippet.end + + check_members(envelope.result.data, plain, with_custom) + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['channel']['id']), ('membership', 'set', second)) + + memberships = pubnub.get_memberships(uuid=plain).sync().result.data + a.eq(sorted(channel_ids(memberships)), sorted([first, second]), 'the user side sees both channels') + + # Set adds to the stored members, it does not replace them + extra = names.with_suffix('some_uuid_extra') + record(uuids=[extra]) + try: + pubnub.set_uuid_metadata(uuid=extra, name='extra').sync() + added = pubnub.set_channel_members(channel=first, uuids=[PNUUID.uuid(extra)]).sync() + a.eq(sorted(member_ids(added.result.data)), sorted([plain, with_custom, extra]), 'set keeps the others') + finally: + cleanup(pubnub, [extra], []) + finally: + cleanup(pubnub, [plain, with_custom], [first, second]) + + +@step('remove_channel_members_builder', 'remove_channel_members_named', role='open', timeout=180) +def test_remove_channel_members(clients): + pubnub = clients.sync('open', 'remover') + first = names.with_suffix('channel_id') + second = names.with_suffix('channel_id_named') + plain = names.with_suffix('some_uuid') + with_custom = names.with_suffix('some_uuid_with_custom') + record([plain, with_custom], [first, second]) + try: + seed_users(pubnub, plain, with_custom) + for channel in (first, second): + pubnub.set_channel_metadata(channel=channel, name='Doc channel').sync() + seed_members(pubnub, channel, plain, with_custom) + seen = watch(clients, first) + + # snippet.remove_channel_members_builder + # snippet.hide + from pubnub.endpoints.objects_v2.objects_endpoint import UUIDIncludeEndpoint + from pubnub.models.consumer.objects_v2.channel_members import PNUUID + # snippet.show + envelope = pubnub.remove_channel_members() \ + .channel("channel_id") \ + .uuids([PNUUID.uuid("some_uuid")]) \ + .include_custom(True) \ + .include_uuid(UUIDIncludeEndpoint.UUID_WITH_CUSTOM) \ + .sync() + # snippet.end + + left = envelope.result.data + a.eq(member_ids(left), [with_custom], 'the response lists the remaining members') + a.eq(left[0]['custom'], CUSTOM_2) + a.eq(left[0]['uuid']['custom'], CUSTOM_1) + kind, event, payload = first_event(seen) + a.eq((kind, event, payload['channel']['id'], payload['uuid']['id']), ('membership', 'delete', first, plain)) + eventually(lambda: a.eq(member_ids(pubnub.get_channel_members(channel=first).sync().result.data), + [with_custom]), what='member removed') + a.eq(len(pubnub.get_channel_members(channel=second).sync().result.data), 2, 'other channels keep theirs') + + # snippet.remove_channel_members_named + envelope = pubnub.remove_channel_members(channel="channel_id_named", + uuids=[PNUUID.uuid("some_uuid")], + include=MemberIncludes(custom=True, user=True, user_custom=True)) \ + .sync() + # snippet.end + + a.eq(member_ids(envelope.result.data), [with_custom]) + a.eq(envelope.result.data[0]['uuid']['custom'], CUSTOM_1) + eventually(lambda: a.eq(member_ids(pubnub.get_channel_members(channel=second).sync().result.data), + [with_custom]), what='second member removed') + a.eq(pubnub.get_uuid_metadata(uuid=plain).sync().result.data['id'], plain, 'the user stays') + finally: + cleanup(pubnub, [plain, with_custom], [first, second]) + + +@step('manage_channel_members_builder', 'manage_channel_members_named', role='open', timeout=180) +def test_manage_channel_members(clients): + pubnub = clients.sync('open', 'manager') + first = names.with_suffix('channel_id') + second = names.with_suffix('channel_id_named') + plain = names.with_suffix('some_uuid') + with_custom = names.with_suffix('some_uuid_with_custom') + record([plain, with_custom], [first, second]) + try: + seed_users(pubnub, plain, with_custom) + for channel in (first, second): + pubnub.set_channel_metadata(channel=channel, name='Doc channel').sync() + seed_one_member(pubnub, channel, with_custom) + eventually(lambda: a.eq(member_ids(pubnub.get_channel_members(channel=second).sync().result.data), + [with_custom]), what='seed visible') + seen = watch(clients, first) + + # snippet.manage_channel_members_builder + # snippet.hide + from pubnub.models.consumer.objects_v2.channel_members import PNUUID + from pubnub.models.consumer.objects_v2.common import MemberIncludes + # snippet.show + envelope = pubnub.manage_channel_members() \ + .channel("channel_id") \ + .set([PNUUID.uuid("some_uuid")]) \ + .remove([PNUUID.uuid("some_uuid_with_custom")]) \ + .include(MemberIncludes(custom=True, user=True, user_custom=True)) \ + .sync() + # snippet.end + + a.eq(member_ids(envelope.result.data), [plain], 'the response lists the remaining members') + a.eq(envelope.result.data[0]['uuid']['name'], 'some name') + kind, event, payload = first_event(seen) + a.eq(kind, 'membership') + a.contains([('set', plain), ('delete', with_custom)], (event, payload['uuid']['id'])) + eventually(lambda: a.eq(member_ids(pubnub.get_channel_members(channel=first).sync().result.data), [plain]), + what='members changed') + + # snippet.manage_channel_members_named + envelope = pubnub.manage_channel_members(channel="channel_id_named", + uuids_to_set=[PNUUID.uuid("some_uuid")], + uuids_to_remove=[PNUUID.uuid("some_uuid_with_custom")], + include=MemberIncludes(custom=True, user=True, user_custom=True)) \ + .sync() + # snippet.end + + a.eq(member_ids(envelope.result.data), [plain]) + eventually(lambda: a.eq(member_ids(pubnub.get_channel_members(channel=second).sync().result.data), [plain]), + what='second channel changed') + finally: + cleanup(pubnub, [plain, with_custom], [first, second]) + + +@step('pnchannelmembership_class', role='open') +def test_channel_membership_class(clients): + pubnub = clients.sync('open', 'classes') + uid = names.with_suffix('some_uuid') + plain = names.with_suffix('somechannel') + with_custom = names.with_suffix('somechannel_with_custom') + record([uid], [plain, with_custom]) + try: + seed_channels(pubnub, plain, with_custom) + + # snippet.pnchannelmembership_class + # snippet.hide + from pubnub.models.consumer.objects_v2.memberships import PNChannelMembership + # snippet.show + membership = PNChannelMembership.channel("somechannel") + membership_with_custom = PNChannelMembership.channel_with_custom("somechannel_with_custom", + {"key5": "val1", "key6": "val2"}) + # snippet.end + + a.eq(type(membership).__name__, 'JustChannel') + a.eq(type(membership_with_custom).__name__, 'ChannelWithCustom') + a.eq(membership.to_payload_dict(), {'channel': {'id': plain}}) + a.eq(membership_with_custom.to_payload_dict(), {'channel': {'id': with_custom}, 'custom': CUSTOM_2}) + pubnub.set_memberships(uuid=uid, channel_memberships=[membership, membership_with_custom]).sync() + eventually(lambda: a.eq(sorted(channel_ids(pubnub.get_memberships(uuid=uid).sync().result.data)), + sorted([plain, with_custom])), what='memberships set from the helper objects') + found = pubnub.get_memberships(uuid=uid, include=MembershipIncludes(custom=True)).sync().result.data + a.eq(by_channel(found)[with_custom]['custom'], CUSTOM_2) + a.eq(by_channel(found)[plain]['custom'], None) + finally: + cleanup(pubnub, [uid], [plain, with_custom]) + + +@step('pnuuid_class', role='open') +def test_uuid_class(clients): + pubnub = clients.sync('open', 'classes') + channel = names.with_suffix('channel_id') + plain = names.with_suffix('some_uuid') + with_custom = names.with_suffix('some_uuid_with_custom') + record([plain, with_custom], [channel]) + try: + seed_users(pubnub, plain, with_custom) + pubnub.set_channel_metadata(channel=channel, name='Doc channel').sync() + + # snippet.pnuuid_class + # snippet.hide + from pubnub.models.consumer.objects_v2.channel_members import PNUUID + # snippet.show + member = PNUUID.uuid("some_uuid") + member_with_custom = PNUUID.uuid_with_custom("some_uuid_with_custom", {"key5": "val1", "key6": "val2"}) + # snippet.end + + a.eq(type(member).__name__, 'JustUUID') + a.eq(type(member_with_custom).__name__, 'UUIDWithCustom') + a.eq(member.to_payload_dict(), {'uuid': {'id': plain}}) + a.eq(member_with_custom.to_payload_dict(), {'uuid': {'id': with_custom}, 'custom': CUSTOM_2}) + pubnub.set_channel_members(channel=channel, uuids=[member, member_with_custom]).sync() + eventually(lambda: a.eq(sorted(member_ids(pubnub.get_channel_members(channel=channel).sync().result.data)), + sorted([plain, with_custom])), what='members set from the helper objects') + found = pubnub.get_channel_members(channel=channel, include=MemberIncludes(custom=True)).sync().result.data + a.eq(by_member(found)[with_custom]['custom'], CUSTOM_2) + a.eq(by_member(found)[plain]['custom'], None) + finally: + cleanup(pubnub, [plain, with_custom], [channel]) diff --git a/docs-snippets/sync/presence.py b/docs-snippets/sync/presence.py new file mode 100644 index 00000000..9c7e4429 --- /dev/null +++ b/docs-snippets/sync/presence.py @@ -0,0 +1,329 @@ +"""Docs page: presence (sync). Here now, where now and user state. + +Presence needs channel tracking on the keyset. Joins of a second client are asserted, never the client's own join, +and here_now is polled with eventually because the service reports it with a delay and caches it for 3 seconds. +""" +from pubnub.exceptions import PubNubException + +from harness import assertions as a +from harness import names +from harness import world +from harness.listeners import listen +from harness.step import step +from harness.wait import eventually + +REGISTERED_NAMES = ['cg2', 'cg3', 'gr1', 'gr2', 'gr3', 'such_uuid', 'some-other-uuid'] + + +class Printed: + """Everything the docs code printed so far.""" + + def __init__(self, capsys): + self._capsys = capsys + self._parts = [] + + def text(self): + self._parts.append(self._capsys.readouterr().out) + return ''.join(self._parts) + + +def occupants_of(here_now, channel): + """uuids of one channel in a PNHereNowResult, sorted. The result holds a list of PNHereNowChannelData.""" + found = [c for c in here_now.result.channels if c.channel_name == channel] + return sorted(o.uuid for o in found[0].occupants) if found and found[0].occupants is not None else None + + +def occupancy_of(here_now, channel): + found = [c for c in here_now.result.channels if c.channel_name == channel] + return found[0].occupancy if found else None + + +@step('here_now_builder', 'here_now_named', role='open') +def test_here_now(clients, capsys): + pubnub = clients.sync('open', 'reader') + first, second = names.with_suffix('my_channel'), names.with_suffix('my_channel_2') + printed = Printed(capsys) + one = clients.sync('open', 'user_one') + two = clients.sync('open', 'user_two') + listen(one, first, second) + listen(two, first) + expected = {first: sorted([one.uuid, two.uuid]), second: [one.uuid]} + + def builder(): + # snippet.here_now_builder + here_now = pubnub.here_now() \ + .channels(["my_channel", "my_channel_2"]) \ + .include_uuids(True) \ + .sync() + + if here_now.status.is_error(): + print(f"Error: {here_now.status.error}") + return + + for channel_data in here_now.result.channels: + print("---") + print(f"Channel: {channel_data.channel_name}") + print(f"Occupancy: {channel_data.occupancy}") + for occupant in channel_data.occupants: + print(f"UUID: {occupant.uuid}, State: {occupant.state}") + # snippet.end + return here_now + + def named(): + # snippet.here_now_named + here_now = pubnub.here_now( + channels=["my_channel", "my_channel_2"], + include_uuids=True + ).sync() + + if here_now.status.is_error(): + print(f"Error: {here_now.status.error}") + return + + for channel_data in here_now.result.channels: + print("---") + print(f"Channel: {channel_data.channel_name}") + print(f"Occupancy: {channel_data.occupancy}") + for occupant in channel_data.occupants: + print(f"UUID: {occupant.uuid}, State: {occupant.state}") + # snippet.end + return here_now + + for call in (builder, named): + def check(call=call): + result = call() + a.ok(result is not None, 'here_now returned an error') + a.eq({c: occupants_of(result, c) for c in expected}, expected, 'occupants per channel') + a.eq((result.result.total_channels, result.result.total_occupancy), (2, 3)) + eventually(check, what='here_now lists both clients') + text = printed.text() + a.contains(text, f'Channel: {first}\nOccupancy: 2\n') + a.contains(text, f'UUID: {one.uuid}, State: None') + + +@step('here_now_state', role='open') +def test_here_now_state(clients): + pubnub = clients.sync('open', 'reader') + name = names.with_suffix('my_channel') + one = clients.sync('open', 'user_one') + listen(one, name) + one.set_state().channels(name).state({'abcd': {'age': 15}}).sync() + + def read(): + # snippet.here_now_state + envelope = pubnub.here_now() \ + .channels("my_channel") \ + .include_uuids(True) \ + .include_state(True) \ + .sync() + # snippet.end + return envelope + + def check(): + envelope = read() + found = [(o.uuid, o.state) for c in envelope.result.channels for o in c.occupants] + a.eq(found, [(one.uuid, {'abcd': {'age': 15}})], 'occupants with state') + a.eq((envelope.result.total_channels, envelope.result.total_occupancy), (1, 1)) + eventually(check, what='here_now returns the state') + + +@step('here_now_occupancy', role='open') +def test_here_now_occupancy(clients): + pubnub = clients.sync('open', 'reader') + name = names.with_suffix('my_channel') + for who in ('user_one', 'user_two', 'user_three'): + listen(clients.sync('open', who), name) + + def read(): + # snippet.here_now_occupancy + envelope = pubnub.here_now() \ + .channels("my_channel") \ + .include_uuids(False) \ + .include_state(False) \ + .sync() + # snippet.end + return envelope + + def check(): + envelope = read() + a.eq([(c.channel_name, c.occupancy, c.occupants) for c in envelope.result.channels], [(name, 3, [])]) + a.eq((envelope.result.total_channels, envelope.result.total_occupancy), (1, 3)) + eventually(check, what='here_now counts three occupants') + + +@step('here_now_groups', role='open') +def test_here_now_groups(clients): + pubnub = clients.sync('open', 'reader') + groups = [names.with_suffix(n) for n in ('cg1', 'cg2', 'cg3')] + members = [names.with_suffix(f'grouped_{i}') for i in (1, 2, 3)] + for group, member in zip(groups, members): + pubnub.add_channel_to_channel_group().channels([member]).channel_group(group).sync() + world.record('channel_group', group) + one = clients.sync('open', 'user_one') + listen(one, *members) + + def read(): + # snippet.here_now_groups + envelope = pubnub.here_now() \ + .channel_groups(['cg1', 'cg2', 'cg3']) \ + .include_uuids(True) \ + .include_state(True) \ + .sync() + # snippet.end + return envelope + + def check(): + envelope = read() + a.eq(sorted((c.channel_name, c.occupancy, [o.uuid for o in c.occupants]) for c in envelope.result.channels), + sorted((m, 1, [one.uuid]) for m in members), 'every channel of the groups with its occupant') + a.eq((envelope.result.total_channels, envelope.result.total_occupancy), (3, 3)) + eventually(check, what='here_now by channel group') + + +@step('here_now_paginate', role='open') +def test_here_now_paginate(clients): + pubnub = clients.sync('open', 'reader') + name = names.with_suffix('my_channel') + users = [clients.sync('open', who) for who in ('user_one', 'user_two', 'user_three')] + for user in users: + listen(user, name) + everyone = sorted(user.uuid for user in users) + + def read(): + # snippet.here_now_paginate + # Get first 10 occupants + envelope = pubnub.here_now() \ + .channels("my_channel") \ + .limit(10) \ + .sync() + + # Get next 10 occupants (starting from position 10) + envelope = pubnub.here_now() \ + .channels("my_channel") \ + .limit(10) \ + .offset(10) \ + .sync() + # snippet.end + return envelope + + def check(): + a.eq(occupants_of(read(), name), [], 'nothing from position 10 on') + all_in_ten = pubnub.here_now().channels(name).limit(10).sync() + a.eq(occupants_of(all_in_ten, name), everyone, 'limit 10 returns all three') + eventually(check, what='here_now pagination') + pages = [] + for offset in (0, 2): + page = pubnub.here_now().channels(name).limit(2).offset(offset).sync() + pages.append(occupants_of(page, name)) + a.eq(occupancy_of(page, name), 3, 'occupancy counts all occupants on every page') + a.eq((len(pages[0]), len(pages[1])), (2, 1)) + a.eq(sorted(pages[0] + pages[1]), everyone, 'pages cover every occupant once') + + +@step('where_now_self', 'where_now_other_builder', 'where_now_other_named', role='open') +def test_where_now(clients): + pubnub = clients.sync('open', 'reader') + first, second = names.with_suffix('my_channel'), names.with_suffix('my_channel_2') + listen(pubnub, first, second) + other = clients.sync('open', 'some-other-uuid') + listen(other, first) + + def own(): + # snippet.where_now_self + envelope = pubnub.where_now().sync() + # snippet.end + return envelope + + def builder(): + # snippet.where_now_other_builder + envelope = pubnub.where_now() \ + .uuid('some-other-uuid') \ + .sync() + # snippet.end + return envelope + + def named(): + # snippet.where_now_other_named + envelope = pubnub.where_now(user_id='some-other-uuid').sync() + # snippet.end + return envelope + + eventually(lambda: a.eq(sorted(own().result.channels), sorted([first, second]), 'own channels')) + eventually(lambda: a.eq(builder().result.channels, [first], 'channels of the other user ID (builder)')) + eventually(lambda: a.eq(named().result.channels, [first], 'channels of the other user ID (named)')) + + +@step('set_state_builder', 'set_state_named', 'get_state_builder', 'get_state_named', role='open') +def test_state(clients): + pubnub = clients.sync('open', 'such_uuid') + reader = clients.sync('open', 'reader') + channels = [names.with_suffix(n) for n in ('ch1', 'ch2', 'ch3')] + listen(pubnub, *channels) + + # snippet.set_state_builder + my_state = { + 'age': 20 + } + envelope = pubnub.set_state() \ + .channels(['ch1', 'ch2', 'ch3']) \ + .state(my_state) \ + .sync() + # snippet.end + a.eq(envelope.result.state, {'age': 20}, 'state returned by set_state (builder)') + + # snippet.set_state_named + envelope = pubnub.set_state(channels=['ch1', 'ch2', 'ch3'], state={'age': 21}).sync() + # snippet.end + a.eq(envelope.result.state, {'age': 21}, 'state returned by set_state (named)') + + # snippet.get_state_builder + envelope = pubnub.get_state() \ + .channels(['ch1', 'ch2', 'ch3']) \ + .uuid('such_uuid') \ + .sync() + # snippet.end + a.eq(envelope.result.channels, {c: {'age': 21} for c in channels}, 'state per channel (builder)') + + # snippet.get_state_named + envelope = pubnub.get_state(channels=['ch1', 'ch2', 'ch3'], uuid='such_uuid').sync() + # snippet.end + a.eq(envelope.result.channels, {c: {'age': 21} for c in channels}, 'state per channel (named)') + a.eq(reader.get_state().channels(channels[0]).uuid(pubnub.uuid).sync().result.channels, + {channels[0]: {'age': 21}}, 'another client reads the state') + + +@step('set_state_groups_builder', 'set_state_groups_named', role='open') +def test_state_for_groups(clients): + pubnub = clients.sync('open', 'such_uuid') + groups = [names.with_suffix(n) for n in ('gr1', 'gr2', 'gr3')] + members = [names.with_suffix(f'grouped_{i}') for i in (1, 2, 3)] + own = names.with_suffix('ch1') + for group, member in zip(groups, members): + pubnub.add_channel_to_channel_group().channels([member]).channel_group(group).sync() + world.record('channel_group', group) + listen(pubnub, own, *members, groups=groups) + # The SDK refuses a set_state() that names channel groups and no channel (PubNubException "State setter for + # channel groups is not supported yet"), so the samples below add a channel next to the groups. + refusal = a.raises(PubNubException, lambda: pubnub.set_state().channel_groups(groups).state({'age': 20}).sync()) + a.contains(str(refusal), 'State setter for channel groups is not supported yet') + + # snippet.set_state_groups_builder + my_state = { + 'age': 20 + } + envelope = pubnub.set_state() \ + .channels(['ch1']) \ + .channel_groups(['gr1', 'gr2', 'gr3']) \ + .state(my_state) \ + .sync() + # snippet.end + a.eq(envelope.result.state, {'age': 20}, 'state returned by set_state for groups (builder)') + a.eq(pubnub.get_state().channels(own).sync().result.channels, {own: {'age': 20}}) + + # snippet.set_state_groups_named + envelope = pubnub.set_state(channels=['ch1'], channel_groups=['gr1', 'gr2', 'gr3'], state={'age': 22}).sync() + # snippet.end + a.eq(envelope.result.state, {'age': 22}, 'state returned by set_state for groups (named)') + a.eq(pubnub.get_state().channels(own).sync().result.channels, {own: {'age': 22}}) + group_state = pubnub.get_state().channel_groups(groups).sync().result.channels + a.eq(group_state, {m: {'age': 22} for m in members}, 'state read back through the channel groups') diff --git a/docs-snippets/sync/publish.py b/docs-snippets/sync/publish.py new file mode 100644 index 00000000..784dc190 --- /dev/null +++ b/docs-snippets/sync/publish.py @@ -0,0 +1,224 @@ +"""Docs page: publish (sync). Template for the other pages. + +The docs code sits between the snippet markers. Assertions and client setup sit outside them. +""" +from pubnub.exceptions import PubNubException + +from harness import assertions as a +from harness import names +from harness import world +from harness.listeners import listen +from harness.step import step +from harness.wait import eventually + + +@step('publish_builder', 'publish_named', role='open') +def test_publish_message(clients): + pubnub = clients.sync('open', 'publisher') + channel = names.with_suffix('my_channel') + seen = listen(clients.sync('open', 'subscriber'), channel) + + # snippet.publish_builder + envelope = pubnub.publish() \ + .channel("my_channel") \ + .message({ + 'name': 'Alex', + 'online': True + }) \ + .custom_message_type("text-message") \ + .sync() + # snippet.end + first_timetoken = envelope.result.timetoken + + # snippet.publish_named + envelope = pubnub.publish( + channel="my_channel", + message={ + 'name': 'Sam', + 'online': False + }, + custom_message_type="text-message", + meta={'origin': 'docs'} + ).sync() + # snippet.end + second_timetoken = envelope.result.timetoken + + a.timetoken(first_timetoken) + a.timetoken(second_timetoken) + eventually(lambda: a.eq(len(seen.messages), 2, 'messages received')) + by_token = {m.timetoken: m for m in seen.messages} + a.eq(by_token[first_timetoken].message, {'name': 'Alex', 'online': True}) + a.eq(by_token[first_timetoken].publisher, pubnub.uuid) + a.eq(by_token[first_timetoken].custom_message_type, 'text-message') + a.eq(by_token[second_timetoken].user_metadata, {'origin': 'docs'}) + + +@step('signal_message', role='open') +def test_signal(clients): + pubnub = clients.sync('open', 'signaler') + channel = names.with_suffix('my_channel') + seen = listen(clients.sync('open', 'subscriber'), channel) + + # snippet.signal_message + envelope = pubnub.signal() \ + .channel("my_channel") \ + .message("typing_on") \ + .sync() + # snippet.end + + a.timetoken(envelope.result.timetoken) + eventually(lambda: a.eq([s.message for s in seen.signals], ['typing_on'], 'signals received')) + a.eq(seen.messages, [], 'a signal is not a message') + + +@step('fire_message', role='open') +def test_fire(clients): + pubnub = clients.sync('open', 'firer') + channel = names.with_suffix('my_channel') + seen = listen(clients.sync('open', 'subscriber'), channel) + + # snippet.fire_message + envelope = pubnub.fire() \ + .channel("my_channel") \ + .message({'command': 'refresh'}) \ + .sync() + # snippet.end + + a.timetoken(envelope.result.timetoken) + # A fired message is not replicated to subscribers. A normal publish after it proves the subscriber was listening. + pubnub.publish().channel(channel).message('after-fire').sync() + eventually(lambda: a.eq([m.message for m in seen.messages], ['after-fire'], 'only the later publish arrived')) + + +@step('publish_metadata', role='open') +def test_publish_metadata(clients): + pubnub = clients.sync('open', 'publisher') + channel = names.with_suffix('my_channel') + seen = listen(clients.sync('open', 'subscriber'), channel) + outcomes = [] + + # snippet.publish_metadata + def publish_callback(result, status): + # handle publish result, status always present, result if successful + # status.is_error() to see if error happened + # snippet.hide + outcomes.append((result, status)) + # snippet.show + if status.is_error(): + print(f"Publish failed: {status.error_data}") + else: + print(f"Publish timetoken: {result.timetoken}") + + pubnub.publish().channel("my_channel").message(["hello", "there"]) \ + .meta({'name': 'Alex'}).pn_async(publish_callback) + # snippet.end + + eventually(lambda: a.eq(len(outcomes), 1, 'publish callback calls')) + result, status = outcomes[0] + a.ok(not status.is_error(), 'status is not an error') + a.timetoken(result.timetoken) + eventually(lambda: a.eq(len(seen.messages), 1, 'messages received')) + a.eq(seen.messages[0].message, ['hello', 'there']) + a.eq(seen.messages[0].user_metadata, {'name': 'Alex'}) + a.eq(seen.messages[0].timetoken, result.timetoken) + + +@step('publish_store_post', role='open') +def test_publish_store_and_post(clients): + pubnub = clients.sync('open', 'publisher') + channel = names.with_suffix('my_channel') + seen = listen(clients.sync('open', 'subscriber'), channel) + + # snippet.publish_store_post + stored = pubnub.publish() \ + .channel("my_channel") \ + .message({'kept': True}) \ + .should_store(True) \ + .use_post(True) \ + .sync() + + not_stored = pubnub.publish() \ + .channel("my_channel") \ + .message({'kept': False}) \ + .should_store(False) \ + .use_post(True) \ + .sync() + # snippet.end + + a.timetoken(stored.result.timetoken) + a.timetoken(not_stored.result.timetoken) + eventually(lambda: a.eq(len(seen.messages), 2, 'both messages are delivered live')) + a.eq(sorted(m.message['kept'] for m in seen.messages), [False, True], 'POST bodies arrive as JSON objects') + + def history_messages(): + items = pubnub.fetch_messages(channels=[channel]).sync().result.channels.get(channel, []) + return [item.message for item in items] + + eventually(lambda: a.contains(history_messages(), {'kept': True}, 'the should_store(True) message is stored')) + a.not_contains(history_messages(), {'kept': False}, 'the should_store(False) message is not stored') + world.record('history', channel) + error = a.raises(PubNubException, + lambda: pubnub.publish().channel(channel).message('x' * 33000).use_post(True).sync()) + # The docs promise a "Message Too Large" error, the service answers 413 Request Entity Too Large + a.contains(str(error), '413', 'POST publish of a message over 32 KiB') + small = pubnub.publish().channel(channel).message('x' * 30000).use_post(True).sync() + a.timetoken(small.result.timetoken) + # Over GET the same payload never reaches the size check, the balancer answers 414 first + uri_error = a.raises(PubNubException, lambda: pubnub.publish().channel(channel).message('x' * 40000).sync()) + a.contains(str(uri_error), '414', 'GET publish of an oversize message') + + +@step('fire_post', 'fire_named', role='open') +def test_fire_post_and_named(clients): + pubnub = clients.sync('open', 'firer') + channel = names.with_suffix('my_channel') + seen = listen(clients.sync('open', 'subscriber'), channel) + + # snippet.fire_post + envelope = pubnub.fire() \ + .channel('my_channel') \ + .message('hello there') \ + .use_post(True) \ + .sync() + print(f'fire timetoken: {envelope.result.timetoken}') + # snippet.end + first = envelope.result.timetoken + + # snippet.fire_named + fire = pubnub.fire(channel="my_channel", message='hello there').sync() + print(f'fire timetoken: {fire.result.timetoken}') + # snippet.end + + a.timetoken(first) + a.timetoken(fire.result.timetoken) + pubnub.publish().channel(channel).message('after-fire-post').sync() + eventually(lambda: a.contains([m.message for m in seen.messages], 'after-fire-post', 'the later publish arrived')) + a.not_contains([m.message for m in seen.messages], 'hello there', 'fired messages are not delivered') + + def history_messages(): + items = pubnub.fetch_messages(channels=[channel]).sync().result.channels.get(channel, []) + return [item.message for item in items] + + eventually(lambda: a.contains(history_messages(), 'after-fire-post', 'the later publish is stored')) + a.not_contains(history_messages(), 'hello there', 'fired messages are not stored in history') + world.record('history', channel) + + +@step('signal_named', role='open') +def test_signal_named(clients): + pubnub = clients.sync('open', 'signaler') + channel = names.with_suffix('my_channel') + seen = listen(clients.sync('open', 'subscriber'), channel) + + # snippet.signal_named + signal = pubnub.signal(channel="my_channel", message='hello there', custom_message_type='text-message').sync() + print(f'signal timetoken: {signal.result.timetoken}') + # snippet.end + + a.timetoken(signal.result.timetoken) + eventually(lambda: a.eq(len(seen.signals), 1, 'signals received')) + a.eq(seen.signals[0].message, 'hello there') + a.eq(seen.signals[0].custom_message_type, 'text-message') + a.eq(seen.signals[0].publisher, pubnub.uuid) + too_big = a.raises(PubNubException, lambda: pubnub.signal(channel=channel, message='s' * 100).sync()) + a.ok(too_big is not None, 'the docs promise a 64 byte signal payload limit') diff --git a/docs-snippets/sync/subscribe.py b/docs-snippets/sync/subscribe.py new file mode 100644 index 00000000..4ea13617 --- /dev/null +++ b/docs-snippets/sync/subscribe.py @@ -0,0 +1,995 @@ +"""Docs page: subscribe (sync). Subscribe, Entities, Event listeners, Unsubscribe and the old-style methods.""" +from pubnub.enums import PNStatusCategory +from pubnub.models.server.subscribe import SubscribeMessage + +from harness import assertions as a +from harness import names +from harness import world +from harness.listeners import Collector, listen +from harness.step import compile_only, step +from harness.wait import eventually +from pubnub.models.consumer.message_actions import PNMessageAction + +REGISTERED_NAMES = ['group-test', 'channelGroupName', 'channelMetadata', 'user_metadata', 'cg_1', + 'awesome_channel_group', 'my_channel1', 'my_channel2', 'my_group1', 'my_group2'] + + +def attach(client): + """Add a collector to a client without subscribing. The docs code does the subscribing.""" + collector = Collector() + client.subscribed_by_suite = True + client.add_listener(collector) + return collector + + +def connected(collector): + eventually(lambda: a.ok(PNStatusCategory.PNConnectedCategory in collector.categories, 'not connected yet'), + what='PNConnectedCategory') + + +def texts(collector): + return [m.message for m in collector.messages] + + +@step('subscribe_single', role='open') +def test_subscribe_single(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + name = names.with_suffix('my_channel') + seen = attach(pubnub) + + # snippet.subscribe_single + # single channel subscription + channel = pubnub.channel("my_channel") + t1_subscription = channel.subscription() + t1_subscription.subscribe() + # snippet.end + + a.eq(channel.name, name) + connected(seen) + publisher.publish().channel(name).message('hello single').sync() + eventually(lambda: a.eq(texts(seen), ['hello single'], 'messages received')) + a.eq(seen.messages[0].channel, name) + + +@step('subscribe_multiple', role='open') +def test_subscribe_multiple(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + name = names.with_suffix('my_channel') + group_name = names.with_suffix('group-test') + grouped = names.with_suffix('grouped_channel') + publisher.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).sync() + world.record('channel_group', group_name) + seen = attach(pubnub) + envelopes = [] + original_from_json = SubscribeMessage.from_json.__func__ + + def counting_from_json(cls, json_input): + if json_input.get('d') == 'second': + envelopes.append(json_input['c']) + return original_from_json(cls, json_input) + + SubscribeMessage.from_json = classmethod(counting_from_json) + try: + # snippet.subscribe_multiple + channel = "my_channel" + + # multiple channel subscription + channel_1 = pubnub.channel(channel).subscription() + channel_2 = pubnub.channel(f'{channel}.2').subscription(with_presence=True) + channel_x = pubnub.channel(f'{channel}.*').subscription(with_presence=True) + + group = pubnub.channel_group("group-test").subscription() + + subscription_set = pubnub.subscription_set([channel_1, channel_2, channel_x, group]) + + subscription_set.subscribe() + # snippet.end + + connected(seen) + publisher.publish().channel(name).message('plain').sync() + publisher.publish().channel(f'{name}.2').message('second').sync() + publisher.publish().channel(f'{name}.other').message('wildcard').sync() + publisher.publish().channel(grouped).message('group').sync() + # The message on 'my_channel.2' matches two subscriptions: the channel itself and the wildcard + # 'my_channel.*'. The service sends one envelope per matching subscription, and for the same request the sync + # client got one or two of them, run to run. The SDK delivers every envelope it receives + # (pubnub/managers.py:373-374 loops over all of them, pubnub/workers.py:215 announces each one). The proof: + # copies at the listener equal envelopes received, and never more than the number of matching subscriptions. + matching = {f'{name}.2', f'{name}.*'} + eventually(lambda: a.eq(sorted(set(texts(seen))), ['group', 'plain', 'second', 'wildcard'], + 'distinct messages')) + eventually(lambda: a.eq({text: texts(seen).count(text) for text in set(texts(seen)) if text != 'second'}, + {'plain': 1, 'wildcard': 1, 'group': 1}, + 'messages with one matching subscription arrive once')) + eventually(lambda: a.eq(texts(seen).count('second'), envelopes.count(f'{name}.2'), + 'one delivery per envelope the SDK received')) + a.ok(1 <= envelopes.count(f'{name}.2') <= len(matching), 'at most one envelope per matching subscription') + second = [m for m in seen.messages if m.message == 'second'] + a.eq({m.timetoken for m in second}, {second[0].timetoken}, 'the copies are the same published message') + by_text = {m.message: m for m in seen.messages} + a.eq(by_text['plain'].channel, name) + a.eq(by_text['wildcard'].channel, f'{name}.other') + a.eq(by_text['wildcard'].subscription, f'{name}.*', 'wildcard match is reported as subscription') + a.eq(by_text['group'].channel, grouped) + a.eq(by_text['group'].subscription, group_name, 'channel group match is reported as subscription') + + other = clients.sync('open', 'second_user') + listen(other, f'{name}.2') + eventually(lambda: a.contains([(p.event, p.uuid) for p in seen.presence_events if p.channel == f'{name}.2'], + ('join', other.uuid), 'join of the second client')) + finally: + SubscribeMessage.from_json = classmethod(original_from_json) + + +class AppContextCollector(Collector): + """Collector that also keeps App Context events.""" + + def __init__(self): + super().__init__() + self.channel_events = [] + self.uuid_events = [] + + def channel(self, pubnub, channel): + with self._lock: + self.channel_events.append(channel) + + def uuid(self, pubnub, uuid): + with self._lock: + self.uuid_events.append(uuid) + + +@step('entity_channel', 'entity_channel_group', 'entity_channel_metadata', 'entity_user_metadata', role='open') +def test_entities(clients): + pubnub = clients.sync('open', 'subscriber') + other = clients.sync('open', 'publisher') + name = names.with_suffix('my_channel') + group_name = names.with_suffix('channelGroupName') + meta_name = names.with_suffix('channelMetadata') + user_name = names.with_suffix('user_metadata') + grouped = names.with_suffix('grouped_channel') + seen = AppContextCollector() + pubnub.subscribed_by_suite = True + pubnub.add_listener(seen) + + # snippet.entity_channel + channel = pubnub.channel("my_channel") + # snippet.end + + # snippet.entity_channel_group + channel_group = pubnub.channel_group("channelGroupName") + # snippet.end + + # snippet.entity_channel_metadata + channel_metadata = pubnub.channel_metadata("channelMetadata") + # snippet.end + + # snippet.entity_user_metadata + user_metadata = pubnub.user_metadata("user_metadata") + # snippet.end + + a.eq((type(channel).__name__, channel.name), ('PubNubChannel', name)) + a.eq((type(channel_group).__name__, channel_group.name), ('PubNubChannelGroup', group_name)) + a.eq((type(channel_metadata).__name__, channel_metadata.name), ('PubNubChannelMetadata', meta_name)) + a.eq((type(user_metadata).__name__, user_metadata.name), ('PubNubUserMetadata', user_name)) + + other.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).sync() + world.record('channel_group', group_name) + # SDK bug: PubNubUserMetadata sets _types instead of _type, so its subscription would register the user ID as a + # channel group. Delivery for user metadata is therefore not exercised here, see the report. + pubnub.subscription_set([channel.subscription(), channel_group.subscription(), + channel_metadata.subscription()]).subscribe() + connected(seen) + try: + other.publish().channel(name).message('on the channel').sync() + other.publish().channel(grouped).message('through the group').sync() + other.set_channel_metadata().channel(meta_name).set_name('Entity channel').sync() + eventually(lambda: a.eq(sorted(texts(seen)), ['on the channel', 'through the group'], 'messages received')) + eventually(lambda: a.eq([e.data['id'] for e in seen.channel_events], [meta_name], 'channel metadata event')) + finally: + other.remove_channel_metadata().channel(meta_name).sync() + + +class Printed: + """Everything the docs listeners printed so far. Callbacks run on SDK threads, capsys sees their output.""" + + def __init__(self, capsys): + self._capsys = capsys + self._parts = [] + + def text(self): + self._parts.append(self._capsys.readouterr().out) + return ''.join(self._parts) + + +@step('listeners_event_specific', role='open') +def test_listeners_event_specific(clients, capsys): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + name = names.with_suffix('my_channel') + seen = attach(pubnub) + printed = Printed(capsys) + + # snippet.listeners_event_specific + subscription = pubnub.channel("my_channel").subscription() + + # using closure for reusable listener + def on_message(listener): + def message_callback(message): + print(f"\033[94mMessage received on: {listener}: \n{message.message}\033[0m\n") + return message_callback + + # without closure + def on_message_action(message_action): + print(f"\033[5mMessageAction received: \n{message_action.value}\033[0m\n") + + def on_signal(listener): + def signal_callback(signal): + print(f"\033[0;36mSignal received on: {listener}: \n{signal.publisher} says: \t{signal.message}\033[0m") + return signal_callback + + subscription.on_message = on_message('message_listener') + subscription.on_message_action = on_message_action + subscription.on_signal = on_signal('signal_listener') + + subscription.subscribe() + # snippet.end + + connected(seen) + envelope = publisher.publish().channel(name).message('listener message').sync() + publisher.signal().channel(name).message('listener signal').sync() + action = PNMessageAction().create(type='reaction', value='smile', message_timetoken=envelope.result.timetoken) + publisher.add_message_action().channel(name).message_action(action).sync() + eventually(lambda: a.contains(printed.text(), 'Message received on: message_listener: \n', 'message printed')) + eventually(lambda: a.contains(printed.text(), 'listener message', 'message payload printed')) + eventually(lambda: a.contains(printed.text(), f'{publisher.uuid} says: \tlistener signal', 'signal printed')) + eventually(lambda: a.contains(printed.text(), 'MessageAction received: \nsmile', 'message action printed')) + + +@step('listeners_generic', role='open') +def test_listeners_generic(clients, capsys): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + name = names.with_suffix('my_channel') + seen = attach(pubnub) + printed = Printed(capsys) + + # snippet.listeners_generic + # snippet.hide + from pubnub.callbacks import SubscribeCallback + # snippet.show + + # add generic listener + class PrintListener(SubscribeCallback): + def status(self, _, status): + print(f'\033[92mPrintListener.status:\n{status.category.name}\033[0m') + + def message(self, _, message): + print(f'\033[94mPrintListener.message:\n{message.message}\033[0m') + + def presence(self, _, presence): + print(f'PrintListener.presence:\n{presence.uuid} {presence.event}s ' + f'{presence.subscription or presence.channel}\033[0m') + + def signal(self, _, signal): + print(f'PrintListener.signal:\n{signal.message} from {signal.publisher}\033[0m') + + def channel(self, _, channel): + print(f'\033[0;37mChannel Meta:\n{channel.__dict__}\033[0m') + + def uuid(self, _, uuid): + print(f'User Meta:\n{uuid.__dict__}\033[0m') + + def membership(self, _, membership): + print(f'Membership:\n{membership.__dict__}\033[0m') + + def message_action(self, _, message_action): + print(f'PrintListener.message_action {message_action}\033[0m') + + def file(self, _, file_message): + print(f' {file_message.__dict__}\033[0m') + + # The docs call subscription.add_listener(PrintListener()). The SDK then calls PrintListener.message(message) + # without the pubnub argument and raises TypeError, so the client-level add_listener is used here. + pubnub.add_listener(PrintListener()) + + subscription = pubnub.channel("my_channel").subscription() + subscription.subscribe() + # snippet.end + + connected(seen) + publisher.publish().channel(name).message('generic message').sync() + publisher.signal().channel(name).message('generic signal').sync() + eventually(lambda: a.contains(printed.text(), 'PrintListener.message:\ngeneric message', 'message printed')) + eventually(lambda: a.contains(printed.text(), f'PrintListener.signal:\ngeneric signal from {publisher.uuid}', + 'signal printed')) + + +@step('listeners_subscription_set', role='open') +def test_listeners_subscription_set(clients, capsys): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + names_in_set = [names.with_suffix('my_channel'), names.with_suffix('my_channel_2')] + group_name = names.with_suffix('cg_1') + grouped = names.with_suffix('grouped_channel') + publisher.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).sync() + world.record('channel_group', group_name) + seen = attach(pubnub) + printed = Printed(capsys) + + # snippet.listeners_subscription_set + def on_message(listener): + def message_callback(message): + print(f"\033[94mMessage received on: {listener}: \n{message.message}\033[0m\n") + return message_callback + + subscription_set = pubnub.subscription_set([ + pubnub.channel("my_channel").subscription(), + pubnub.channel("my_channel_2").subscription(), + pubnub.channel_group("cg_1").subscription() + ]) + subscription_set.on_message = on_message('set_listener') + + subscription_set.subscribe() + # snippet.end + + connected(seen) + for target, text in zip(names_in_set + [grouped], ['first', 'second', 'third']): + publisher.publish().channel(target).message(text).sync() + for text in ['first', 'second', 'third']: + eventually(lambda: a.contains(printed.text(), f'Message received on: set_listener: \n{text}\x1b', text)) + + +@step('listeners_presence', role='open') +def test_listeners_presence(clients, capsys): + pubnub = clients.sync('open', 'subscriber') + name = names.with_suffix('my_channel') + seen = attach(pubnub) + printed = Printed(capsys) + + # snippet.listeners_presence + def on_presence(listener): + def presence_callback(presence): + print(f"\033[0;32mPresence received on: {listener}: \t{presence.uuid} {presence.event}s " + f"{presence.subscription or presence.channel}\033[0m") + return presence_callback + + subscription = pubnub.channel("my_channel").subscription(with_presence=True) + subscription.on_presence = on_presence('presence_listener') + + subscription.subscribe() + # snippet.end + + connected(seen) + other = clients.sync('open', 'second_user') + listen(other, name) + eventually(lambda: a.contains(printed.text(), f'presence_listener: \t{other.uuid} joins {name}', 'join printed')) + + +@step('listeners_status', role='open') +def test_listeners_status(clients, capsys): + pubnub = clients.sync('open', 'subscriber') + printed = Printed(capsys) + + # snippet.listeners_status + # snippet.hide + from pubnub.pubnub import SubscribeListener + # snippet.show + + class PrintListener(SubscribeListener): + def status(self, pubnub, status): + print(f'Status:\n{status.__dict__}') + + def message(self, pubnub, message): + pass + + def presence(self, pubnub, presence): + pass + + listener = PrintListener() + pubnub.add_listener(listener) + # snippet.end + + pubnub.subscribed_by_suite = True + pubnub.channel("my_channel").subscription().subscribe() + eventually(lambda: a.contains(printed.text(), 'PNConnectedCategory', 'connected status printed')) + a.contains(printed.text(), 'Status:\n{', 'status dictionary printed') + + +@step('unsubscribe_subscription', role='open') +def test_unsubscribe_subscription(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + single, control = names.with_suffix('my_channel'), names.with_suffix('ch3') + seen = attach(pubnub) + pubnub.channel(control).subscription().subscribe() + + t1_subscription = pubnub.channel(single).subscription() + t1_subscription.subscribe() + connected(seen) + publisher.publish().channel(single).message('before').sync() + eventually(lambda: a.eq(texts(seen), ['before'], 'message before unsubscribe')) + + # snippet.unsubscribe_subscription + t1_subscription.unsubscribe() + # snippet.end + + eventually(lambda: a.eq(pubnub.get_subscribed_channels(), [control], 'channels left')) + publisher.publish().channel(single).message('after single').sync() + publisher.publish().channel(control).message('after control').sync() + eventually(lambda: a.contains(texts(seen), 'after control', 'the control channel still delivers')) + a.not_contains(texts(seen), 'after single', 'no message after unsubscribe') + + +@step('unsubscribe_set', role='open') +def test_unsubscribe_set(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + first, second, control = [names.with_suffix(n) for n in ('ch1', 'ch2', 'ch3')] + seen = attach(pubnub) + pubnub.channel(control).subscription().subscribe() + + subscription_set = pubnub.subscription_set([ + pubnub.channel(first).subscription(), + pubnub.channel(second).subscription() + ]) + subscription_set.subscribe() + connected(seen) + publisher.publish().channel(first).message('before first').sync() + publisher.publish().channel(second).message('before second').sync() + eventually(lambda: a.eq(sorted(texts(seen)), ['before first', 'before second'], 'messages before unsubscribe')) + + # snippet.unsubscribe_set + subscription_set.unsubscribe() + # snippet.end + + eventually(lambda: a.eq(pubnub.get_subscribed_channels(), [control], 'channels left')) + publisher.publish().channel(first).message('after first').sync() + publisher.publish().channel(second).message('after second').sync() + publisher.publish().channel(control).message('after control').sync() + eventually(lambda: a.contains(texts(seen), 'after control', 'the control channel still delivers')) + a.not_contains(texts(seen), 'after first', 'no message after unsubscribe') + a.not_contains(texts(seen), 'after second', 'no message after unsubscribe') + + +@step('unsubscribe_all', role='open') +def test_unsubscribe_all(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + single, first = [names.with_suffix(n) for n in ('my_channel', 'ch1')] + seen = attach(pubnub) + pubnub.channel(single).subscription().subscribe() + pubnub.subscription_set([pubnub.channel(first).subscription()]).subscribe() + connected(seen) + a.eq(sorted(pubnub.get_subscribed_channels()), sorted([single, first])) + + # snippet.unsubscribe_all + pubnub.unsubscribe_all() + # snippet.end + + # unsubscribe_all() leaves the registry unusable (see the report), so the harness must not call it again + pubnub.subscribed_by_suite = False + eventually(lambda: a.eq(pubnub.get_subscribed_channels(), [], 'channels left')) + eventually(lambda: a.contains(seen.categories, PNStatusCategory.PNAcknowledgmentCategory, 'leave acknowledged')) + publisher.publish().channel(single).message('after single').sync() + publisher.publish().channel(first).message('after first').sync() + a.eq(texts(seen), [], 'no message after unsubscribe_all') + + +class StatusCollector(Collector): + """Collector that keeps the status objects, not only their categories.""" + + def __init__(self): + super().__init__() + self.statuses = [] + + def status(self, pubnub, status): + super().status(pubnub, status) + with self._lock: + self.statuses.append(status) + + +def attach_status(client): + collector = StatusCollector() + client.subscribed_by_suite = True + client.add_listener(collector) + return collector + + +@step('subscribe_old_channel', role='open') +def test_subscribe_old_channel(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + name = names.with_suffix('my_channel') + seen = attach_status(pubnub) + + # snippet.subscribe_old_channel + pubnub.subscribe() \ + .channels('my_channel') \ + .execute() + # snippet.end + + connected(seen) + connect = [s for s in seen.statuses if s.category == PNStatusCategory.PNConnectedCategory][0] + a.ok(not connect.is_error(), 'connected status is not an error') + a.eq(connect.status_code, 200) + a.eq(connect.error_data, None) + envelope = publisher.publish().channel(name).message({'hello': 'old'}).meta({'k': 'v'}).sync() + publisher.signal().channel(name).message('old signal').sync() + eventually(lambda: a.eq(len(seen.messages), 1, 'messages received')) + eventually(lambda: a.eq(len(seen.signals), 1, 'signals received')) + message = seen.messages[0] + a.eq((message.message, message.channel, message.subscription), ({'hello': 'old'}, name, None)) + a.eq((message.timetoken, message.user_metadata, message.publisher), + (envelope.result.timetoken, {'k': 'v'}, publisher.uuid)) + signal = seen.signals[0] + a.eq((signal.message, signal.channel, signal.publisher, signal.timetoken > 0), + ('old signal', name, publisher.uuid, True)) + + +@step('subscribe_old_multiple', role='open') +def test_subscribe_old_multiple(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + both = [names.with_suffix('my_channel1'), names.with_suffix('my_channel2')] + seen = attach(pubnub) + + # snippet.subscribe_old_multiple + pubnub.subscribe() \ + .channels(["my_channel1", "my_channel2"]) \ + .execute() + # snippet.end + + connected(seen) + for target in both: + publisher.publish().channel(target).message(f'to {target}').sync() + eventually(lambda: a.eq(sorted(texts(seen)), sorted(f'to {t}' for t in both), 'both channels deliver')) + + +@step('subscribe_old_wildcard', role='open') +def test_subscribe_old_wildcard(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + matching = f"foo.{names.with_suffix('bar')}" + seen = attach(pubnub) + + # snippet.subscribe_old_wildcard + pubnub.subscribe() \ + .channels("foo.*") \ + .execute() + # snippet.end + + connected(seen) + publisher.publish().channel(matching).message('wildcard hit').sync() + eventually(lambda: a.contains(texts(seen), 'wildcard hit', 'wildcard delivers')) + hit = [m for m in seen.messages if m.message == 'wildcard hit'][0] + a.eq((hit.channel, hit.subscription), (matching, 'foo.*')) + + +@step('subscribe_old_group', role='open') +def test_subscribe_old_group(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + group_name = names.with_suffix('awesome_channel_group') + grouped = names.with_suffix('grouped_channel') + publisher.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).sync() + world.record('channel_group', group_name) + seen = attach(pubnub) + + # snippet.subscribe_old_group + pubnub.subscribe() \ + .channel_groups("awesome_channel_group") \ + .execute() + # snippet.end + + connected(seen) + publisher.publish().channel(grouped).message('group hit').sync() + eventually(lambda: a.eq(texts(seen), ['group hit'], 'group delivers')) + a.eq((seen.messages[0].channel, seen.messages[0].subscription), (grouped, group_name)) + + +@step('subscribe_old_presence', 'subscribe_old_group_presence', role='open') +def test_subscribe_old_presence(clients): + pubnub = clients.sync('open', 'subscriber') + name = names.with_suffix('my_channel') + group_name = names.with_suffix('awesome_channel_group') + grouped = names.with_suffix('grouped_channel') + clients.sync('open', 'publisher').add_channel_to_channel_group().channels([grouped]) \ + .channel_group(group_name).sync() + world.record('channel_group', group_name) + seen = attach(pubnub) + + # snippet.subscribe_old_presence + pubnub.subscribe()\ + .channels("my_channel")\ + .with_presence()\ + .execute() + # snippet.end + + # snippet.subscribe_old_group_presence + pubnub.subscribe() \ + .channel_groups("awesome_channel_group") \ + .with_presence() \ + .execute() + # snippet.end + + connected(seen) + other = clients.sync('open', 'second_user') + listen(other, name) + eventually(lambda: a.contains([(p.event, p.channel) for p in seen.presence_events if p.uuid == other.uuid], + ('join', name), 'join on the channel')) + # Presence of a channel group is announced per member channel: a client that subscribes to the group itself + # produces no join, a client that subscribes to a member channel does, and the event carries the group in + # `subscription` (workers.py _process_incoming_payload, subscription_match with the -pnpres suffix stripped). + member = clients.sync('open', 'member_user') + listen(member, grouped) + eventually(lambda: a.contains([(p.event, p.channel, p.subscription) for p in seen.presence_events + if p.uuid == member.uuid], + ('join', grouped, group_name), 'join of a member channel through the group')) + join = [p for p in seen.presence_events if p.event == 'join' and p.uuid == other.uuid and p.channel == name][0] + a.ok(isinstance(join.timestamp, int) and join.occupancy >= 1, 'join carries timestamp and occupancy') + + +@step('subscribe_old_state', role='open') +def test_subscribe_old_state(clients): + pubnub = clients.sync('open', 'subscriber') + name = names.with_suffix('my_channel') + + # snippet.subscribe_old_state + from pubnub.enums import PNStatusCategory + from pubnub.pubnub import SubscribeListener + + class MySubscribeListener(SubscribeListener): + def status(self, pubnub, status): + if status.category == PNStatusCategory.PNConnectedCategory: + state = { + 'field_a': 'awesome', + 'field_b': 10 + } + envelope = pubnub.set_state().channels('my_channel').state(state).sync() + print(f'State set: {envelope.result.state}') + else: + pass + + def message(self, pubnub, message): + pass + + def presence(self, pubnub, presence): + pass + + my_listener = MySubscribeListener() + pubnub.add_listener(my_listener) + + pubnub.subscribe().channels("my_channel").execute() + # snippet.end + + pubnub.subscribed_by_suite = True + reader = clients.sync('open', 'reader') + + def read_state(): + return reader.get_state().channels(name).uuid(pubnub.uuid).sync().result.channels + + eventually(lambda: a.eq(read_state(), {name: {'field_a': 'awesome', 'field_b': 10}}, + 'state set from the status callback')) + here = reader.here_now().channels(name).include_state(True).sync().result + a.eq([(o.uuid, o.state) for o in here.channels[0].occupants], + [(pubnub.uuid, {'field_a': 'awesome', 'field_b': 10})]) + + +@compile_only('subscribe_old_logging', reason='DEBUG logging writes every request URL with the keys to the output') +def logging_sample(): + # snippet.subscribe_old_logging + import logging + import pubnub + + from pubnub.pnconfiguration import PNConfiguration + from pubnub.pubnub import PubNub, SubscribeListener + + pubnub.set_stream_logger('pubnub', logging.DEBUG) + + pnconfig = PNConfiguration() + + pnconfig.subscribe_key = 'demo' + pnconfig.publish_key = 'demo' + pnconfig.user_id = 'my_custom_user_id' + + pubnub = PubNub(pnconfig) + + pubnub.add_listener(SubscribeListener()) + pubnub.subscribe().channels("my_channel").execute() + # snippet.end + return pubnub + + +@step('listeners_old_add', role='open') +def test_listeners_old_add(clients, capsys): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + name = names.with_suffix('my_channel') + printed = Printed(capsys) + seen = attach(pubnub) + + # snippet.listeners_old_add + from pubnub.callbacks import SubscribeCallback + from pubnub.enums import PNOperationType, PNStatusCategory + + class MySubscribeCallback(SubscribeCallback): + def message(self, pubnub, message): + print("Message channel: %s" % message.channel) + print("Message subscription: %s" % message.subscription) + print("Message timetoken: %s" % message.timetoken) + print("Message payload: %s" % message.message) + print("Message publisher: %s" % message.publisher) + + def presence(self, pubnub, presence): + # Can be join, leave, state-change, timeout, or interval + print("Presence event: %s" % presence.event) + + # The channel to which the message was published + print("Presence channel: %s" % presence.channel) + + # Number of users subscribed to the channel + print("Presence occupancy: %s" % presence.occupancy) + + # User state + print("Presence state: %s" % presence.state) + + # Channel group or wildcard subscription match, if any + print("Presence subscription: %s" % presence.subscription) + + # UUID to which this event is related + print("Presence UUID: %s" % presence.uuid) + + # Publish timetoken + print("Presence timestamp: %s" % presence.timestamp) + + # Current timetoken + print("Presence timetoken: %s" % presence.timetoken) + + # List of users that have joined the channel (if event is 'interval') + print("Presence joined: %s" % presence.join) + + # List of users that have left the channel (if event is 'interval') + print("Presence left: %s" % presence.leave) + + # List of users that have timed-out off the channel (if event is 'interval') + print("Presence timed out: %s" % presence.timeout) + + def status(self, pubnub, status): + # The status object returned is always related to subscribe but could contain + # information about subscribe, heartbeat, or errors + # use the operationType to switch on different options + if status.operation == PNOperationType.PNSubscribeOperation \ + or status.operation == PNOperationType.PNUnsubscribeOperation: + if status.category == PNStatusCategory.PNConnectedCategory: + pass + # This is expected for a subscribe, this means there is no error or issue whatsoever + elif status.category == PNStatusCategory.PNReconnectedCategory: + pass + # This usually occurs if subscribe temporarily fails but reconnects. This means + # there was an error but there is no longer any issue + elif status.category == PNStatusCategory.PNDisconnectedCategory: + pass + # This is the expected category for an unsubscribe. This means there + # was no error in unsubscribing from everything + elif status.category == PNStatusCategory.PNUnexpectedDisconnectCategory: + pass + # This is usually an issue with the internet connection, this is an error, handle + # appropriately retry will be called automatically + elif status.category == PNStatusCategory.PNAccessDeniedCategory: + pass + # This means that Access Manager does not allow this client to subscribe to this + # channel and channel group configuration. This is another explicit error + else: + pass + # This is usually an issue with the internet connection, this is an error, handle appropriately + # retry will be called automatically + else: + pass + # Encountered unknown status type + + def signal(self, pubnub, signal): + print("Signal channel: %s" % signal.channel) + print("Signal subscription: %s" % signal.subscription) + print("Signal timetoken: %s" % signal.timetoken) + print("Signal payload: %s" % signal.message) + print("Signal publisher: %s" % signal.publisher) + + def message_action(self, pubnub, message_action): + print("Message action type: %s" % message_action.type) + print("Message action value: %s" % message_action.value) + print("Message action uuid: %s" % message_action.uuid) + print("Message action action_timetoken: %s" % message_action.action_timetoken) + print("Message action message_timetoken: %s" % message_action.message_timetoken) + + pubnub.add_listener(MySubscribeCallback()) + # snippet.end + + pubnub.subscribe().channels(name).with_presence().execute() + connected(seen) + envelope = publisher.publish().channel(name).message({'n': 1}).sync() + publisher.signal().channel(name).message('sig').sync() + action = PNMessageAction().create(type='reaction', value='smile', message_timetoken=envelope.result.timetoken) + publisher.add_message_action().channel(name).message_action(action).sync() + other = clients.sync('open', 'second_user') + listen(other, name) + for text in (f'Message channel: {name}', "Message payload: {'n': 1}", f'Message publisher: {publisher.uuid}', + f'Message timetoken: {envelope.result.timetoken}', 'Signal payload: sig', + 'Message action type: reaction', 'Message action value: smile', + f'Message action message_timetoken: {envelope.result.timetoken}', + 'Presence event: join', f'Presence channel: {name}', f'Presence UUID: {other.uuid}'): + eventually(lambda: a.contains(printed.text(), text, 'printed by the docs listener')) + a.ok(len(seen.messages) == 1) + + +@step('listeners_old_remove', role='open') +def test_listeners_old_remove(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + name = names.with_suffix('my_channel') + keeper = attach(pubnub) + + class MySubscribeCallback(Collector): + pass + + # snippet.listeners_old_remove + my_listener = MySubscribeCallback() + + pubnub.add_listener(my_listener) + + # some time later + pubnub.remove_listener(my_listener) + # snippet.end + + pubnub.add_listener(my_listener) + pubnub.subscribe().channels(name).execute() + connected(keeper) + publisher.publish().channel(name).message('first').sync() + eventually(lambda: a.eq(texts(keeper), ['first'], 'the kept listener receives')) + eventually(lambda: a.eq(texts(my_listener), ['first'], 'a listener that is added receives')) + pubnub.remove_listener(my_listener) + publisher.publish().channel(name).message('second').sync() + eventually(lambda: a.eq(texts(keeper), ['first', 'second'], 'the kept listener still receives')) + a.eq(texts(my_listener), ['first'], 'a removed listener receives nothing more') + + +@step('listeners_old_disconnects', role='open') +def test_listeners_old_disconnects(clients, caplog): + pubnub = clients.sync('open', 'subscriber') + seen = attach(pubnub) + + # snippet.listeners_old_disconnects + import logging + + from pubnub.callbacks import SubscribeCallback + from pubnub.enums import PNStatusCategory + + logger = logging.getLogger(__name__) + + class HandleDisconnectsCallback(SubscribeCallback): + def status(self, pubnub, status): + if status.category == PNStatusCategory.PNUnexpectedDisconnectCategory: + # internet got lost, do some magic and call reconnect when ready + pubnub.reconnect() + elif status.category == PNStatusCategory.PNTimeoutCategory: + # do some magic and call reconnect when ready + pubnub.reconnect() + else: + logger.debug(status) + + def presence(self, pubnub, presence): + pass + + def message(self, pubnub, message): + pass + + def signal(self, pubnub, signal): + pass + + disconnect_listener = HandleDisconnectsCallback() + + pubnub.add_listener(disconnect_listener) + # snippet.end + + with caplog.at_level(logging.DEBUG): + pubnub.subscribe().channels(names.with_suffix('my_channel')).execute() + connected(seen) + eventually(lambda: a.ok(any('PNConnectedCategory' in r.getMessage() or 'PNStatus' in r.getMessage() + for r in caplog.records if r.name == __name__ or r.name.endswith('subscribe')), + 'the docs logger saw the connected status')) + + +@step('unsubscribe_old_channel', role='open') +def test_unsubscribe_old_channel(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + single, control = names.with_suffix('my_channel'), names.with_suffix('ch3') + seen = attach(pubnub) + observer = listen(clients.sync('open', 'observer'), single, presence=True) + pubnub.subscribe().channels([single, control]).execute() + connected(seen) + publisher.publish().channel(single).message('before').sync() + eventually(lambda: a.eq(texts(seen), ['before'], 'message before unsubscribe')) + + # snippet.unsubscribe_old_channel + pubnub.unsubscribe() \ + .channels("my_channel") \ + .execute() + # snippet.end + + eventually(lambda: a.eq(pubnub.get_subscribed_channels(), [control], 'channels left')) + publisher.publish().channel(single).message('after single').sync() + publisher.publish().channel(control).message('after control').sync() + eventually(lambda: a.contains(texts(seen), 'after control', 'the control channel still delivers')) + a.not_contains(texts(seen), 'after single', 'no message after unsubscribe') + eventually(lambda: a.contains([(p.event, p.uuid) for p in observer.presence_events], ('leave', pubnub.uuid), + 'leave event')) + leave = [p for p in observer.presence_events if p.event == 'leave' and p.uuid == pubnub.uuid][0] + a.ok(isinstance(leave.timestamp, int) and isinstance(leave.occupancy, int), 'leave carries timestamp, occupancy') + + +@step('unsubscribe_old_multiple', 'unsubscribe_old_group', role='open') +def test_unsubscribe_old_multiple_and_group(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + both = [names.with_suffix('my_channel1'), names.with_suffix('my_channel2')] + groups = [names.with_suffix('my_group1'), names.with_suffix('my_group2')] + control = names.with_suffix('ch3') + grouped = [names.with_suffix('grouped_one'), names.with_suffix('grouped_two')] + for group_name, channel_name in zip(groups, grouped): + publisher.add_channel_to_channel_group().channels([channel_name]).channel_group(group_name).sync() + world.record('channel_group', group_name) + seen = attach(pubnub) + pubnub.subscribe().channels(both + [control]).channel_groups(groups).execute() + connected(seen) + for target in both + grouped: + publisher.publish().channel(target).message(f'before {target}').sync() + eventually(lambda: a.eq(len(seen.messages), 4, 'messages before unsubscribe')) + + # snippet.unsubscribe_old_multiple + pubnub.unsubscribe() \ + .channels(["my_channel1", "my_channel2"]) \ + .execute() + # snippet.end + + # snippet.unsubscribe_old_group + pubnub.unsubscribe() \ + .channel_groups(["my_group1", "my_group2"]) \ + .execute() + # snippet.end + + eventually(lambda: a.eq((pubnub.get_subscribed_channels(), pubnub.get_subscribed_channel_groups()), + ([control], []), 'subscriptions left')) + for target in both + grouped + [control]: + publisher.publish().channel(target).message(f'after {target}').sync() + eventually(lambda: a.contains(texts(seen), f'after {control}', 'the control channel still delivers')) + a.eq(sorted(m for m in texts(seen) if m.startswith('after')), [f'after {control}'], 'nothing else after') + + +@step('unsubscribe_all_old', role='open') +def test_unsubscribe_all_old(clients): + pubnub = clients.sync('open', 'subscriber') + publisher = clients.sync('open', 'publisher') + single = names.with_suffix('my_channel') + group_name = names.with_suffix('my_group1') + grouped = names.with_suffix('grouped_one') + publisher.add_channel_to_channel_group().channels([grouped]).channel_group(group_name).sync() + world.record('channel_group', group_name) + seen = attach(pubnub) + pubnub.subscribe().channels(single).channel_groups(group_name).execute() + connected(seen) + a.eq((pubnub.get_subscribed_channels(), pubnub.get_subscribed_channel_groups()), ([single], [group_name])) + + # snippet.unsubscribe_all_old + pubnub.unsubscribe_all() + # snippet.end + + # unsubscribe_all() leaves the registry unusable (see the report), so the harness must not call it again + pubnub.subscribed_by_suite = False + eventually(lambda: a.eq((pubnub.get_subscribed_channels(), pubnub.get_subscribed_channel_groups()), ([], []), + 'subscriptions left')) + eventually(lambda: a.contains(seen.categories, PNStatusCategory.PNAcknowledgmentCategory, 'leave acknowledged')) + publisher.publish().channel(single).message('after single').sync() + publisher.publish().channel(grouped).message('after group').sync() + a.eq(texts(seen), [], 'no message after unsubscribe_all') diff --git a/docs-snippets/sync/zz_application_level.py b/docs-snippets/sync/zz_application_level.py new file mode 100644 index 00000000..54a6073c --- /dev/null +++ b/docs-snippets/sync/zz_application_level.py @@ -0,0 +1,46 @@ +"""Docs page: access-manager-v2 (sync), the application level grant. Kept in its own module so it runs last. + +An application level grant opens every channel. A downgrade is not guaranteed to apply at once, so this module +asserts only what the docs claim: the grant call succeeds and the service accepts it. The grant is closed in a finally +(all flags false, ttl 1) and the janitor revokes it again at the end of every pam run. +""" +from harness import assertions as a +from harness.step import step + +FLAGS = ('read', 'write', 'manage', 'delete', 'get', 'update', 'join') + + +def close_application_level(admin): + builder = admin.grant() + for flag in FLAGS: + builder = getattr(builder, flag)(False) + return builder.ttl(1).sync() + + +@step('grant_subkey_all', 'grant_application_level', role='pam', timeout=300) +def test_grant_application_level(clients): + pubnub = clients.sync('pam', 'admin') + try: + # snippet.grant_subkey_all + envelope = pubnub.grant().read(True).write(True).sync() + # snippet.end + + a.eq(envelope.result.level, 'subkey') + a.eq(envelope.result.ttl, 1440) + a.eq((envelope.result.read_enabled, envelope.result.write_enabled), (True, True)) + finally: + closed = close_application_level(pubnub) + + try: + # snippet.grant_application_level + envelope = pubnub.grant(). \ + read(True).write(True). \ + sync() + # snippet.end + + a.eq(envelope.result.level, 'subkey') + a.eq((envelope.result.read_enabled, envelope.result.write_enabled), (True, True)) + finally: + closed = close_application_level(pubnub) + a.eq(closed.result.level, 'subkey') + a.eq((closed.result.read_enabled, closed.result.write_enabled), (False, False)) diff --git a/setup.py b/setup.py index c3be7e7c..c85b73aa 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ 'Source': 'https://github.com/pubnub/python', 'Documentation': 'https://www.pubnub.com/docs/sdks/python', }, - packages=find_packages(exclude=("examples*", 'tests*')), + packages=find_packages(exclude=("examples*", 'tests*', 'docs-snippets*')), license='PubNub Software Development Kit License', classifiers=( 'Development Status :: 5 - Production/Stable', From 38d922154720661d1371b4fdb1ffdabdb6847b54 Mon Sep 17 00:00:00 2001 From: Mateusz Wiktor Date: Thu, 12 Nov 2026 11:18:51 +0100 Subject: [PATCH 3/4] test: put the repository root on the import path of the staged run The staged copy under docs-snippets/.run// cannot find the checkout from its own harness package, so pytest failed with "No module named 'pubnub'" wherever the package is not installed (the CI runner installs only the dev requirements). The run now sets PYTHONPATH to the repository root. The key status line prints only the roles the shard needs. Co-Authored-By: Claude Sonnet 5.5 --- docs-snippets/harness/cli.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs-snippets/harness/cli.py b/docs-snippets/harness/cli.py index 84318f60..0ea4dfe8 100644 --- a/docs-snippets/harness/cli.py +++ b/docs-snippets/harness/cli.py @@ -38,15 +38,21 @@ def run(argv): if os.environ.get('CI') and ('-k' in extra or os.environ.get('DOCS_SNIPPETS_MUTATE')): print('subset selection and mutation are refused in CI') return 2 + needed = ('open', 'pam') if shard == 'all' else (shard,) for line in env.status(): - print('keys', line) + if line.split(':')[0] in needed: + print('keys', line) if env.missing('admin'): print('preflight: not run, PUBNUB_ADMIN_API_KEY is missing') elif preflight.main(): print('preflight: a keyset lacks a feature, see above. The steps that need it will fail.') target = stage.stage() print('staged:', target.name) - environ = {**os.environ, 'DOCS_SNIPPETS_STAGED': '1', 'PYTHONUNBUFFERED': '1', + # The staged copy lives in .run//, so its harness cannot find the checkout by itself. Put the repository root on + # the import path, otherwise pytest only works where pubnub is installed. + repo_root = str(registry.ROOT.parent) + python_path = os.pathsep.join(p for p in (repo_root, os.environ.get('PYTHONPATH', '')) if p) + environ = {**os.environ, 'DOCS_SNIPPETS_STAGED': '1', 'PYTHONUNBUFFERED': '1', 'PYTHONPATH': python_path, 'DOCS_SNIPPETS_REPORT': str(registry.ROOT / 'report' / shard), 'DOCS_SNIPPETS_LEDGER_DIR': str(registry.ROOT / '.run')} import pubnub From 7fbcd6a04c148cd6d2e83d143e0224d55033310a Mon Sep 17 00:00:00 2001 From: Mateusz Wiktor Date: Thu, 12 Nov 2026 11:32:42 +0100 Subject: [PATCH 4/4] test: retry the channel group state read in the presence steps The presence service resolves a channel group some seconds after it was created and answers 400 "No valid channels specified" until then. The read after the write now retries for up to 45 seconds with the same assertion. Co-Authored-By: Claude Sonnet 5.5 --- docs-snippets/asyncio/presence.py | 13 +++++++++++-- docs-snippets/sync/presence.py | 12 ++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs-snippets/asyncio/presence.py b/docs-snippets/asyncio/presence.py index 0c1d6aac..753dbce1 100644 --- a/docs-snippets/asyncio/presence.py +++ b/docs-snippets/asyncio/presence.py @@ -248,5 +248,14 @@ async def test_state_for_groups(aclients): a.eq(envelope.result.state, {'age': 20}, 'state returned by set_state for groups') own_state = await pubnub.get_state().channels(own).future() a.eq(own_state.result.channels, {own: {'age': 20}}) - group_state = await pubnub.get_state().channel_groups(groups).future() - a.eq(group_state.result.channels, {m: {'age': 20} for m in members}, 'state read back through the channel groups') + + async def group_state(): + # The presence service resolves a channel group some seconds after it was created, an error answers until then. + envelope = await pubnub.get_state().channel_groups(groups).future() + a.ok(not envelope.is_error() and envelope.result is not None, 'channel groups not resolved yet') + return envelope.result.channels + + async def read_back(): + a.eq(await group_state(), {m: {'age': 20} for m in members}, 'state read back through the channel groups') + + await aeventually(read_back, timeout=45, what='state read back through the channel groups') diff --git a/docs-snippets/sync/presence.py b/docs-snippets/sync/presence.py index 9c7e4429..4e1fb501 100644 --- a/docs-snippets/sync/presence.py +++ b/docs-snippets/sync/presence.py @@ -325,5 +325,13 @@ def test_state_for_groups(clients): # snippet.end a.eq(envelope.result.state, {'age': 22}, 'state returned by set_state for groups (named)') a.eq(pubnub.get_state().channels(own).sync().result.channels, {own: {'age': 22}}) - group_state = pubnub.get_state().channel_groups(groups).sync().result.channels - a.eq(group_state, {m: {'age': 22} for m in members}, 'state read back through the channel groups') + + def group_state(): + # The presence service resolves a channel group some seconds after it was created and answers 400 until then. + try: + return pubnub.get_state().channel_groups(groups).sync().result.channels + except PubNubException as error: + raise AssertionError(f'channel groups not resolved yet: {error}') + + eventually(lambda: a.eq(group_state(), {m: {'age': 22} for m in members}, 'state read back through the groups'), + timeout=45, what='state read back through the channel groups')