From ff9ec9583a88b0f483275b9c21520f8ef93bffa2 Mon Sep 17 00:00:00 2001 From: Ehsan Rahman Date: Fri, 30 Oct 2026 13:20:31 +0000 Subject: [PATCH 1/2] Add 33 advanced pitfalls: concurrency, object model, interpreter, data systems - docs/concurrency.md (20-30): asyncio task garbage collection, forgotten await, blocking the loop, swallowed cancellation, cross-thread loop calls, non-atomic +=, vanishing thread exceptions, Lock re-entry, threading.local under asyncio, fork() deadlocks, daemon threads. - docs/object-model.md (31-37): property vs __getattr__, lru_cache leaks, __eq__ removing __hash__, __exit__ swallowing, weakrefs to bound methods, mock.patch targets, double-imported __main__. - docs/interpreter.md (38-42): -O stripping asserts, class-body comprehension scope, locals() writes, id() reuse, int/str limit. - docs/data-systems.md (43-52): hash() sharding, NaN, nanosecond timestamps, float prices, DST arithmetic, fixed UTC offsets, utcnow(), struct alignment, non-standard JSON. - README: advanced contents table and an "In the wild" section on Home Assistant bug fixes and guard rails, each linked to the PR or source file. Examples 48 and 49 reproduce HA PRs #188537 and #188836. - One self-checking example per pitfall; all 52 run in CI. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 2 +- README.md | 41 ++- docs/concurrency.md | 257 ++++++++++++++++++ docs/data-systems.md | 208 ++++++++++++++ docs/interpreter.md | 115 ++++++++ docs/object-model.md | 159 +++++++++++ examples/20_asyncio_task_garbage_collected.py | 38 +++ examples/21_asyncio_forgotten_await.py | 30 ++ examples/22_asyncio_blocking_call.py | 34 +++ examples/23_asyncio_swallowed_cancellation.py | 45 +++ examples/24_asyncio_cross_thread_calls.py | 55 ++++ examples/25_thread_increment_not_atomic.py | 53 ++++ examples/26_thread_exceptions_vanish.py | 35 +++ examples/27_lock_not_reentrant.py | 27 ++ examples/28_threading_local_in_asyncio.py | 39 +++ examples/29_fork_with_lock_held.py | 41 +++ examples/30_daemon_threads_skip_cleanup.py | 44 +++ examples/31_property_getattr_masking.py | 44 +++ examples/32_lru_cache_method_leak.py | 37 +++ examples/33_eq_disables_hash.py | 40 +++ examples/34_exit_swallows_exceptions.py | 41 +++ examples/35_weakref_bound_method.py | 23 ++ examples/36_mock_patch_location.py | 23 ++ examples/37_main_module_imported_twice.py | 31 +++ examples/38_assert_stripped_by_O.py | 27 ++ examples/39_class_body_comprehension_scope.py | 38 +++ examples/40_locals_writes_ignored.py | 29 ++ examples/41_id_reuse.py | 35 +++ examples/42_int_str_digit_limit.py | 32 +++ examples/43_hash_randomization_sharding.py | 26 ++ examples/44_nan_breaks_containers.py | 25 ++ examples/45_float_nanosecond_timestamps.py | 20 ++ .../46_datetime_microsecond_truncation.py | 20 ++ examples/47_float_prices.py | 23 ++ ...48_aware_datetime_wall_clock_arithmetic.py | 28 ++ examples/49_fixed_offset_not_timezone.py | 20 ++ examples/50_utcnow_naive_timestamp.py | 30 ++ examples/51_struct_native_alignment.py | 23 ++ examples/52_json_nan_infinity.py | 19 ++ 39 files changed, 1854 insertions(+), 3 deletions(-) create mode 100644 docs/concurrency.md create mode 100644 docs/data-systems.md create mode 100644 docs/interpreter.md create mode 100644 docs/object-model.md create mode 100644 examples/20_asyncio_task_garbage_collected.py create mode 100644 examples/21_asyncio_forgotten_await.py create mode 100644 examples/22_asyncio_blocking_call.py create mode 100644 examples/23_asyncio_swallowed_cancellation.py create mode 100644 examples/24_asyncio_cross_thread_calls.py create mode 100644 examples/25_thread_increment_not_atomic.py create mode 100644 examples/26_thread_exceptions_vanish.py create mode 100644 examples/27_lock_not_reentrant.py create mode 100644 examples/28_threading_local_in_asyncio.py create mode 100644 examples/29_fork_with_lock_held.py create mode 100644 examples/30_daemon_threads_skip_cleanup.py create mode 100644 examples/31_property_getattr_masking.py create mode 100644 examples/32_lru_cache_method_leak.py create mode 100644 examples/33_eq_disables_hash.py create mode 100644 examples/34_exit_swallows_exceptions.py create mode 100644 examples/35_weakref_bound_method.py create mode 100644 examples/36_mock_patch_location.py create mode 100644 examples/37_main_module_imported_twice.py create mode 100644 examples/38_assert_stripped_by_O.py create mode 100644 examples/39_class_body_comprehension_scope.py create mode 100644 examples/40_locals_writes_ignored.py create mode 100644 examples/41_id_reuse.py create mode 100644 examples/42_int_str_digit_limit.py create mode 100644 examples/43_hash_randomization_sharding.py create mode 100644 examples/44_nan_breaks_containers.py create mode 100644 examples/45_float_nanosecond_timestamps.py create mode 100644 examples/46_datetime_microsecond_truncation.py create mode 100644 examples/47_float_prices.py create mode 100644 examples/48_aware_datetime_wall_clock_arithmetic.py create mode 100644 examples/49_fixed_offset_not_timezone.py create mode 100644 examples/50_utcnow_naive_timestamp.py create mode 100644 examples/51_struct_native_alignment.py create mode 100644 examples/52_json_nan_infinity.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b0c2d81..7c74548 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Found a Python behavior that surprised you? Please add it. ## Adding one 1. Create `examples/NN_short_name.py`, numbered after the last example. Show the trap, `assert` the surprising result, then show and `assert` the fix. Follow the layout of the existing files. -2. Add a section to `README.md` under the matching category, with **The Trap** snippet, a short **Why**, **The fix**, and a link to your example. Add a row to the Contents table. +2. Add a section under the matching category: `README.md` for fundamentals, or the relevant page in `docs/` for advanced topics, with **The Trap** snippet, a short **Why**, **The fix**, and a link to your example. Add a row to the Contents table. 3. Run every example locally: ```sh diff --git a/README.md b/README.md index ebece65..426e37b 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,18 @@ ![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue) [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE) -A catalog of Python gotchas, traps and surprising behaviors, each with a minimal reproduction, an explanation of *why* it happens, and the fix. +A catalog of 52 Python gotchas, traps and surprising behaviors, from beginner-level surprises to asyncio, threading, CPython internals and data-engineering bugs. Each comes with a minimal reproduction, an explanation of *why* it happens, and the fix. Python's syntax prioritizes readability, but a readable surface can hide complex, leaky internal state. John Ousterhout's *A Philosophy of Software Design* calls the worst symptom of complexity *unknown unknowns*: behavior you cannot predict from the interface in front of you. When the language's mechanics don't match its readable syntax, developers are forced to hold a mental model of the interpreter's state, and usually discover the truth through runtime bugs. Here is a log of the design traps developers learn the hard way. -> Every snippet below has a runnable, self-checking twin in [`examples/`](examples/). CI runs them on CPython 3.10 to 3.14, so the outputs shown here are verified, not remembered. +> Every snippet here and in [`docs/`](docs/) has a runnable, self-checking twin in [`examples/`](examples/). CI runs them on CPython 3.10 to 3.14, so the outputs shown here are verified, not remembered. ## Contents +### Part I: Fundamentals + | # | Pitfall | What you see | The fix | |---|---------|--------------|---------| | | **Iterators & laziness** | | | @@ -42,6 +44,17 @@ Here is a log of the design traps developers learn the hard way. | 18 | [Banker's rounding](#18-round-uses-bankers-rounding) | `round(2.5) == 2` | `Decimal` + `ROUND_HALF_UP` | | 19 | [`return` in `finally`](#19-return-in-finally-swallows-exceptions) | Exception vanishes | Never return from `finally` | +### Part II: Advanced + +| Page | # | Covers | +|------|---|--------| +| [Concurrency: asyncio & threading](docs/concurrency.md) | 20–30 | Garbage-collected tasks, forgotten `await`, blocking the event loop, swallowed cancellation, cross-thread loop calls, non-atomic `+=`, vanishing thread exceptions, `Lock` re-entry, `threading.local` under asyncio, `fork()` deadlocks, daemon threads | +| [Object model & imports](docs/object-model.md) | 31–37 | Properties vs `__getattr__`, `@lru_cache` leaks, `__eq__` vs `__hash__`, `__exit__` swallowing errors, weak refs to bound methods, `mock.patch` targets, double-imported `__main__` | +| [Interpreter internals](docs/interpreter.md) | 38–42 | `python -O` stripping asserts, class-body comprehension scope, `locals()` writes, `id()` reuse, the 4300-digit `int`/`str` limit | +| [Data systems: time, numbers, bytes](docs/data-systems.md) | 43–52 | `hash()` sharding, NaN in containers, nanosecond timestamps in floats and `datetime`, float prices, DST arithmetic, fixed UTC offsets, `utcnow()`, `struct` alignment, non-standard JSON | + +See also [In the wild](#in-the-wild-home-assistant) for how a large production codebase hits and guards against these. + --- ## Iterators & laziness @@ -205,6 +218,8 @@ print(bob.items) # Output: ['apple'] (Bob never added anything!) **The fix:** Create per-instance state in `__init__`, or use a dataclass with `field(default_factory=list)`. Dataclasses refuse a bare `items: list = []` default for exactly this reason. +**In the wild:** Home Assistant [removed class-level mutable defaults](https://github.com/home-assistant/core/pull/170119) from an integration after a reviewer spotted them. They hadn't caused a bug yet only because nothing mutated them. + [Example](examples/06_shared_class_attributes.py) ### 7. Modifying a List During Iteration @@ -484,6 +499,28 @@ print(save()) # Output: saved --- +## In the wild: Home Assistant + +[Home Assistant Core](https://github.com/home-assistant/core) is one of the largest asyncio codebases in open source: thousands of integrations, written by different people, sharing a single event loop. Several pitfalls above show up in its recent bug fixes, and its core has guard rails built specifically against them. + +**Bugs fixed:** + +| Fix | Pitfall | +|-----|---------| +| [Alert notifications fired in a tight loop during the DST fall-back hour](https://github.com/home-assistant/core/pull/188537) | [#48](docs/data-systems.md#48-aware-datetime-arithmetic-is-wall-clock-arithmetic): `now() + delay` is wall-clock arithmetic | +| [Automatic backups ran an hour off after every DST change](https://github.com/home-assistant/core/pull/188836) | [#49](docs/data-systems.md#49-a-utc-offset-is-not-a-time-zone): a parsed `+02:00` is not a time zone | +| [Hikvision updated state from a library thread](https://github.com/home-assistant/core/pull/160027) | [#24](docs/concurrency.md#24-loop-methods-arent-thread-safe): loop APIs aren't thread-safe | +| [Class-level mutable defaults in an integration](https://github.com/home-assistant/core/pull/170119) | [#6](#6-shared-mutable-class-attributes): shared class attributes | + +**Guard rails in core:** + +- **Task tracking.** `hass.async_create_task()` stores every task in a set and removes it on completion, so tasks can't be garbage-collected mid-flight ([#20](docs/concurrency.md#20-fire-and-forget-tasks-get-garbage-collected)). +- **Blocking-call detection.** [`block_async_io.py`](https://github.com/home-assistant/core/blob/dev/homeassistant/block_async_io.py) wraps `time.sleep`, `open`, `os.listdir`, `importlib.import_module`, SSL certificate loading and more, and reports any call made from the event loop ([#22](docs/concurrency.md#22-blocking-calls-freeze-the-event-loop)). +- **Thread checks.** `verify_event_loop_thread()` compares `threading.get_ident()` with the loop's thread and raises when a loop-only API is called from elsewhere ([#24](docs/concurrency.md#24-loop-methods-arent-thread-safe)). +- **Linting.** The project [enabled 19 more flake8-bugbear rules](https://github.com/home-assistant/core/pull/170494) through ruff, covering mutable defaults, loop-variable shadowing and similar traps. + +The lesson generalizes: when a pitfall can't be designed away, make it fail loudly. Every guard rail above turns a silent, intermittent bug into an immediate error with a clear message. + ## Running the examples Each file in [`examples/`](examples/) prints the surprising output and `assert`s it, so a script that runs without error confirms the behavior described above. diff --git a/docs/concurrency.md b/docs/concurrency.md new file mode 100644 index 0000000..d099b18 --- /dev/null +++ b/docs/concurrency.md @@ -0,0 +1,257 @@ +# Concurrency Pitfalls: asyncio and Threading + +[← Back to all pitfalls](../README.md) + +Concurrency bugs are the purest form of *unknown unknowns*. The code reads top to bottom, but it runs in an order decided by the event loop, the OS scheduler and the GIL. Most of these pass every test and fail under production load. + +## Contents + +| # | Pitfall | What you see | The fix | +|---|---------|--------------|---------| +| | **asyncio** | | | +| 20 | [Fire-and-forget tasks get garbage-collected](#20-fire-and-forget-tasks-get-garbage-collected) | Work silently never finishes | Keep a reference in a set | +| 21 | [Forgetting `await`](#21-forgetting-await-silently-does-nothing) | Function "ran" but did nothing | `await` it; enable a type checker | +| 22 | [Blocking calls freeze the loop](#22-blocking-calls-freeze-the-event-loop) | Every task stalls at once | `asyncio.to_thread()` | +| 23 | [Catching `BaseException`](#23-catching-baseexception-breaks-cancellation) | `cancel()` and timeouts do nothing | Catch `Exception` | +| 24 | [Loop methods aren't thread-safe](#24-loop-methods-arent-thread-safe) | Callbacks run late, or never | `call_soon_threadsafe()` | +| | **Threading** | | | +| 25 | [`+=` is not atomic](#25--is-not-atomic) | Lost updates | `threading.Lock` | +| 26 | [Exceptions in threads vanish](#26-exceptions-in-threads-vanish) | Failures with no traceback | Call `future.result()` | +| 27 | [`Lock` is not reentrant](#27-lock-is-not-reentrant) | Thread deadlocks on itself | `threading.RLock` | +| 28 | [`threading.local` in asyncio](#28-threadinglocal-leaks-between-asyncio-tasks) | One request sees another's data | `contextvars` | +| 29 | [`fork()` with threads running](#29-fork-copies-locked-locks) | Child process hangs forever | `spawn` / `forkserver` | +| 30 | [Daemon threads skip cleanup](#30-daemon-threads-skip-cleanup) | Unflushed data at exit | Stop event + `join()` | + +--- + +## asyncio + +### 20. Fire-and-Forget Tasks Get Garbage-Collected + +`asyncio.create_task(job())` looks like "start this in the background". But the event loop keeps only a *weak* reference to tasks. If your code doesn't hold the returned task, the garbage collector can destroy it while it's still pending, and the work never completes. All you get is a `Task was destroyed but it is pending!` log line, if you're lucky. + +```python +# The Trap +async def main(): + asyncio.create_task(send_heartbeats()) # Nothing keeps this task alive. + ... +``` + +**Why:** A task parked on an `Event`, `Queue`, `Lock` or `Future` is only reachable through a reference cycle (task → coroutine → future → callback → task). A cycle with no outside reference is garbage. Tasks waiting on sockets or timers are usually held by the loop, which is why this is so intermittent. + +**The fix:** Keep a strong reference until the task finishes, or use `asyncio.TaskGroup` (Python 3.11+), which owns its tasks. + +```python +background_tasks = set() + +task = asyncio.create_task(send_heartbeats()) +background_tasks.add(task) +task.add_done_callback(background_tasks.discard) +``` + +**In the wild:** This is the pattern the [asyncio docs](https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task) recommend, and the one [Home Assistant's core](https://github.com/home-assistant/core/blob/dev/homeassistant/core.py) uses: `hass.async_create_task()` adds every task to `self._tasks` and removes it with `add_done_callback`, so integration authors never hold the reference themselves. Ruff's `RUF006` flags unstored `create_task` results. + +[Example](../examples/20_asyncio_task_garbage_collected.py) + +### 21. Forgetting `await` Silently Does Nothing + +Calling an `async def` function doesn't run it. It returns a coroutine object, and if you don't `await` it, the body never executes. The line looks exactly like a normal function call. + +```python +# The Trap +async def save(order): + await db.insert(order) + +async def checkout(order): + save(order) # Creates a coroutine and throws it away. Nothing is saved. +``` + +At best, Python prints `RuntimeWarning: coroutine 'save' was never awaited` when the coroutine is garbage-collected, which is easy to miss in production logs. + +**The fix:** `await save(order)`. Type checkers catch this reliably (pyright's `reportUnusedCoroutine`, mypy's `unused-coroutine`), and running tests with `-W error::RuntimeWarning` turns the warning into a failure. + +[Example](../examples/21_asyncio_forgotten_await.py) + +### 22. Blocking Calls Freeze the Event Loop + +asyncio is cooperative: a task gives up control only at an `await`. Any synchronous call that blocks, such as `time.sleep()`, `requests.get()`, file I/O, or a heavy computation, stops *every* task on the loop until it returns. + +```python +# The Trap +async def handler(): + time.sleep(0.2) # Every other task is frozen for 200 ms. +``` + +**The fix:** Use the async equivalent (`await asyncio.sleep()`, `aiohttp`, `httpx.AsyncClient`), or run blocking work in a thread with `await asyncio.to_thread(func, ...)`. Run with `PYTHONASYNCIODEBUG=1` during development: asyncio then logs any step that holds the loop for more than 100 ms. + +**In the wild:** Home Assistant patches `time.sleep`, `open`, `os.listdir`, `importlib.import_module`, SSL certificate loading and more at startup ([`block_async_io.py`](https://github.com/home-assistant/core/blob/dev/homeassistant/block_async_io.py)), so any integration that calls them from the event loop is reported immediately. The `time.sleep` check was [added in 2022](https://github.com/home-assistant/core/pull/63766). + +[Example](../examples/22_asyncio_blocking_call.py) + +### 23. Catching `BaseException` Breaks Cancellation + +`asyncio` cancels a task by raising `CancelledError` inside it. Since Python 3.8 that's a subclass of `BaseException`, not `Exception`. A bare `except:` or `except BaseException:` written to "never crash" also means "never cancel", and timeouts (`asyncio.timeout()`, `wait_for()`) are built on cancellation. + +```python +# The Trap +async def poll_forever(): + while True: + try: + await fetch_quotes() + except BaseException: # Also catches CancelledError. + log.exception("poll failed") + +task.cancel() # Logged, ignored. The loop keeps polling. +``` + +**The fix:** Catch `Exception`. If you really must catch `BaseException`, re-raise `CancelledError`: + +```python +except asyncio.CancelledError: + raise +except Exception: + log.exception("poll failed") +``` + +[Example](../examples/23_asyncio_swallowed_cancellation.py) + +### 24. Loop Methods Aren't Thread-Safe + +Many libraries deliver callbacks on their own threads. Calling `loop.call_soon()`, `loop.create_task()` or any async API from such a thread is undefined behavior. The most visible symptom is that `call_soon()` queues the callback without waking the loop, so it sits there until something unrelated wakes the loop up, which could be seconds later or never. + +```python +# The Trap +def on_message(msg): # Called on the library's network thread. + loop.call_soon(handle, msg) # Queued, but the sleeping loop isn't woken. +``` + +In the example, the callback is delayed from 100 ms to 500 ms. With `loop.set_debug(True)`, asyncio raises `RuntimeError: Non-thread-safe operation invoked on an event loop other than the current one` instead. + +**The fix:** Use `loop.call_soon_threadsafe(handle, msg)` for callbacks, or `asyncio.run_coroutine_threadsafe(coro, loop)` for coroutines. Both wake the loop immediately. + +**In the wild:** Home Assistant's [Hikvision integration](https://github.com/home-assistant/core/pull/160027) updated entity state directly from its library's event-stream thread. The fix switched to a method that hops to the loop with `call_soon_threadsafe`. HA now raises on this mistake: core APIs call [`verify_event_loop_thread()`](https://github.com/home-assistant/core/blob/dev/homeassistant/core.py), which compares `threading.get_ident()` with the loop's thread. + +[Example](../examples/24_asyncio_cross_thread_calls.py) + +--- + +## Threading + +### 25. `+=` Is Not Atomic + +The GIL guarantees that only one thread runs Python bytecode at a time, not that a line of Python runs uninterrupted. `counter += 1` compiles to separate load, add and store instructions, and a thread switch between the load and the store loses the other thread's update. + +```python +# The Trap +import dis +dis.dis("counter += 1") +# LOAD_NAME counter +# LOAD_CONST 1 +# BINARY_OP += (INPLACE_ADD before 3.11) +# STORE_NAME counter <- another thread may have stored in between +``` + +The example widens the window with `time.sleep(0)` so the race shows every time. Two threads doing 1,000 increments each end up near 1,000 instead of 2,000. The free-threaded build (Python 3.13t and later) has no GIL at all, so races that were rare become common. + +**The fix:** Guard shared read-modify-write with a `threading.Lock`, use `queue.Queue` to hand work between threads, or give each thread its own data and combine the results at the end. + +[Example](../examples/25_thread_increment_not_atomic.py) + +### 26. Exceptions in Threads Vanish + +An exception in a `threading.Thread` prints a traceback to stderr and kills that thread, but `join()` returns normally and the main thread carries on. With `concurrent.futures`, it's quieter still: the exception is stored on the `Future` and nothing is printed at all. + +```python +# The Trap +with ThreadPoolExecutor() as pool: + pool.submit(process_batch, batch) # If this raises, you'll never know. +``` + +**The fix:** Always collect results. `future.result()` re-raises the worker's exception in the caller. With many futures, iterate over `concurrent.futures.as_completed()` and call `.result()` on each. For raw threads, set `threading.excepthook` to route errors into your logging or alerting. + +[Example](../examples/26_thread_exceptions_vanish.py) + +### 27. `Lock` Is Not Reentrant + +A `threading.Lock` doesn't know which thread holds it. If a function holding the lock calls a helper that also acquires it, the thread waits forever for itself. + +```python +# The Trap +lock = threading.Lock() + +def update_balance(): + with lock: + audit() # Deadlock. + +def audit(): + with lock: + ... +``` + +**The fix:** Use `threading.RLock`, which the owning thread can acquire repeatedly. Better still, structure code so locks are taken in one place: public methods lock, private helpers assume the lock is held. + +[Example](../examples/27_lock_not_reentrant.py) + +### 28. `threading.local` Leaks Between asyncio Tasks + +`threading.local()` gives each *thread* its own storage, which is how many frameworks store "the current request". But every asyncio task runs on the same thread, so they all share one `threading.local`, and each task overwrites the others' values at every `await`. + +```python +# The Trap +request = threading.local() + +async def handle(user): + request.user = user + await asyncio.sleep(0) # Another task runs and sets request.user. + return request.user + +await asyncio.gather(handle("alice"), handle("bob")) # ['bob', 'bob'] +``` + +**The fix:** Use `contextvars.ContextVar`. Each asyncio task runs in its own copy of the context, so values don't bleed between tasks. Context variables also work correctly with threads. + +[Example](../examples/28_threading_local_in_asyncio.py) + +### 29. `fork()` Copies Locked Locks + +`os.fork()` copies the whole process memory, including every lock, but only the thread that called `fork()`. If another thread was holding a lock at that moment (inside `logging`, a connection pool, or `malloc` in a C extension), the child gets the lock in its locked state, and the thread that would release it doesn't exist in the child. The first time the child touches that lock, it hangs forever. + +```python +# The Trap +threading.Thread(target=log_metrics_forever).start() # Holds logging's lock now and then. +pool = multiprocessing.Pool() # fork() on Linux before 3.14. +# Child processes occasionally hang on their first log call. +``` + +**The fix:** Use the `spawn` or `forkserver` start method (`multiprocessing.get_context("forkserver")`), or create processes before starting any threads. Python 3.12 emits a `DeprecationWarning` when you fork a multi-threaded process, and since [Python 3.14](https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods) no platform defaults to `fork`. + +[Example](../examples/29_fork_with_lock_held.py) (POSIX only) + +### 30. Daemon Threads Skip Cleanup + +When the main thread exits, Python kills daemon threads abruptly. Their `finally` blocks and `with` exits never run, so buffers aren't flushed, files aren't closed and transactions aren't committed. + +```python +# The Trap +def writer(): + try: + while True: + buffer.append(next_record()) + finally: + flush(buffer) # Never runs at interpreter exit. + +threading.Thread(target=writer, daemon=True).start() +``` + +**The fix:** Use a normal thread with a stop signal, and join it on shutdown: + +```python +stop = threading.Event() +t = threading.Thread(target=writer, args=(stop,)) +t.start() +... +stop.set() +t.join() +``` + +[Example](../examples/30_daemon_threads_skip_cleanup.py) diff --git a/docs/data-systems.md b/docs/data-systems.md new file mode 100644 index 0000000..2742937 --- /dev/null +++ b/docs/data-systems.md @@ -0,0 +1,208 @@ +# Data Systems Pitfalls: Time, Numbers and Bytes + +[← Back to all pitfalls](../README.md) + +These are the pitfalls that matter when Python moves real data: market feeds, sensor readings, binary protocols and distributed workers. Each one silently corrupts values rather than crashing, so it surfaces as a reconciliation break, a dropped event or a job that fires at the wrong hour. + +## Contents + +| # | Pitfall | What you see | The fix | +|---|---------|--------------|---------| +| | **Hashing & numbers** | | | +| 43 | [`hash()` changes between processes](#43-hash-changes-between-processes) | Workers disagree on sharding | `zlib.crc32` / `hashlib` | +| 44 | [NaN breaks `in`, `max()` and `sorted()`](#44-nan-breaks-in-max-and-sorted) | Unsorted "sorted" output | Filter NaNs first | +| 45 | [Floats can't hold nanosecond timestamps](#45-floats-cant-hold-nanosecond-timestamps) | Timestamps off by ~100 ns | Integer nanoseconds | +| 46 | [`datetime` truncates to microseconds](#46-datetime-truncates-to-microseconds) | Distinct events compare equal | Integer nanoseconds | +| 47 | [Float prices](#47-float-prices-land-on-the-wrong-tick) | `int(1.15 * 100) == 114` | Fixed-point integers | +| | **Time zones** | | | +| 48 | [Aware datetime arithmetic is wall-clock](#48-aware-datetime-arithmetic-is-wall-clock-arithmetic) | Jobs fire an hour off, or in a loop | Do arithmetic in UTC | +| 49 | [A UTC offset is not a time zone](#49-a-utc-offset-is-not-a-time-zone) | Schedule drifts after DST | Convert to a `ZoneInfo` | +| 50 | [`datetime.utcnow()`](#50-datetimeutcnow-is-off-by-your-utc-offset) | Timestamps off by hours | `datetime.now(timezone.utc)` | +| | **Wire formats** | | | +| 51 | [`struct` pads and byte-swaps](#51-struct-pads-and-byte-swaps-by-default) | Garbage fields, wrong sizes | Always use `<`, `>` or `!` | +| 52 | [`json.dumps` emits invalid JSON](#52-jsondumps-emits-invalid-json) | Other languages reject the payload | `allow_nan=False` | + +--- + +## Hashing & numbers + +### 43. `hash()` Changes Between Processes + +The built-in `hash()` of `str` and `bytes` is salted with a random seed chosen at interpreter startup (to stop attackers from crafting hash collisions). The same symbol hashes differently in every process, so `hash(key) % n_workers` sends the same key to different shards depending on which process computes it. + +```python +# The Trap: run in two processes +shard = hash("AAPL") % 4 +# Worker A: [1, 0, 2, 2, 3, 2, 0, 1] (shards for 8 symbols) +# Worker B: [1, 2, 3, 1, 3, 0, 2, 0] +``` + +It also makes `set` iteration order of strings differ between runs, which makes tests flaky when they depend on it. (Integers hash to themselves and are not randomized, so the bug hides in tests that use numeric IDs.) + +**The fix:** Use a stable hash for anything that crosses a process boundary or gets persisted: `zlib.crc32(key.encode())`, or `hashlib.blake2b(key.encode(), digest_size=8)` for a better distribution. `PYTHONHASHSEED=0` disables randomization, but it's a workaround, not a design. + +[Example](../examples/43_hash_randomization_sharding.py) + +### 44. NaN Breaks `in`, `max()` and `sorted()` + +`float("nan")` isn't equal to anything, including itself. Every algorithm that relies on comparisons gives inconsistent results when a NaN is present, and none of them raise an error. + +```python +# The Trap +nan = float("nan") +nan in [nan] # True (`in` checks identity before ==) +float("nan") in [nan] # False (a different NaN object) +max([nan, 1, 2]) # nan +max([1, nan, 2]) # 2 (depends on position!) +sorted([3, nan, 1, 2]) # [3, nan, 1, 2] (not sorted) +``` + +**Why:** Sorting and `max()` assume that if `a < b` is false then `a >= b`. Every comparison with NaN is false, so that assumption breaks and the result depends on the order in which elements were compared. + +**The fix:** Remove or replace NaNs before aggregating: `[x for x in xs if not math.isnan(x)]`. Market data formats often use a sentinel for a missing price; Databento's [DBN](https://docs.rs/dbn) uses `i64::MAX` (`UNDEF_PRICE`), which its float conversion helpers turn into NaN. Handle that case at the boundary, before it reaches your analytics. + +[Example](../examples/44_nan_breaks_containers.py) + +### 45. Floats Can't Hold Nanosecond Timestamps + +A Unix timestamp in nanoseconds is about 1.7 × 10¹⁸, which needs 61 bits. A float has 53 bits of precision, so near today's date adjacent float values are about 238 ns apart. Converting nanoseconds to float seconds, whether with `ts / 1e9`, `time.time()` or a float column in a DataFrame, quietly rounds the timestamp. + +```python +# The Trap +ts_ns = 1_700_000_000_123_456_789 +int(ts_ns / 1e9 * 1e9) - ts_ns # -21 (nanoseconds lost) +(ts_ns + 100) / 1e9 == ts_ns / 1e9 # True (two events, one timestamp) +``` + +**The fix:** Keep integer nanoseconds end to end. Use `time.time_ns()` (and `time.monotonic_ns()` or `time.perf_counter_ns()` for latency) rather than `time.time()`, and use `divmod(ts_ns, 1_000_000_000)` to split seconds from nanoseconds. Python ints are exact at any size. + +[Example](../examples/45_float_nanosecond_timestamps.py) + +### 46. `datetime` Truncates to Microseconds + +`datetime` and `timedelta` have microsecond resolution. Exchange and network timestamps are nanoseconds, so converting to `datetime` drops the last three digits, and events that were distinct become equal. Sorting loses their true order, and de-duplicating by timestamp drops real records. + +```python +# The Trap +trade_a = 1_700_000_000_123_456_100 +trade_b = trade_a + 500 # 500 ns later +dt_a = datetime.fromtimestamp(trade_a / 1e9, timezone.utc) +dt_b = datetime.fromtimestamp(trade_b / 1e9, timezone.utc) +dt_a == dt_b # True +len({dt_a, dt_b}) # 1 (one trade de-duplicated away) +``` + +**The fix:** Treat integer nanoseconds as the source of truth and convert to `datetime` only for display. When you need datetime-like operations at nanosecond resolution, use `numpy.datetime64[ns]` or `pandas.Timestamp`. + +[Example](../examples/46_datetime_microsecond_truncation.py) + +### 47. Float Prices Land on the Wrong Tick + +Most decimal fractions, including `0.1` and `1.15`, have no exact binary representation. The error is invisible in `repr()` but appears as soon as you convert to an integer number of cents or ticks. + +```python +# The Trap +int(1.15 * 100) # 114 (1.15 * 100 is 114.99999999999999) +int(1.15 / 0.01) # 114 ticks, not 115 +0.1 + 0.2 == 0.3 # False +``` + +**The fix:** Store prices as fixed-point integers. Databento's [DBN format](https://docs.rs/dbn), for example, encodes every price as a signed 64-bit integer in units of 10⁻⁹, so `1.15` is `1_150_000_000` and tick arithmetic is exact. Convert to float only at the edge, for display. For human-entered amounts, `Decimal("1.15")` from a *string* is exact too. Note that `Decimal(1.15)` from a float keeps the float's error. + +[Example](../examples/47_float_prices.py) + +--- + +## Time zones + +### 48. Aware `datetime` Arithmetic Is Wall-Clock Arithmetic + +Adding a `timedelta` to an aware `datetime` adds to the *wall-clock* time and then looks up the offset again. Across a DST transition, "now + 3 minutes" can land before now or an hour later. Subtracting two datetimes that share a `tzinfo` is wall-clock too, so it ignores the transition. + +```python +# The Trap: clocks in Rome go back from 03:00 CEST to 02:00 CET +now = datetime(2026, 10, 25, 1, 4, 22, tzinfo=timezone.utc).astimezone(ZoneInfo("Europe/Rome")) +# 02:04:22+01:00, the second pass through 02:00 +now + timedelta(minutes=3) +# 02:07:22+02:00, i.e. 00:07 UTC: an hour in the PAST + +ny = ZoneInfo("America/New_York") +sat = datetime(2026, 3, 7, 12, tzinfo=ny) +sun = sat + timedelta(days=1) # Clocks spring forward overnight. +sun - sat # 1 day, 0:00:00, but only 23 hours passed +``` + +**Why:** Wall-clock arithmetic is what you want for calendar logic ("same time tomorrow") but not for durations ("in 3 minutes"). Python can't tell which one you mean. Arithmetic also resets `fold`, the attribute that says which pass through a repeated hour a time belongs to. + +**The fix:** Do duration arithmetic in UTC and convert to local time for display: `datetime.now(timezone.utc) + delay`. Use local-time arithmetic only for calendar rules, and test both DST transitions. + +**In the wild:** Home Assistant's `alert` integration scheduled repeat notifications with `now() + delay` in local time. During the repeated hour, the next notification landed in the past, fired at once and rescheduled itself the same way, so [notifiers were called in a tight loop](https://github.com/home-assistant/core/pull/188537) until the alert cleared. The fix was a one-word change from `now()` to `utcnow()`. + +[Example](../examples/48_aware_datetime_wall_clock_arithmetic.py) + +### 49. A UTC Offset Is Not a Time Zone + +`datetime.fromisoformat("2024-10-26T03:45:00+02:00")` returns a datetime with a *fixed* `+02:00` offset, not `Europe/Amsterdam`. Calendar arithmetic on it keeps `+02:00` forever, so after DST ends, "03:45 tomorrow" is really 02:45 local time. + +```python +# The Trap +last = datetime.fromisoformat("2024-10-26T03:45:00+02:00") # Parsed from storage. +nxt = last + timedelta(days=1) +nxt.astimezone(ZoneInfo("Europe/Amsterdam")) # 02:45+01:00 (an hour early) +``` + +**Why:** ISO 8601 strings store an offset, not the rules that produced it. Once serialized, a datetime has forgotten which time zone it came from. + +**The fix:** Convert to the real zone before doing calendar arithmetic: `last.astimezone(ZoneInfo("Europe/Amsterdam")) + timedelta(days=1)`. If the zone matters, store its name (`"Europe/Amsterdam"`) alongside the timestamp. + +**In the wild:** Home Assistant's [automatic backups ran an hour off schedule after every DST change](https://github.com/home-assistant/core/pull/188836), until Core restarted. The schedule iterator was seeded with the last backup time loaded from storage, whose tzinfo was the fixed offset in effect at the time. The fix converts the seed with `dt_util.as_local()` first. + +[Example](../examples/49_fixed_offset_not_timezone.py) + +### 50. `datetime.utcnow()` Is Off by Your UTC Offset + +`datetime.utcnow()` returns the current UTC time as a *naive* datetime, with no tzinfo. Every API that receives a naive datetime assumes it's in local time, so `.timestamp()` on it is wrong by your machine's UTC offset. On a UTC server it's correct by coincidence, which is why the bug ships. + +```python +# The Trap (on a machine in New York) +datetime.utcnow().timestamp() - time.time() # +14,400 or +18,000 s (4 or 5 hours off) +``` + +**The fix:** Use aware datetimes everywhere: `datetime.now(timezone.utc)` (or `datetime.now(UTC)` on Python 3.11+). `utcnow()` and `utcfromtimestamp()` are deprecated as of Python 3.12 for exactly this reason. Ruff's `DTZ003` flags them. + +[Example](../examples/50_utcnow_naive_timestamp.py) + +--- + +## Wire formats + +### 51. `struct` Pads and Byte-Swaps by Default + +A `struct` format without a prefix uses *native* mode: the host CPU's byte order and C alignment rules. `"BI"` (a byte then a 4-byte int) is 8 bytes, not 5, because native mode pads the int to a 4-byte boundary. Network protocols and most file formats are packed and have a fixed byte order. + +```python +# The Trap +struct.calcsize("BI") # 8 (3 padding bytes) +struct.unpack("BI", five_byte_record) # struct.error: unpack requires a buffer of 8 bytes +struct.unpack("=I", b"\x00\x00\x01\x2c") # (738263040,) on x86/ARM, not 300 +``` + +**The fix:** Always start format strings with an explicit byte order: `<` (little-endian), `>` or `!` (big-endian/network). These also disable padding, so `struct.calcsize(" before + + hb.cancel() + + +asyncio.run(main()) diff --git a/examples/23_asyncio_swallowed_cancellation.py b/examples/23_asyncio_swallowed_cancellation.py new file mode 100644 index 0000000..7fa3a96 --- /dev/null +++ b/examples/23_asyncio_swallowed_cancellation.py @@ -0,0 +1,45 @@ +"""Catching BaseException (or a bare except) swallows CancelledError.""" + +import asyncio + +stop = False + + +async def stubborn(): + while not stop: + try: + await asyncio.sleep(0.01) + except BaseException: # Same problem with a bare `except:` + pass # "Never crash" also means "never cancel". + + +async def cooperative(): + while True: + try: + await asyncio.sleep(0.01) + except Exception: # CancelledError is a BaseException, so it passes through. + pass + + +async def main(): + global stop + + # The trap + task = asyncio.create_task(stubborn()) + await asyncio.sleep(0) + task.cancel() + await asyncio.sleep(0.05) + print(f"cancelled task still running: {not task.done()}") # True + assert not task.done() + stop = True + await task + + # The fix: catch Exception, or re-raise CancelledError. + task = asyncio.create_task(cooperative()) + await asyncio.sleep(0) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert task.cancelled() + + +asyncio.run(main()) diff --git a/examples/24_asyncio_cross_thread_calls.py b/examples/24_asyncio_cross_thread_calls.py new file mode 100644 index 0000000..4e5fd86 --- /dev/null +++ b/examples/24_asyncio_cross_thread_calls.py @@ -0,0 +1,55 @@ +"""Event loop methods are not thread-safe; call_soon from another thread can stall.""" + +import asyncio +import threading +import time + + +async def main(): + loop = asyncio.get_running_loop() + + # The trap: a library thread schedules a callback with call_soon. + done = asyncio.Event() + + def library_callback(): + time.sleep(0.1) # Make sure the loop is idle and waiting. + loop.call_soon(done.set) # Queued, but the sleeping loop is never woken. + + loop.call_later(0.5, lambda: None) # Some unrelated timer wakes the loop later. + threading.Thread(target=library_callback).start() + start = time.monotonic() + await done.wait() + stalled = time.monotonic() - start + print(f"call_soon from a thread took {stalled:.2f}s") # ~0.50s, not ~0.10s + assert stalled >= 0.4 + + # In debug mode asyncio refuses outright. + loop.set_debug(True) + errors = [] + + def debug_callback(): + try: + loop.call_soon(done.set) + except RuntimeError as e: + errors.append(e) + + t = threading.Thread(target=debug_callback) + t.start() + t.join() + loop.set_debug(False) + assert errors, "expected RuntimeError in debug mode" + + # The fix: call_soon_threadsafe wakes the loop immediately. + done = asyncio.Event() + + def library_callback_fixed(): + time.sleep(0.1) + loop.call_soon_threadsafe(done.set) + + threading.Thread(target=library_callback_fixed).start() + start = time.monotonic() + await done.wait() + assert time.monotonic() - start < 0.4 + + +asyncio.run(main()) diff --git a/examples/25_thread_increment_not_atomic.py b/examples/25_thread_increment_not_atomic.py new file mode 100644 index 0000000..b4d0f35 --- /dev/null +++ b/examples/25_thread_increment_not_atomic.py @@ -0,0 +1,53 @@ +"""`counter += 1` is several bytecode steps; threads can interleave between them.""" + +import dis +import threading +import time + +# Why: one line of Python, separate load and store instructions. +ops = [i.opname for i in dis.get_instructions("counter += 1")] +print(ops) # [..., 'LOAD_NAME', ..., 'BINARY_OP', 'STORE_NAME', ...] +assert any(op.startswith("LOAD_NAME") for op in ops) +assert any(op.startswith("STORE_NAME") for op in ops) + +# The trap: read-modify-write from two threads. +counter = 0 + + +def work(): + global counter + for _ in range(1000): + current = counter + time.sleep(0) # Stands in for any thread switch between load and store. + counter = current + 1 + + +threads = [threading.Thread(target=work) for _ in range(2)] +for t in threads: + t.start() +for t in threads: + t.join() + +print(counter) # Far less than 2000 +assert counter < 2000 + +# The fix: guard the read-modify-write with a lock. +counter = 0 +lock = threading.Lock() + + +def work_fixed(): + global counter + for _ in range(1000): + with lock: + current = counter + time.sleep(0) + counter = current + 1 + + +threads = [threading.Thread(target=work_fixed) for _ in range(2)] +for t in threads: + t.start() +for t in threads: + t.join() +assert counter == 2000 diff --git a/examples/26_thread_exceptions_vanish.py b/examples/26_thread_exceptions_vanish.py new file mode 100644 index 0000000..b6073e3 --- /dev/null +++ b/examples/26_thread_exceptions_vanish.py @@ -0,0 +1,35 @@ +"""Exceptions in threads don't reach the caller; in executors they vanish entirely.""" + +import threading +from concurrent.futures import ThreadPoolExecutor + + +def job(): + raise ValueError("boom") + + +# The trap (1): Thread prints a traceback to stderr, then join() returns normally. +seen = [] +original_hook = threading.excepthook +threading.excepthook = lambda args: seen.append(args.exc_type) +t = threading.Thread(target=job) +t.start() +t.join() # No exception here. +threading.excepthook = original_hook +print("main thread carried on") +assert seen == [ValueError] + +# The trap (2): an executor stores the exception and prints nothing at all. +with ThreadPoolExecutor() as pool: + future = pool.submit(job) +assert isinstance(future.exception(), ValueError) + +# The fix: always collect results. future.result() re-raises in the caller. +with ThreadPoolExecutor() as pool: + future = pool.submit(job) +try: + future.result() +except ValueError as e: + print(f"surfaced: {e}") +else: + raise AssertionError("expected ValueError") diff --git a/examples/27_lock_not_reentrant.py b/examples/27_lock_not_reentrant.py new file mode 100644 index 0000000..fed3dd4 --- /dev/null +++ b/examples/27_lock_not_reentrant.py @@ -0,0 +1,27 @@ +"""threading.Lock deadlocks if the same thread acquires it twice.""" + +import threading + +# The trap +lock = threading.Lock() + + +def update_balance(): + with lock: + return audit() # Calls a helper that also takes the lock. + + +def audit(): + # With `with lock:` this would hang forever; a timeout lets us observe it. + acquired = lock.acquire(timeout=0.1) + if acquired: + lock.release() + return acquired + + +print(update_balance()) # False (deadlocked on itself) +assert update_balance() is False + +# The fix: RLock can be re-acquired by the thread that holds it. +lock = threading.RLock() +assert update_balance() is True diff --git a/examples/28_threading_local_in_asyncio.py b/examples/28_threading_local_in_asyncio.py new file mode 100644 index 0000000..c0b1a30 --- /dev/null +++ b/examples/28_threading_local_in_asyncio.py @@ -0,0 +1,39 @@ +"""threading.local is shared by every asyncio task on the same thread.""" + +import asyncio +import contextvars +import threading + +# The trap +request = threading.local() + + +async def handle(user): + request.user = user + await asyncio.sleep(0) # Another task runs and overwrites request.user. + return request.user + + +async def main(): + return await asyncio.gather(handle("alice"), handle("bob")) + + +results = asyncio.run(main()) +print(results) # ['bob', 'bob'] (alice's request now belongs to bob) +assert results == ["bob", "bob"] + +# The fix: contextvars gives each task its own copy. +current_user = contextvars.ContextVar("current_user") + + +async def handle_fixed(user): + current_user.set(user) + await asyncio.sleep(0) + return current_user.get() + + +async def main_fixed(): + return await asyncio.gather(handle_fixed("alice"), handle_fixed("bob")) + + +assert asyncio.run(main_fixed()) == ["alice", "bob"] diff --git a/examples/29_fork_with_lock_held.py b/examples/29_fork_with_lock_held.py new file mode 100644 index 0000000..fe3b2d5 --- /dev/null +++ b/examples/29_fork_with_lock_held.py @@ -0,0 +1,41 @@ +"""fork() copies a lock held by another thread, but not the thread that would release it.""" + +import os +import sys +import threading +import time +import warnings + +if not hasattr(os, "fork"): + print("os.fork is not available on this platform; skipping") + sys.exit(0) + +# Python 3.12+ warns about forking a multi-threaded process. +warnings.simplefilter("ignore", DeprecationWarning) + +# The trap: a background thread holds a lock (think logging, or a connection pool). +lock = threading.Lock() +held = threading.Event() + + +def background(): + with lock: + held.set() + time.sleep(1) + + +threading.Thread(target=background).start() +held.wait() + +pid = os.fork() +if pid == 0: + # Child: the lock is copied in its locked state, and nobody will ever release it. + os._exit(0 if lock.acquire(timeout=0.5) else 1) + +_, status = os.waitpid(pid, 0) +child_got_lock = os.waitstatus_to_exitcode(status) == 0 +print(f"child acquired the lock: {child_got_lock}") # False (deadlock) +assert not child_got_lock + +# The fix: use the "spawn" or "forkserver" start method (the default on every +# platform since Python 3.14), or fork before starting any threads. diff --git a/examples/30_daemon_threads_skip_cleanup.py b/examples/30_daemon_threads_skip_cleanup.py new file mode 100644 index 0000000..f12e919 --- /dev/null +++ b/examples/30_daemon_threads_skip_cleanup.py @@ -0,0 +1,44 @@ +"""Daemon threads are killed at exit without running finally blocks.""" + +import subprocess +import sys +import textwrap + +# The trap +child = textwrap.dedent(""" + import threading, time + + def flush_buffer(): + try: + time.sleep(10) + finally: + print("flushed") # Never runs. + + threading.Thread(target=flush_buffer, daemon=True).start() + time.sleep(0.1) + print("main done") +""") +out = subprocess.run([sys.executable, "-c", child], capture_output=True, text=True, timeout=30) +print(out.stdout.splitlines()) # ['main done'] +assert out.stdout.splitlines() == ["main done"] + +# The fix: a normal thread with a stop signal, joined on shutdown. +child_fixed = textwrap.dedent(""" + import threading + + stop = threading.Event() + + def flush_buffer(): + try: + stop.wait(10) + finally: + print("flushed") + + t = threading.Thread(target=flush_buffer) + t.start() + print("main done") + stop.set() + t.join() +""") +out = subprocess.run([sys.executable, "-c", child_fixed], capture_output=True, text=True, timeout=30) +assert out.stdout.splitlines() == ["main done", "flushed"] diff --git a/examples/31_property_getattr_masking.py b/examples/31_property_getattr_masking.py new file mode 100644 index 0000000..2457995 --- /dev/null +++ b/examples/31_property_getattr_masking.py @@ -0,0 +1,44 @@ +"""An AttributeError inside a property makes Python fall back to __getattr__.""" + + +class Device: + pass + + +# The trap +class SensorProxy: + def __init__(self, device): + self._device = device + + @property + def temperature(self): + return self._raw_reading * 0.1 # Bug: _raw_reading doesn't exist. + + def __getattr__(self, name): + return getattr(self._device, name) # Delegate unknown attributes. + + +try: + SensorProxy(Device()).temperature +except AttributeError as e: + message = str(e) + +print(message) # 'Device' object has no attribute 'temperature' +assert "temperature" in message +assert "_raw_reading" not in message # The real bug is nowhere in the error. + + +# The fix: don't let AttributeError escape a property when __getattr__ exists. +class SensorProxyFixed(SensorProxy): + @property + def temperature(self): + try: + return self._raw_reading * 0.1 + except AttributeError as e: + raise RuntimeError("failed to compute temperature") from e + + +try: + SensorProxyFixed(Device()).temperature +except RuntimeError as e: + assert "_raw_reading" in str(e.__cause__) diff --git a/examples/32_lru_cache_method_leak.py b/examples/32_lru_cache_method_leak.py new file mode 100644 index 0000000..d685094 --- /dev/null +++ b/examples/32_lru_cache_method_leak.py @@ -0,0 +1,37 @@ +"""@lru_cache on a method caches `self`, so instances are never freed.""" + +import functools +import gc +import weakref + + +# The trap +class Portfolio: + @functools.lru_cache(maxsize=None) + def value(self): + return 42 + + +p = Portfolio() +p.value() +ref = weakref.ref(p) +del p +gc.collect() +print(f"instance still alive: {ref() is not None}") # True (leaked) +assert ref() is not None +assert Portfolio.value.cache_info().currsize == 1 + + +# The fix: cache per instance with cached_property (or a dict on the instance). +class PortfolioFixed: + @functools.cached_property + def value(self): + return 42 + + +p = PortfolioFixed() +assert p.value == 42 +ref = weakref.ref(p) +del p +gc.collect() +assert ref() is None diff --git a/examples/33_eq_disables_hash.py b/examples/33_eq_disables_hash.py new file mode 100644 index 0000000..ed0436b --- /dev/null +++ b/examples/33_eq_disables_hash.py @@ -0,0 +1,40 @@ +"""Defining __eq__ without __hash__ makes a class unhashable.""" + +from dataclasses import dataclass + + +# The trap +class Symbol: + def __init__(self, ticker): + self.ticker = ticker + + def __eq__(self, other): + return self.ticker == other.ticker + + +print(Symbol.__hash__) # None +assert Symbol.__hash__ is None +try: + {Symbol("AAPL")} +except TypeError as e: + print(e) # unhashable type: 'Symbol' +else: + raise AssertionError("expected TypeError") + + +# A plain @dataclass does the same: eq=True sets __hash__ to None. +@dataclass +class SymbolData: + ticker: str + + +assert SymbolData.__hash__ is None + + +# The fix: define a matching __hash__, or use a frozen dataclass. +@dataclass(frozen=True) +class SymbolFixed: + ticker: str + + +assert {SymbolFixed("AAPL"), SymbolFixed("AAPL")} == {SymbolFixed("AAPL")} diff --git a/examples/34_exit_swallows_exceptions.py b/examples/34_exit_swallows_exceptions.py new file mode 100644 index 0000000..a559dc2 --- /dev/null +++ b/examples/34_exit_swallows_exceptions.py @@ -0,0 +1,41 @@ +"""A context manager whose __exit__ returns a truthy value swallows exceptions.""" + + +# The trap +class Transaction: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return self.rollback() if exc_type else self.commit() + + def rollback(self): + return True # "Rollback succeeded"... which also means "suppress the error". + + def commit(self): + return True + + +with Transaction(): + raise ValueError("invalid order") + +print("execution continued as if nothing happened") + + +# The fix: return None (or False) unless you deliberately mean to suppress. +class TransactionFixed(Transaction): + def __exit__(self, exc_type, exc, tb): + if exc_type: + self.rollback() + else: + self.commit() + return None + + +try: + with TransactionFixed(): + raise ValueError("invalid order") +except ValueError: + pass +else: + raise AssertionError("expected ValueError") diff --git a/examples/35_weakref_bound_method.py b/examples/35_weakref_bound_method.py new file mode 100644 index 0000000..4f9a874 --- /dev/null +++ b/examples/35_weakref_bound_method.py @@ -0,0 +1,23 @@ +"""A weak reference to a bound method dies immediately.""" + +import weakref + + +class Listener: + def on_tick(self): + return "tick" + + +listener = Listener() + +# The trap: each attribute access builds a new, temporary bound method. +assert listener.on_tick is not listener.on_tick +callback = weakref.ref(listener.on_tick) +print(callback()) # None (the listener is alive, but the callback is gone) +assert callback() is None + +# The fix: WeakMethod tracks the instance and the function separately. +callback = weakref.WeakMethod(listener.on_tick) +assert callback()() == "tick" +del listener +assert callback() is None # Now it dies with the instance, as intended. diff --git a/examples/36_mock_patch_location.py b/examples/36_mock_patch_location.py new file mode 100644 index 0000000..29629fa --- /dev/null +++ b/examples/36_mock_patch_location.py @@ -0,0 +1,23 @@ +"""mock.patch must target the name where it is looked up, not where it is defined.""" + +import sys +import types +from unittest import mock + +# Two tiny modules: report.py does `from prices import fetch`. +prices = types.ModuleType("prices") +exec("def fetch():\n return 'live'", prices.__dict__) +sys.modules["prices"] = prices + +report = types.ModuleType("report") +exec("from prices import fetch\ndef run():\n return fetch()", report.__dict__) +sys.modules["report"] = report + +# The trap: patching the definition doesn't touch report's copy of the name. +with mock.patch("prices.fetch", return_value="mocked"): + print(report.run()) # 'live' (the test just hit the real service) + assert report.run() == "live" + +# The fix: patch the module that uses the name. +with mock.patch("report.fetch", return_value="mocked"): + assert report.run() == "mocked" diff --git a/examples/37_main_module_imported_twice.py b/examples/37_main_module_imported_twice.py new file mode 100644 index 0000000..782bef1 --- /dev/null +++ b/examples/37_main_module_imported_twice.py @@ -0,0 +1,31 @@ +"""A script that imports itself runs twice, as __main__ and as a module.""" + +import pathlib +import subprocess +import sys +import tempfile +import textwrap + +app = textwrap.dedent(""" + print("module body ran as", __name__) + + class Config: + pass + + if __name__ == "__main__": + import app # Or any module that imports app. + print(isinstance(Config(), app.Config)) +""") + +with tempfile.TemporaryDirectory() as d: + path = pathlib.Path(d, "app.py") + path.write_text(app) + out = subprocess.run([sys.executable, str(path)], capture_output=True, text=True, cwd=d) + +lines = out.stdout.splitlines() +print(lines) +# ['module body ran as __main__', 'module body ran as app', 'False'] +assert lines == ["module body ran as __main__", "module body ran as app", "False"] + +# The fix: keep the entry point thin. Put code in modules and have a small +# main.py (or `python -m package`) call into them. diff --git a/examples/38_assert_stripped_by_O.py b/examples/38_assert_stripped_by_O.py new file mode 100644 index 0000000..9420897 --- /dev/null +++ b/examples/38_assert_stripped_by_O.py @@ -0,0 +1,27 @@ +"""`python -O` deletes assert statements, along with any checks inside them.""" + +import subprocess +import sys + +code = """ +def withdraw(balance, amount): + assert amount <= balance, "insufficient funds" + return balance - amount + +print(withdraw(100, 500)) +""" + +normal = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) +optimized = subprocess.run([sys.executable, "-O", "-c", code], capture_output=True, text=True) + +assert "AssertionError: insufficient funds" in normal.stderr +print(optimized.stdout.strip()) # -400 (the check silently disappeared) +assert optimized.stdout.strip() == "-400" + +# The fix: validate with an explicit raise. Use assert only for internal invariants. +code_fixed = code.replace( + 'assert amount <= balance, "insufficient funds"', + 'if amount > balance:\n raise ValueError("insufficient funds")', +) +optimized = subprocess.run([sys.executable, "-O", "-c", code_fixed], capture_output=True, text=True) +assert "ValueError: insufficient funds" in optimized.stderr diff --git a/examples/39_class_body_comprehension_scope.py b/examples/39_class_body_comprehension_scope.py new file mode 100644 index 0000000..d409033 --- /dev/null +++ b/examples/39_class_body_comprehension_scope.py @@ -0,0 +1,38 @@ +"""Comprehensions in a class body can't see class variables, and may find globals instead.""" + +rate = 0.0 # An unrelated module-level name. + + +# The trap +class Fees: + rate = 0.01 + tiers = [rate * n for n in (1, 2, 3)] + + +print(Fees.tiers) # [0.0, 0.0, 0.0] (used the global, not Fees.rate) +assert Fees.tiers == [0.0, 0.0, 0.0] + +# Without the global it's a NameError instead. +del rate +try: + + class Fees2: + rate = 0.01 + tiers = [rate * n for n in (1, 2, 3)] + +except NameError as e: + print(e) # name 'rate' is not defined +else: + raise AssertionError("expected NameError") + + +# The fix: a plain loop runs in the class scope. +class FeesFixed: + rate = 0.01 + tiers = [] + for n in (1, 2, 3): + tiers.append(rate * n) + del n + + +assert FeesFixed.tiers == [0.01, 0.02, 0.03] diff --git a/examples/40_locals_writes_ignored.py b/examples/40_locals_writes_ignored.py new file mode 100644 index 0000000..b17436a --- /dev/null +++ b/examples/40_locals_writes_ignored.py @@ -0,0 +1,29 @@ +"""Writing to locals() or exec-ing an assignment doesn't change a function's variables.""" + + +# The trap +def configure(): + debug = False + locals()["debug"] = True + return debug + + +def configure_exec(): + debug = False + exec("debug = True") + return debug + + +print(configure(), configure_exec()) # False False +assert configure() is False +assert configure_exec() is False + + +# The fix: use an explicit dict when names are dynamic. +def configure_fixed(): + settings = {"debug": False} + exec("debug = True", {}, settings) + return settings["debug"] + + +assert configure_fixed() is True diff --git a/examples/41_id_reuse.py b/examples/41_id_reuse.py new file mode 100644 index 0000000..9585e08 --- /dev/null +++ b/examples/41_id_reuse.py @@ -0,0 +1,35 @@ +"""id() values are reused once an object is freed, so they aren't unique over time.""" + +# The trap +print(id([]) == id([])) # True (the second list reuses the first one's memory) +assert id([]) == id([]) +assert [] is not [] + +# A cache keyed on id() hands back stale results for new objects. +cache = {} + + +def describe(obj): + key = id(obj) + if key not in cache: + cache[key] = repr(obj) + return cache[key] + + +describe([1, 2, 3]) # The temporary list is freed right after this call. +print(describe([4, 5, 6])) # '[1, 2, 3]' +assert describe([4, 5, 6]) == "[1, 2, 3]" + +# The fix: key on the value itself, or keep the object alive (e.g. WeakKeyDictionary). +cache = {} + + +def describe_fixed(obj): + key = tuple(obj) + if key not in cache: + cache[key] = repr(obj) + return cache[key] + + +describe_fixed([1, 2, 3]) +assert describe_fixed([4, 5, 6]) == "[4, 5, 6]" diff --git a/examples/42_int_str_digit_limit.py b/examples/42_int_str_digit_limit.py new file mode 100644 index 0000000..239d256 --- /dev/null +++ b/examples/42_int_str_digit_limit.py @@ -0,0 +1,32 @@ +"""Converting between int and str fails above 4300 digits (CVE-2020-10735).""" + +import json +import sys + +# The trap: arithmetic is unlimited, but decimal conversion is capped. +big = 10**5000 +try: + str(big) +except ValueError as e: + print(e) # Exceeds the limit (4300 digits) for integer string conversion... +else: + raise AssertionError("expected ValueError") + +# It also applies to parsing, including json.loads. +try: + json.loads("9" * 5000) +except ValueError: + pass +else: + raise AssertionError("expected ValueError") + +# Power-of-two bases are linear time, so they are not limited. +assert hex(big).startswith("0x") + +# The fix: raise the limit deliberately where you trust the input. +old_limit = sys.get_int_max_str_digits() +sys.set_int_max_str_digits(0) # 0 means unlimited +try: + assert len(str(big)) == 5001 +finally: + sys.set_int_max_str_digits(old_limit) diff --git a/examples/43_hash_randomization_sharding.py b/examples/43_hash_randomization_sharding.py new file mode 100644 index 0000000..959d163 --- /dev/null +++ b/examples/43_hash_randomization_sharding.py @@ -0,0 +1,26 @@ +"""hash() of str and bytes changes between processes, so it can't route data.""" + +import subprocess +import sys + +symbols = ["AAPL", "MSFT", "NVDA", "TSLA", "AMZN", "GOOG", "META", "NFLX"] + + +def shards_in_new_process(expr, seed): + code = f"symbols = {symbols!r}\nimport zlib\nprint([{expr} for s in symbols])" + env = {"PYTHONHASHSEED": seed} + out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env) + return out.stdout.strip() + + +# The trap: two workers (two processes) disagree about where a symbol lives. +worker_a = shards_in_new_process("hash(s) % 4", "1") +worker_b = shards_in_new_process("hash(s) % 4", "2") +print(worker_a) +print(worker_b) +assert worker_a != worker_b + +# The fix: use a stable hash such as zlib.crc32 or hashlib. +worker_a = shards_in_new_process("zlib.crc32(s.encode()) % 4", "1") +worker_b = shards_in_new_process("zlib.crc32(s.encode()) % 4", "2") +assert worker_a == worker_b diff --git a/examples/44_nan_breaks_containers.py b/examples/44_nan_breaks_containers.py new file mode 100644 index 0000000..43dffe1 --- /dev/null +++ b/examples/44_nan_breaks_containers.py @@ -0,0 +1,25 @@ +"""NaN is not equal to itself, which breaks membership, max() and sorting.""" + +import math + +nan = float("nan") + +# The trap +assert nan != nan +assert nan in [nan] # `in` checks identity before equality... +assert float("nan") not in [nan] # ...so a different NaN object isn't found. + +# max() and min() depend on where the NaN sits. +print(max([nan, 1, 2]), max([1, nan, 2])) # nan 2 +assert math.isnan(max([nan, 1, 2])) +assert max([1, nan, 2]) == 2 + +# sorted() silently returns an unsorted list. +result = sorted([3, nan, 1, 2]) +print(result) # [3, nan, 1, 2] +assert [x for x in result if not math.isnan(x)] != [1, 2, 3] + +# The fix: remove or replace NaNs before aggregating or sorting. +clean = [x for x in [3, nan, 1, 2] if not math.isnan(x)] +assert sorted(clean) == [1, 2, 3] +assert max(clean) == 3 diff --git a/examples/45_float_nanosecond_timestamps.py b/examples/45_float_nanosecond_timestamps.py new file mode 100644 index 0000000..a123d99 --- /dev/null +++ b/examples/45_float_nanosecond_timestamps.py @@ -0,0 +1,20 @@ +"""A float can't hold a nanosecond Unix timestamp exactly.""" + +import time + +ts_ns = 1_700_000_000_123_456_789 # 2023-11-14, nanosecond precision + +# The trap: a float64 has 53 bits of precision; this needs about 61. +seconds = ts_ns / 1e9 +round_trip = int(seconds * 1e9) +print(round_trip - ts_ns) # -21 (nanoseconds gained or lost) +assert round_trip != ts_ns + +# Near today's epoch, floats are ~238 ns apart, so distinct events collide. +assert (ts_ns + 100) / 1e9 == ts_ns / 1e9 + +# The fix: keep integer nanoseconds end to end. +now_ns = time.time_ns() # not time.time() +assert isinstance(now_ns, int) +whole, frac = divmod(ts_ns, 1_000_000_000) +assert (whole, frac) == (1_700_000_000, 123_456_789) diff --git a/examples/46_datetime_microsecond_truncation.py b/examples/46_datetime_microsecond_truncation.py new file mode 100644 index 0000000..4387c58 --- /dev/null +++ b/examples/46_datetime_microsecond_truncation.py @@ -0,0 +1,20 @@ +"""datetime only stores microseconds, so nanosecond events collapse together.""" + +from datetime import datetime, timezone + +UTC = timezone.utc # datetime.UTC needs Python 3.11+ + +# The trap: two trades 500 ns apart. +trade_a = 1_700_000_000_123_456_100 +trade_b = trade_a + 500 + +dt_a = datetime.fromtimestamp(trade_a / 1e9, UTC) +dt_b = datetime.fromtimestamp(trade_b / 1e9, UTC) +print(dt_a, dt_b) # identical +assert dt_a == dt_b +assert len({dt_a, dt_b}) == 1 # De-duplicating by timestamp drops a trade. + +# The fix: keep integer nanoseconds as the source of truth and convert only for +# display (or use a nanosecond type such as numpy.datetime64[ns] / pandas.Timestamp). +assert trade_a != trade_b +assert len({trade_a, trade_b}) == 2 diff --git a/examples/47_float_prices.py b/examples/47_float_prices.py new file mode 100644 index 0000000..7c190b2 --- /dev/null +++ b/examples/47_float_prices.py @@ -0,0 +1,23 @@ +"""Binary floats can't represent most decimal prices exactly.""" + +from decimal import Decimal + +# The trap +print(int(1.15 * 100)) # 114 (1.15 * 100 is 114.99999999999999) +assert int(1.15 * 100) == 114 +assert 0.1 + 0.2 != 0.3 + +# Converting a price to ticks can land on the wrong tick. +price, tick = 1.15, 0.01 +assert int(price / tick) == 114 + +# The fix: fixed-point integers (e.g. price * 10**9 as an int, as Databento's DBN +# format does), or Decimal built from strings. +PRICE_SCALE = 1_000_000_000 +price_fixed = 1_150_000_000 # 1.15 +tick_fixed = 10_000_000 # 0.01 +assert price_fixed // tick_fixed == 115 +assert price_fixed / PRICE_SCALE == 1.15 # Convert to float only for display. + +assert Decimal("1.15") * 100 == 115 +assert Decimal(1.15) != Decimal("1.15") # A Decimal from a float keeps its error. diff --git a/examples/48_aware_datetime_wall_clock_arithmetic.py b/examples/48_aware_datetime_wall_clock_arithmetic.py new file mode 100644 index 0000000..e22a247 --- /dev/null +++ b/examples/48_aware_datetime_wall_clock_arithmetic.py @@ -0,0 +1,28 @@ +"""Arithmetic on aware datetimes is wall-clock arithmetic, which breaks across DST.""" + +from datetime import datetime, timedelta, timezone +from zoneinfo import ZoneInfo + +UTC = timezone.utc # datetime.UTC needs Python 3.11+ + +rome = ZoneInfo("Europe/Rome") + +# The trap (Home Assistant core PR #188537): during the repeated hour after +# clocks go back, now() + 3 minutes lands *before* now. +now_utc = datetime(2026, 10, 25, 1, 4, 22, tzinfo=UTC) +now_local = now_utc.astimezone(rome) # 02:04:22+01:00 (second pass through 02:00) +next_alert = now_local + timedelta(minutes=3) +print(now_local, "->", next_alert) # 02:04:22+01:00 -> 02:07:22+02:00 +assert next_alert < now_utc # Scheduled in the past, so it fires again at once. + +# Subtraction is wall-clock too when both sides share a tzinfo. +ny = ZoneInfo("America/New_York") +open_sat = datetime(2026, 3, 7, 12, 0, tzinfo=ny) +open_sun = open_sat + timedelta(days=1) # Clocks spring forward overnight. +print(open_sun - open_sat) # 1 day, 0:00:00 (but only 23 real hours passed) +assert open_sun - open_sat == timedelta(days=1) +assert open_sun.astimezone(UTC) - open_sat.astimezone(UTC) == timedelta(hours=23) + +# The fix: do arithmetic in UTC, convert to local time only for display. +next_alert = now_utc + timedelta(minutes=3) +assert next_alert > now_utc diff --git a/examples/49_fixed_offset_not_timezone.py b/examples/49_fixed_offset_not_timezone.py new file mode 100644 index 0000000..d77d991 --- /dev/null +++ b/examples/49_fixed_offset_not_timezone.py @@ -0,0 +1,20 @@ +"""A parsed "+02:00" is a fixed offset, not a time zone, so it ignores DST.""" + +from datetime import datetime, timedelta, timezone +from zoneinfo import ZoneInfo + +amsterdam = ZoneInfo("Europe/Amsterdam") + +# The trap (Home Assistant core PR #188836): a stored timestamp is parsed back +# and used to schedule tomorrow's 03:45 backup. +last_backup = datetime.fromisoformat("2024-10-26T03:45:00+02:00") +assert last_backup.tzinfo == timezone(timedelta(hours=2)) # Not Europe/Amsterdam. + +next_backup = last_backup + timedelta(days=1) +print(next_backup.astimezone(amsterdam)) # 2024-10-27 02:45:00+01:00 (an hour early) +assert next_backup.astimezone(amsterdam).hour == 2 + +# The fix: convert to the real zone before doing calendar arithmetic. +next_backup = last_backup.astimezone(amsterdam) + timedelta(days=1) +assert next_backup.hour == 3 +assert next_backup.utcoffset() == timedelta(hours=1) diff --git a/examples/50_utcnow_naive_timestamp.py b/examples/50_utcnow_naive_timestamp.py new file mode 100644 index 0000000..6d3710e --- /dev/null +++ b/examples/50_utcnow_naive_timestamp.py @@ -0,0 +1,30 @@ +"""datetime.utcnow() returns a naive datetime that .timestamp() treats as local time.""" + +import os +import sys +import time +import warnings +from datetime import datetime, timezone + +UTC = timezone.utc # datetime.UTC needs Python 3.11+ + +if not hasattr(time, "tzset"): + print("time.tzset is not available on this platform; skipping") + sys.exit(0) + +os.environ["TZ"] = "America/New_York" +time.tzset() + +# The trap +with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) # Deprecated in 3.12. + naive = datetime.utcnow() + +assert naive.tzinfo is None +error_hours = (naive.timestamp() - time.time()) / 3600 +print(f"off by {error_hours:.0f} hours") # off by 4 or 5 hours +assert abs(error_hours) > 3 + +# The fix: use aware datetimes. +aware = datetime.now(UTC) +assert abs(aware.timestamp() - time.time()) < 5 diff --git a/examples/51_struct_native_alignment.py b/examples/51_struct_native_alignment.py new file mode 100644 index 0000000..849317d --- /dev/null +++ b/examples/51_struct_native_alignment.py @@ -0,0 +1,23 @@ +"""struct's default format uses native padding and byte order.""" + +import struct + +# The trap: a 5-byte wire record (uint8 side, uint32 size). +assert struct.calcsize("BI") == 8 # Native alignment pads 3 bytes after the B. +assert struct.calcsize("I", network_uint32)[0] == 300 + +# The fix: always state byte order and size with <, > or !. +assert struct.unpack(" Date: Fri, 30 Oct 2026 15:01:48 +0000 Subject: [PATCH 2/2] Address review: example 24 hang in debug mode, -W error claim - examples/24: disable inherited asyncio debug mode before the stall demo. Under PYTHONASYNCIODEBUG=1 or -X dev, call_soon from a thread raised in the worker and the example hung forever. Waits now have a 5s timeout, so a regression fails instead of hanging. - docs/concurrency.md #21: -W error::RuntimeWarning does not fail on a never-awaited coroutine, because the warning fires during finalization and becomes an unraisable exception. Document the pytest filters that do fail (verified with pytest 9.2). - CI: also run every example under -X dev. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/verify.yml | 2 ++ docs/concurrency.md | 4 +++- examples/24_asyncio_cross_thread_calls.py | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 8b4393d..8a2713b 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -23,6 +23,8 @@ jobs: for f in examples/*.py; do echo "::group::$f" python "$f" || { echo "::error file=$f::example failed"; status=1; } + # Development mode enables asyncio debug checks, which change behavior. + python -X dev "$f" > /dev/null || { echo "::error file=$f::example failed under -X dev"; status=1; } echo "::endgroup::" done exit $status diff --git a/docs/concurrency.md b/docs/concurrency.md index d099b18..96b1a0b 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -68,7 +68,9 @@ async def checkout(order): At best, Python prints `RuntimeWarning: coroutine 'save' was never awaited` when the coroutine is garbage-collected, which is easy to miss in production logs. -**The fix:** `await save(order)`. Type checkers catch this reliably (pyright's `reportUnusedCoroutine`, mypy's `unused-coroutine`), and running tests with `-W error::RuntimeWarning` turns the warning into a failure. +**The fix:** `await save(order)`. Type checkers catch this reliably (pyright's `reportUnusedCoroutine`, mypy's `unused-coroutine`). + +Don't rely on `-W error::RuntimeWarning` alone. The warning is raised while the coroutine is being garbage-collected, where an exception can't propagate. Python prints it as `Exception ignored in: ` and carries on with exit status 0. Under pytest, the warning only fails the test with both filters together, `-W error::RuntimeWarning -W error::pytest.PytestUnraisableExceptionWarning`, or with `filterwarnings = ["error"]` in your pytest config. [Example](../examples/21_asyncio_forgotten_await.py) diff --git a/examples/24_asyncio_cross_thread_calls.py b/examples/24_asyncio_cross_thread_calls.py index 4e5fd86..4146dce 100644 --- a/examples/24_asyncio_cross_thread_calls.py +++ b/examples/24_asyncio_cross_thread_calls.py @@ -7,6 +7,8 @@ async def main(): loop = asyncio.get_running_loop() + # Debug mode (PYTHONASYNCIODEBUG=1, -X dev) raises instead of stalling; see below. + loop.set_debug(False) # The trap: a library thread schedules a callback with call_soon. done = asyncio.Event() @@ -18,7 +20,7 @@ def library_callback(): loop.call_later(0.5, lambda: None) # Some unrelated timer wakes the loop later. threading.Thread(target=library_callback).start() start = time.monotonic() - await done.wait() + await asyncio.wait_for(done.wait(), timeout=5) stalled = time.monotonic() - start print(f"call_soon from a thread took {stalled:.2f}s") # ~0.50s, not ~0.10s assert stalled >= 0.4 @@ -48,7 +50,7 @@ def library_callback_fixed(): threading.Thread(target=library_callback_fixed).start() start = time.monotonic() - await done.wait() + await asyncio.wait_for(done.wait(), timeout=5) assert time.monotonic() - start < 0.4