diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 5ef7d5f..aa247dc 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -4,5 +4,5 @@ contact_links: url: https://github.com/LitePy/PythonWrap/security/advisories/new about: Please report security issues privately, not in a public issue. - name: Documentation - url: https://python-wrap.readthedocs.io + url: https://pythonwrap.readthedocs.io about: Check the docs first; most questions are answered there. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fd8cbb..e887860 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ -# Continuous integration. +# Continuous integration after changes land on main. # -# Four jobs: lint (seconds), the test matrix, mypy, and a packaging check. +# Five jobs: lint (seconds), the test matrix, free-threaded Python with the stress +# suite, mypy, and a packaging check. # # python-wrap is pure Python with no dependencies, so the matrix is wide rather than # deep: every interpreter the classifiers name, on all three operating systems. Time and @@ -12,8 +13,6 @@ name: CI on: push: branches: [main] - pull_request: - workflow_dispatch: # A force-push should not leave the previous run burning minutes on a dead commit. concurrency: @@ -43,10 +42,10 @@ jobs: - run: uv sync --group dev --frozen - name: ruff check - run: uv run --no-sync ruff check src tests + run: uv run --no-sync ruff check src tests scripts/check_installed.py - name: ruff format - run: uv run --no-sync ruff format --check --diff src tests + run: uv run --no-sync ruff format --check --diff src tests scripts/check_installed.py test: name: py${{ matrix.python-version }} · ${{ matrix.os }} @@ -84,6 +83,36 @@ jobs: path: coverage.xml if-no-files-found: warn + free-threaded: + # CPython 3.14t with the GIL off, plus the thread-safety stress suite, which only + # runs when PW_STRESS=1. This is where real data races show up. + name: py3.14t (free-threaded) · ubuntu-latest + runs-on: ubuntu-latest + env: + PYTHON_GIL: "0" + steps: + - uses: actions/checkout@v7 + + - name: Install uv + uses: astral-sh/setup-uv@v10.3.2 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.14t" + + - run: uv sync --group dev --frozen + + - name: The GIL must really be off + run: uv run --no-sync python -c "import sys; assert not sys._is_gil_enabled()" + + - name: pytest + run: uv run --no-sync pytest -q + + - name: Thread-safety stress suite + run: uv run --no-sync pytest -q tests/stress + env: + PW_STRESS: "1" + typecheck: name: mypy runs-on: ubuntu-latest @@ -136,8 +165,16 @@ jobs: run: | uv venv "$RUNNER_TEMP/smoke" --python 3.13 uv pip install --python "$RUNNER_TEMP/smoke" dist/*.whl - cd "$RUNNER_TEMP" - "$RUNNER_TEMP/smoke/bin/python" -c "import python_wrap; print(python_wrap.__version__)" + version=$(sed -nE 's/^__version__ *= *"([^"]+)".*/\1/p' src/python_wrap/__init__.py) + "$RUNNER_TEMP/smoke/bin/python" -I scripts/check_installed.py "$version" + + - name: The source distribution includes its test fixtures + run: | + mkdir "$RUNNER_TEMP/sdist" + tar -xzf dist/*.tar.gz -C "$RUNNER_TEMP/sdist" --strip-components=1 + cd "$RUNNER_TEMP/sdist" + uv sync --group dev + uv run --no-sync pytest tests/test_examples.py tests/test_benchmarks_smoke.py - name: Upload the distributions uses: actions/upload-artifact@v7 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 818a72e..8579907 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,15 +1,13 @@ # Build check for the documentation site. # # Read the Docs does the hosting (see .readthedocs.yaml). This workflow only catches -# breakage before it gets there: `--strict` turns a broken link, a page missing from the -# nav or a docstring mkdocstrings cannot render into a failed check on the pull request. +# breakage after changes land on main: `--strict` turns a broken link, a page missing +# from the nav or a docstring mkdocstrings cannot render into a failed check. name: Docs on: push: branches: [main] - pull_request: - workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2860073..6cd9fac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,9 +5,8 @@ # pushed against an unregistered publisher fails at the upload, and a version number on # PyPI can never be reused. # -# Routing: a final version (0.2.0, 0.2.0.post1) goes to PyPI. Anything else (a, b, rc, -# .dev) goes to TestPyPI only, so a release candidate can be rehearsed without burning a -# real version number. +# Every tagged version goes to TestPyPI and is installed back from that index first. +# Final versions then go to PyPI; pre-releases remain on TestPyPI only. # # No API token is involved anywhere in this file, and none should be added. Trusted # Publishing mints a short-lived credential scoped to this workflow and environment. @@ -78,8 +77,7 @@ jobs: python3 -m zipfile -l dist/*.whl | grep -q 'python_wrap/py.typed' uv venv "$RUNNER_TEMP/smoke" --python 3.13 uv pip install --python "$RUNNER_TEMP/smoke" dist/*.whl - cd "$RUNNER_TEMP" - "$RUNNER_TEMP/smoke/bin/python" -c "import python_wrap; print(python_wrap.__version__)" + "$RUNNER_TEMP/smoke/bin/python" -I scripts/check_installed.py "${{ steps.meta.outputs.version }}" - uses: actions/upload-artifact@v7 with: @@ -90,7 +88,7 @@ jobs: publish-testpypi: name: Publish to TestPyPI needs: build - if: github.ref_type == 'tag' && needs.build.outputs.prerelease == 'true' + if: github.ref_type == 'tag' runs-on: ubuntu-latest environment: name: testpypi @@ -103,16 +101,41 @@ jobs: with: name: dist path: dist/ - - uses: pypa/gh-action-pypi-publish@v1.14.2 + - uses: pypa/gh-action-pypi-publish@v1.14.3 with: repository-url: https://test.pypi.org/legacy/ # Re-running after a failed attempt should not be blocked by files that # already landed. skip-existing: true + verify-testpypi: + name: Install and verify from TestPyPI + needs: [build, publish-testpypi] + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + python-version: "3.13" + - name: Install the exact published version without fallback indexes + env: + VERSION: ${{ needs.build.outputs.version }} + run: | + uv venv "$RUNNER_TEMP/testpypi-smoke" --python 3.13 + for attempt in 1 2 3 4 5 6; do + if uv pip install --python "$RUNNER_TEMP/testpypi-smoke" --no-deps --no-cache \ + --index-url https://test.pypi.org/simple/ "python-wrap==$VERSION"; then + break + fi + if [ "$attempt" = 6 ]; then exit 1; fi + sleep 10 + done + "$RUNNER_TEMP/testpypi-smoke/bin/python" -I scripts/check_installed.py "$VERSION" + publish-pypi: name: Publish to PyPI - needs: build + needs: [build, verify-testpypi] if: github.ref_type == 'tag' && needs.build.outputs.prerelease == 'false' runs-on: ubuntu-latest environment: @@ -127,19 +150,45 @@ jobs: with: name: dist path: dist/ - - uses: pypa/gh-action-pypi-publish@v1.14.2 + - uses: pypa/gh-action-pypi-publish@v1.14.3 with: # PEP 740 attestations, signed by the same OIDC identity. attestations: true + verify-pypi: + name: Install and verify from PyPI + needs: [build, publish-pypi] + if: github.ref_type == 'tag' && needs.build.outputs.prerelease == 'false' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + python-version: "3.13" + - name: Install the exact published version + env: + VERSION: ${{ needs.build.outputs.version }} + run: | + uv venv "$RUNNER_TEMP/pypi-smoke" --python 3.13 + for attempt in 1 2 3 4 5 6; do + if uv pip install --python "$RUNNER_TEMP/pypi-smoke" --no-deps --no-cache \ + --index-url https://pypi.org/simple/ "python-wrap==$VERSION"; then + break + fi + if [ "$attempt" = 6 ]; then exit 1; fi + sleep 10 + done + "$RUNNER_TEMP/pypi-smoke/bin/python" -I scripts/check_installed.py "$VERSION" + github-release: name: Create the GitHub release - needs: [build, publish-testpypi, publish-pypi] + needs: [build, verify-testpypi, publish-pypi, verify-pypi] # Whichever publish job applied must have succeeded; the other is skipped. if: >- !cancelled() && github.ref_type == 'tag' && needs.build.result == 'success' - && contains(fromJSON('["success", "skipped"]'), needs.publish-testpypi.result) - && contains(fromJSON('["success", "skipped"]'), needs.publish-pypi.result) + && needs.verify-testpypi.result == 'success' + && ((needs.build.outputs.prerelease == 'true' && needs.publish-pypi.result == 'skipped') + || (needs.publish-pypi.result == 'success' && needs.verify-pypi.result == 'success')) runs-on: ubuntu-latest permissions: contents: write diff --git a/CHANGELOG.md b/CHANGELOG.md index a339208..1aba24c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ A ground-up rework and a much larger toolkit. Every decorator works on plain fun configures logging. Several v0.1 decorators that did not do what their names promised (`transactional`, `revert_on_failure`, async `timing`) now work. See the -[migration guide](https://python-wrap.readthedocs.io/en/latest/migration/) for the full upgrade +[migration guide](https://pythonwrap.readthedocs.io/en/latest/project/migration/) for the full upgrade path. ### Highlights @@ -70,6 +70,16 @@ path. Resilience: +- `RetryBudget` (with `RetryBudgetStats`): one Finagle/Envoy-style retry allowance shared + across functions, threads and event loops via `retry(budget=...)`, `retrying(budget=...)` or + `configure(retry={"budget": ...})`. When it is spent, retrying stops at once and emits + `retry.budget_exhausted`, which prevents retry storms against a failing dependency. +- `retry_if_result=` on `retry`/`retrying` retries when the function returns a matching value; + once retries run out the last value is returned. `retrying` blocks set `attempt.result`. +- For `async def` functions, `retry_if`, `retry_if_result` and `on_retry` may be async and are + awaited; `async with attempt:` is supported in `retrying`. +- `timeout(..., cleanup_grace=)` bounds async cleanup after the limit: the call is cancelled, + given the grace to finish its cleanup, then `TimeoutExpired` is raised. - `retry`: `delay_from=` (with the `retry_after_header` helper for `Retry-After`), `max_time=`, `log_level=` and `log_exc_info=`; `retry_stats(func)` and `RetryStats`; `retrying(...)` / `Retrying` for retrying an inline block (`for` / `async for` with `with attempt`); retries @@ -85,6 +95,11 @@ Resilience: Async toolkit and concurrency: +- `periodic(interval, *, jitter=, initial_delay=, max_runs=, on_error=)` runs a sync function in + a daemon thread, or an async function as a task, every N seconds without drift (missed ticks + are skipped). The function stays callable and gains `start()`, `stop()`, `running`, `runs`, + `last_error` and `running_in_background()` for clean shutdown. Events `periodic.run` and + `periodic.failed`. - `python_wrap.aio`: `gather_limit`, `map_async` and `amap_iter` (bounded concurrency, lazy input, ordered results; everything is cancelled and awaited on error, cancellation or early exit), `race` (first success wins; `RaceFailed`) and `run_sync` (refuses to run inside a running @@ -98,12 +113,24 @@ Async toolkit and concurrency: Caching: +- `memoize(ttl=..., stale_ttl=...)`: stale-while-revalidate. After `ttl` a stale value is served + at once while one background refresh runs (a task for async functions, a daemon thread for + sync ones); a failed refresh keeps the stale value and is logged. Events `cache.stale` and + `cache.refresh_failed`. +- `memoize(store=...)` and the `CacheStore` protocol: keep entries in any store (a dict, Redis, + ...); the `cache_*` helpers work against it. +- `memoize(per_instance=True)`: per-instance method caches held through weak references, so + instances stay collectable and need not be hashable. - `cached_property(ttl=)` (thread-safe, `async def` getters, `invalidate()`). - `memoize`: `cache_exceptions=` / `exception_ttl=`, a per-item `ttl=` callable, and `cache_contains()`, `cache_get()` and `cache_parameters()`. Observability, configuration and testing: +- `trace(otel=True, span_name=...)` wraps calls in an OpenTelemetry span (sync and async) with + `code.function`, `code.namespace` and `python_wrap.context.*` attributes and exception + recording. `opentelemetry-api` is imported lazily and only when asked for; python-wrap still + has no runtime dependencies. - `count_calls`: calls, errors, in-flight, mean, p50, p95 and p99 per function. - `python_wrap.events`: `subscribe()`, `listening()`, `filter_events()` and `EVENT_NAMES`, one hook for retry, cache, circuit, limiter, bulkhead and timeout events (Prometheus, OpenTelemetry and @@ -131,6 +158,9 @@ Observability, configuration and testing: Core: +- Decorated functions, including `periodic` objects and `wrap_methods` classes, pickle by + reference, so they work with `multiprocessing` and `ProcessPoolExecutor` under the "spawn" + start method. Cache, limiter and breaker state is per worker process. - `retry`: `backoff`, `max_delay`, `jitter`, `on=` exception filter, `retry_if=` predicate, `on_retry=` hook receiving a `RetryState`, unlimited attempts (`max_attempts=None`), and bare `@retry` usage. @@ -153,14 +183,22 @@ Core: - `revert_on_failure(attrs=, deep=, target=)`. - `deprecate(since=, removed_in=, alternative=, category=)`, which also works on classes; `mock_data(enabled=bool | callable)`; `compose(*decorators)`. -- Exceptions in `python_wrap.errors`, each subclassing the closest builtin: `PythonWrapError`, - `TimeoutExpired` (`TimeoutError`), `RateLimitExceeded`, `CircuitOpenError`, `ValidationError` - (`ValueError`) and `ArgumentTypeError` (`TypeError`). +- Exceptions in `python_wrap.errors`, all subclassing `PythonWrapError`: `TimeoutExpired` + (also a `TimeoutError`), `RateLimitExceeded`, `CircuitOpenError`, `ValidationError` (also a + `ValueError`) and `ArgumentTypeError` (also a `TypeError`). - `once` gained `.reset()` and `.called`. - A `py.typed` marker, and a `NullHandler` on the `python_wrap` logger. ### Changed +- `CircuitBreaker`, `RateLimiter`, `Bulkhead`, `RetryBudget`, `Transaction`, `Policy` and + `testing.FakeClock` raise a clear `TypeError` when pickled or copied (their state belongs to + one process), instead of an error about a thread lock or a copy that silently shares a lock. +- `RetryState.exception` can be `None` (when a returned value was rejected by + `retry_if_result`), and `RetryState` has a new `result` field. A sync function given an async + retry hook raises `TypeError` when decorated. +- On free-threaded Python 3.14t new threads inherit the caller's context, so `log_context` + fields carry over into them (`sys.flags.thread_inherit_context`). - The project uses a `src/` layout and Hatchling, and requires Python 3.10 or newer. - The runtime dependencies `wrapt` and `typing-extensions` were unused and have been removed. - `retry` waits with `asyncio.sleep` for `async def` functions, and re-raises the final @@ -190,6 +228,20 @@ Core: ### Fixed +- Final releases now pass through TestPyPI and an installation check before PyPI. + Both indexes are verified after upload before creating the GitHub release. +- Source distributions include the examples and benchmark script used by their tests. +- Release smoke checks validate metadata, public imports, `py.typed`, retry, memoization + and async behavior in an isolated installed environment. + +- `AsyncPeriodic.stop()` no longer swallows a cancellation aimed at the task awaiting it, and + ends a loop whose run swallows the cancel; `stop()` called inside a run ends the loop as soon + as that run returns. +- `timeout(..., cleanup_grace=...)` lets an outside cancellation propagate as `CancelledError` + (matching `asyncio.timeout`), and waits for and retrieves the child task when the caller is + cancelled instead of logging "Task exception was never retrieved". +- `periodic(on_error="raise")` logs an error that `stop()` never collected when `start()` is + called again, instead of dropping it silently. - `revert_on_failure` never restored anything because its snapshot was always empty. - `transactional` was a no-op; it now commits and rolls back through a `Transaction`. - `memoize` and `cache` built keys from `str(args)`, which collided and ignored keyword @@ -215,13 +267,14 @@ Core: ### Performance +- Async `memoize` and `cache` hits are about 30-35% faster (one lock instead of two). - The `memoize` hit path is about 9x faster. - `validate_args`, `check_type`, `enforce_types`, `log_args` and `audit` skip signature binding on plain positional calls (5x to 40x faster). - `audit` skips building a record when nobody would receive it, and `timing` skips formatting when its log level is disabled. - Numbers and a benchmark script are on the - [performance page](https://python-wrap.readthedocs.io/en/latest/performance/). + [performance page](https://pythonwrap.readthedocs.io/en/latest/project/performance/). ### Deprecated @@ -240,7 +293,8 @@ Core: ### Infrastructure - CI runs ruff, `mypy --strict`, the test suite on Linux, macOS and Windows across Python 3.10 to - 3.14, and a wheel check. + 3.14, the test and thread-safety stress suites on free-threaded Python 3.14t, and a wheel + check. - Tag-driven releases publish to PyPI through Trusted Publishing, with no stored token. - The documentation build is checked in CI and hosted on Read the Docs. - Added Dependabot, issue and pull request templates, `CONTRIBUTING.md`, `SECURITY.md`, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ea322c..c0c969c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,9 @@ There is no Makefile; the commands below are the whole workflow. ## Run the checks -CI runs exactly these, so if they pass locally the pull request will be green. +Run these checks before opening or merging a pull request. CI and docs run after +changes land on `main`; pull requests and feature-branch pushes do not start them. +Release publishing remains tied to version tags. ```bash uv run pytest # tests diff --git a/README.md b/README.md index 5ba5965..5478154 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
- + python-wrap @@ -14,13 +14,13 @@ Retry (with `Retry-After`), circuit breaker, timeout, cache, rate limit, bulkhea [![PyPI](https://img.shields.io/pypi/v/python-wrap?logo=pypi&logoColor=white)](https://pypi.org/project/python-wrap/) [![Python](https://img.shields.io/pypi/pyversions/python-wrap?logo=python&logoColor=white)](https://pypi.org/project/python-wrap/) [![License: MIT](https://img.shields.io/pypi/l/python-wrap)](https://github.com/LitePy/PythonWrap/blob/main/LICENSE) -[![Docs](https://readthedocs.org/projects/python-wrap/badge/?version=latest)](https://python-wrap.readthedocs.io) +[![Docs](https://readthedocs.org/projects/pythonwrap/badge/?version=latest)](https://pythonwrap.readthedocs.io) [![Typed: mypy strict](https://img.shields.io/badge/typed-mypy%20strict-2a6db2)](https://mypy.readthedocs.io/en/stable/command_line.html#cmdoption-mypy-strict) -[**Documentation**](https://python-wrap.readthedocs.io) · -[Cookbook](https://python-wrap.readthedocs.io/en/latest/cookbook/) · -[Performance](https://python-wrap.readthedocs.io/en/latest/project/performance/) · -[Migration guide](https://python-wrap.readthedocs.io/en/latest/project/migration/) · +[**Documentation**](https://pythonwrap.readthedocs.io) · +[Cookbook](https://pythonwrap.readthedocs.io/en/latest/cookbook/) · +[Performance](https://pythonwrap.readthedocs.io/en/latest/project/performance/) · +[Migration guide](https://pythonwrap.readthedocs.io/en/latest/project/migration/) · [Changelog](https://github.com/LitePy/PythonWrap/blob/main/CHANGELOG.md)
@@ -33,7 +33,7 @@ Retry (with `Retry-After`), circuit breaker, timeout, cache, rate limit, bulkhea - **Zero dependencies.** Pure Python, nothing to audit or pin. - **Typed.** Decorators preserve the wrapped signature (`ParamSpec`), and stacking keeps the exact type (`@retry` over `@memoize` still has `.cache_info()`). Ships `py.typed`, checked with `mypy --strict`. - **Sync and async, one API.** Works on `def`, `async def` and methods. Async wrappers stay coroutine functions and never block the event loop. -- **Fast.** Cache hits and validation skip signature binding on the hot path ([numbers](https://python-wrap.readthedocs.io/en/latest/project/performance/)). +- **Fast.** Cache hits and validation skip signature binding on the hot path ([numbers](https://pythonwrap.readthedocs.io/en/latest/project/performance/)). - **Observable and testable.** One event stream for retries, cache, circuit, limiters and timeouts; `python_wrap.testing` makes tests instant and deterministic. - **Never configures your logging**, is thread-safe, and is tested on Python 3.10 to 3.14 on Linux, macOS and Windows. @@ -113,32 +113,35 @@ print(f"{time.monotonic() - start:.2f}s") ## What is in the box -Everything below is importable from `python_wrap` unless a module is shown. [Choose a decorator](https://python-wrap.readthedocs.io/en/latest/get-started/choosing/) is the quick way in. +Everything below is importable from `python_wrap` unless a module is shown. [Choose a decorator](https://pythonwrap.readthedocs.io/en/latest/get-started/choosing/) is the quick way in. | Area | Name | What it does | |---|---|---| -| **Resilience** | `retry` | Backoff, jitter, `max_delay`, `max_time`, `retry_if`, `on_retry`; `delay_from=retry_after_header` honours `Retry-After`; never retries cancellation | +| **Resilience** | `retry` | Backoff, jitter, `max_delay`, `max_time`, `retry_if`, `retry_if_result`, `on_retry` (async hooks too); `delay_from=retry_after_header` honours `Retry-After`; never retries cancellation | +| | `RetryBudget` | One retry allowance shared across functions, so a failing dependency does not trigger a retry storm | | | `retrying`, `Retrying` | Retry an inline block (`for attempt in retrying(...)`, sync and `async for`) | | | `retry_stats`, `RetryStats` | Attempts, successes and failures per retried function | | | `fallback`, `suppress` | Return a default (or handler result), or swallow chosen exceptions | | | `circuit_breaker`, `CircuitBreaker` | Fail fast while a dependency is down; `success_threshold`, `exclude`, `on_state_change`, `.stats`; one breaker can guard many functions | -| | `timeout`, `Timeout` | Decorator or `with timeout(5):` / `async with`; raises `TimeoutExpired` | +| | `timeout`, `Timeout` | Decorator or `with timeout(5):` / `async with`; raises `TimeoutExpired`; `cleanup_grace=` bounds async cleanup | | | `deadline`, `Deadline` | A shared time budget; `retry` and `timeout` clamp to it | | | `hedge` | Fire a backup call when the first is slow and take the first success (idempotent calls only) | | **Async toolkit** | `gather_limit`, `map_async`, `amap_iter` | Bounded concurrency, lazy input, ordered results; everything is cancelled and awaited on error | | | `race`, `RaceFailed` | First success wins, the rest are cancelled | | | `run_sync` | Call async code from sync code (refuses inside a running loop) | -| **Caching** | `memoize` | LRU + TTL, `typed`, `key=`, per-item `ttl=`, `cache_exceptions=`, `cache_info/clear/delete/contains/get`; async in-flight dedupe | +| **Caching** | `memoize` | LRU + TTL, `typed`, `key=`, per-item `ttl=`, `cache_exceptions=`, `stale_ttl=` (serve stale while refreshing), `per_instance=True`, `cache_info/clear/delete/contains/get`; async in-flight dedupe | +| | `CacheStore` | Protocol to keep `memoize` entries in your own store (dict, Redis, ...) | | | `cached_property`, `invalidate` | `functools.cached_property` plus `ttl`, locking and `async def` getters | | | `once`, `cache` | Run once and reuse (thread-safe, `.reset()`); `cache` is the v0.1 name for `memoize(ttl=...)` | | **Concurrency & limits** | `rate_limit`, `RateLimiter` | Sliding window, `block=True`, `key=`, several limits, one budget shared across functions | | | `Bulkhead`, `limit_concurrency` | Cap concurrent calls, bound the queue, raise `BulkheadFull` | | | `throttle`, `debounce` | At most once per interval; run after calls stop (`.cancel()`, `.flush()`) | | | `singleflight`, `run_in_thread` | Share one execution between identical concurrent calls; run in a thread, get a `Future` | +| | `periodic` | Run a function every N seconds in the background, without drift, with clean `start()`/`stop()` | | **Batching** | `batch_calls` | DataLoader pattern: many single-item calls become one bulk call; sync and async, deduped | | **Observability & metrics** | `count_calls` | `.stats`: calls, errors, in-flight, mean, p50, p95, p99 | | | `subscribe`, `listening` | One event hook for retry, cache, circuit, limiter, bulkhead and timeout events (`python_wrap.events` also has `filter_events`, `EVENT_NAMES`) | -| | `log_args`, `log_return`, `trace`, `audit` | Structured, redacted logging; `AuditRecord` to a sink | +| | `log_args`, `log_return`, `trace`, `audit` | Structured, redacted logging; `AuditRecord` to a sink; `trace(otel=True)` opens an OpenTelemetry span | | | `log_context`, `current_log_context` | Contextvar fields in every log line and audit record | | | `timing`, `profile`, `measure_memory`, `benchmark` | Duration with `threshold=`/`callback=`, `cProfile`, `tracemalloc`, `benchmark_async` | | **Validation** | `enforce_types`, `validate_args`, `check_type` | Check from annotations, with predicates, or with explicit types | @@ -232,7 +235,7 @@ except TimeoutExpired: print("out of time") ``` -After `recovery_timeout` the breaker goes half-open and lets a trial call through. Share one `CircuitBreaker` across functions for one dependency, or use `@circuit_breaker(...)` for a private one (`func.breaker`). `timeout` interrupts sync code with `SIGALRM` on the Unix main thread, cancels `async def` with `asyncio.wait_for`, and elsewhere falls back to a thread that cannot stop the work (see [Limitations](https://python-wrap.readthedocs.io/en/latest/project/limitations/)). +After `recovery_timeout` the breaker goes half-open and lets a trial call through. Share one `CircuitBreaker` across functions for one dependency, or use `@circuit_breaker(...)` for a private one (`func.breaker`). `timeout` interrupts sync code with `SIGALRM` on the Unix main thread, cancels `async def` with `asyncio.wait_for`, and elsewhere falls back to a thread that cannot stop the work (see [Limitations](https://pythonwrap.readthedocs.io/en/latest/project/limitations/)). ### Caching @@ -359,7 +362,7 @@ print(names) # retry.attempt_failed, ..., retry.succeed print(work.stats.calls, work.stats.p95) ``` -Subscribe once and forward events to Prometheus, OpenTelemetry or statsd (recipes in the [guide](https://python-wrap.readthedocs.io/en/latest/features/observability/metrics/)). Listeners cannot break the decorated function. +Subscribe once and forward events to Prometheus, OpenTelemetry or statsd (recipes in the [guide](https://pythonwrap.readthedocs.io/en/latest/features/observability/metrics/)). Listeners cannot break the decorated function. ### Validation and logging with redaction @@ -411,7 +414,7 @@ Decorators are applied bottom-up and run top-down, so order matters: | `log_args`, `timing`, `audit` | everything | They observe what callers actually experience | | `enforce_types`, `validate_args` | the rest | Reject bad input before doing any work | -You do not have to memorise this. `policy(observe=, validate=, fallback=, cache=, limiter=, bulkhead=, breaker=, retry=, timeout=, guard=)` takes each decorator by role and applies them in the recommended order, whatever order you wrote them in. Policies are immutable (`.with_(timeout=timeout(30))`), work on methods and `async def`, and `wrap_methods(policy)` protects a whole class. `compose(a, b, c)(f)` is `a(b(c(f)))` when you just want a named stack. Details: [Stacking](https://python-wrap.readthedocs.io/en/latest/get-started/stacking/) and [Policies](https://python-wrap.readthedocs.io/en/latest/features/resilience/policies/). +You do not have to memorise this. `policy(observe=, validate=, fallback=, cache=, limiter=, bulkhead=, breaker=, retry=, timeout=, guard=)` takes each decorator by role and applies them in the recommended order, whatever order you wrote them in. Policies are immutable (`.with_(timeout=timeout(30))`), work on methods and `async def`, and `wrap_methods(policy)` protects a whole class. `compose(a, b, c)(f)` is `a(b(c(f)))` when you just want a named stack. Details: [Stacking](https://pythonwrap.readthedocs.io/en/latest/get-started/stacking/) and [Policies](https://pythonwrap.readthedocs.io/en/latest/features/resilience/policies/). ## How it compares @@ -432,15 +435,15 @@ None of them cover bounded `gather`, bulkheads, batching, hedging, policies or t ## Learn more -- [Cookbook](https://python-wrap.readthedocs.io/en/latest/cookbook/): resilient HTTP client, async crawler, FastAPI, Django, CLI tools, DataLoader batching, testing, observability. -- [Performance](https://python-wrap.readthedocs.io/en/latest/project/performance/): per-call overhead and how to benchmark it yourself. -- [Testing](https://python-wrap.readthedocs.io/en/latest/testing/): `fast_mode`, `FakeClock`, `spy` and pytest fixtures. -- [Migration guide](https://python-wrap.readthedocs.io/en/latest/project/migration/) and [CHANGELOG](https://github.com/LitePy/PythonWrap/blob/main/CHANGELOG.md). +- [Cookbook](https://pythonwrap.readthedocs.io/en/latest/cookbook/): resilient HTTP client, async crawler, FastAPI, Django, CLI tools, DataLoader batching, testing, observability. +- [Performance](https://pythonwrap.readthedocs.io/en/latest/project/performance/): per-call overhead and how to benchmark it yourself. +- [Testing](https://pythonwrap.readthedocs.io/en/latest/testing/): `fast_mode`, `FakeClock`, `spy` and pytest fixtures. +- [Migration guide](https://pythonwrap.readthedocs.io/en/latest/project/migration/) and [CHANGELOG](https://github.com/LitePy/PythonWrap/blob/main/CHANGELOG.md). - Runnable [examples](https://github.com/LitePy/PythonWrap/tree/main/examples). ## Upgrading from 0.1 -0.2.0 fixes decorators that did not do what their names promised (`transactional`, `revert_on_failure`, async `timing`) and makes a few breaking changes: `run_in_thread` returns a `Future`, `benchmark` is a function, `retry_on_exception` is deprecated and bounded, `rate_limit` raises `RateLimitExceeded`, and the logging decorators default to `DEBUG`. Python 3.8 and 3.9 are no longer supported. `from python_wrap.decorators import ...` still works. See the [migration guide](https://python-wrap.readthedocs.io/en/latest/project/migration/). +0.2.0 fixes decorators that did not do what their names promised (`transactional`, `revert_on_failure`, async `timing`) and makes a few breaking changes: `run_in_thread` returns a `Future`, `benchmark` is a function, `retry_on_exception` is deprecated and bounded, `rate_limit` raises `RateLimitExceeded`, and the logging decorators default to `DEBUG`. Python 3.8 and 3.9 are no longer supported. `from python_wrap.decorators import ...` still works. See the [migration guide](https://pythonwrap.readthedocs.io/en/latest/project/migration/). ## Contributing diff --git a/RELEASING.md b/RELEASING.md index ff5f40f..0ef4d51 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -37,17 +37,24 @@ was removed; revoke the token it used at int: + return version # imagine a slow HTTP call + +assert settings() == 1 # miss: computed in the caller +version = 2 +time.sleep(0.06) # older than ttl: now stale +assert settings() == 1 # served at once; a refresh starts in the background +time.sleep(0.05) +assert settings() == 2 # the refreshed value +``` + +- **Sync functions** refresh in a daemon thread, at most one per key, running in a copy of the caller's `contextvars` context (so request ids and similar context reach it). +- **`async def` functions** refresh in a task on the running event loop. It shares the in-flight bookkeeping, so concurrent stale hits and misses of one key start a single task. +- **A failed refresh keeps the stale value.** It is logged at `WARNING` (on `logger`, default `python_wrap`), emits `cache.refresh_failed`, and the next call starts another refresh. +- **Past `ttl + stale_ttl`** the entry is dropped and the next call is an ordinary miss, computed in the caller. +- `stale_ttl` requires `ttl` (a number, a `timedelta` or a [function of the result](#per-item-ttl)). Cached exceptions are never served stale. + +Refresh decisions read the package clock, so `fake_clock` drives them in tests without waiting: + +```python +import threading +from python_wrap import memoize +from python_wrap.testing import fake_clock + +calls = [] + +@memoize(ttl=10, stale_ttl=50) +def rates() -> int: + calls.append(1) + return len(calls) + +with fake_clock() as clock: + assert rates() == 1 + clock.advance(15) + assert rates() == 1 # stale: served, refresh started + for t in threading.enumerate(): + if t.name.startswith("python_wrap-refresh-"): + t.join() + assert rates() == 2 +``` + +## Custom stores + +By default entries live in a dict inside the decorator. Pass `store=` to keep them somewhere else: a dict you own, a shared registry, Redis. A store is any object with these five methods (the `CacheStore` protocol; no base class needed): + +| Method | Contract | +| --- | --- | +| `get(key)` | The entry stored under `key`, or `None`. | +| `set(key, entry, ttl)` | Store `entry`. `ttl` is the number of seconds after which it is useless and may be discarded (`None`: keep it). | +| `delete(key)` | Remove the entry; return `True` if there was one. | +| `clear()` | Remove every entry. | +| `__len__()` | The number of stored entries (`cache_info().currsize`). | + +```python +from python_wrap import memoize + +class DictStore: + def __init__(self) -> None: + self.data = {} + + def get(self, key): + return self.data.get(key) + + def set(self, key, entry, ttl): + self.data[key] = entry + + def delete(self, key): + return self.data.pop(key, None) is not None + + def clear(self): + self.data.clear() + + def __len__(self): + return len(self.data) + +store = DictStore() + +@memoize(store=store, ttl=300) +def price(sku: str) -> float: + return 9.99 + +price("A-1"); price("A-1") +assert len(store) == 1 +assert price.cache_info().hits == 1 +assert price.cache_get("A-1") == 9.99 # the cache_* helpers all work +price.cache_clear() +assert len(store) == 0 +``` + +What a store can rely on, and what it must do: + +- **Entries are opaque.** Return exactly what was stored and never look inside. A store that serialises (Redis, files) must round-trip entries, for example with `pickle`. Entries are never `None`. +- **Entries carry their own deadlines**, as Unix timestamps, and `memoize` checks them. A store may forget an entry at any time (that is just a miss) and does not have to implement expiry, though honouring `ttl` keeps it from filling up. Processes sharing a store agree on expiry as long as their clocks agree. +- **Keys are hashable, but not strings.** They are tuples of the normalised arguments (or whatever `key=` returns). A store that needs `str` or `bytes` keys must serialise them itself, with a prefix per function when several functions share one backend. +- **The store must be thread-safe.** `memoize` does not hold its lock while calling the store, so a network round trip never blocks other callers. Single `dict` operations are atomic in CPython, so the store above qualifies. +- **Methods are synchronous.** For `async def` functions they run on the event loop, so keep them fast. +- Exceptions raised by the store propagate to the caller. +- `maxsize` (LRU) only applies to the built-in store: passing both `store=` and `maxsize=` raises `TypeError`. + +A sketch of a Redis adapter, using the `redis` package (not a python-wrap dependency): + + +```python +import pickle +import redis +from python_wrap import memoize + +class RedisStore: + def __init__(self, client: redis.Redis, prefix: str) -> None: + self.client, self.prefix = client, prefix + + def _key(self, key) -> bytes: + return self.prefix.encode() + pickle.dumps(key) # keys are tuples, not strings + + def get(self, key): + raw = self.client.get(self._key(key)) + return None if raw is None else pickle.loads(raw) + + def set(self, key, entry, ttl): + expire = None if ttl is None else max(1, int(ttl) + 1) + self.client.set(self._key(key), pickle.dumps(entry), ex=expire) + + def delete(self, key): + return bool(self.client.delete(self._key(key))) + + def clear(self): + for k in self.client.scan_iter(match=self.prefix + "*"): + self.client.delete(k) + + def __len__(self): + return sum(1 for _ in self.client.scan_iter(match=self.prefix + "*")) + +@memoize(store=RedisStore(redis.Redis(), prefix="pricing:"), ttl=300) +def price(sku: str) -> float: ... +``` + +`pickle.dumps` of a key is only stable when the arguments pickle deterministically (strings, numbers, tuples of them), so prefer simple arguments or a `key=` that returns a string. Only unpickle data from a store you trust. + +## Per-instance method caches + +On a plain `@memoize` method, `self` is part of the key: the cache holds a strong reference to every instance, keeping it alive, and instances must be hashable. `per_instance=True` fixes both. Each instance gets its own cache, found through a weak reference and keyed by the arguments after `self`: + +```python +import dataclasses +import gc +import weakref +from python_wrap import memoize + +@dataclasses.dataclass # eq=True: instances are unhashable +class Account: + id: int + + @memoize(per_instance=True, maxsize=100) + def balance(self, currency: str) -> float: + return 100.0 + +a, b = Account(1), Account(1) # equal, but two objects: two caches +a.balance("EUR"); a.balance("EUR"); b.balance("EUR") +assert Account.balance.cache_info().hits == 1 +assert Account.balance.cache_info().currsize == 2 # summed over live instances + +ref = weakref.ref(a) +del a +gc.collect() +assert ref() is None # the cache did not keep it alive +assert Account.balance.cache_info().currsize == 1 # its entries went with it +``` + +- **Identity, not equality.** Caches belong to objects, so two equal instances never share results, and unhashable classes (an `eq=True` dataclass) work. When an instance is garbage-collected its cache is dropped, so a later object that gets the same `id()` starts empty. +- **Instances must be weak-referenceable.** A class with `__slots__` must list `"__weakref__"`; otherwise the call raises `TypeError`. +- `maxsize` applies to each instance. `cache_info()` adds up hits, misses and sizes over every live instance and `cache_clear()` clears them all. `cache_delete`, `cache_get` and `cache_contains` take the instance first, as on any method: `obj.balance.cache_get(obj, "EUR")`. +- A custom `key=` still receives `self`; leave it out of the key, since the cache is already per instance. +- A cached result that references its own instance (returning `self`, a bound method, a closure over `self`) keeps that instance alive. +- `per_instance` uses the built-in store and cannot be combined with `store=`. + ## Async and methods **Async.** Concurrent awaits of a missing key share one in-flight task per event loop, so the function runs once. @@ -229,11 +419,11 @@ Unlike v0.1, every decorated function gets its own store, even when one `cache(. ## Events -`memoize` (and `cache`) emit `cache.hit`, `cache.miss` and `cache.evicted` (field `reason`: `"lru"` or `"ttl"`) to `python_wrap.events` listeners, with the function's qualified name. With no listener registered this costs one list check. See [Metrics](../observability/metrics.md) for recipes. +`memoize` (and `cache`) emit `cache.hit`, `cache.miss` and `cache.evicted` (field `reason`: `"lru"` or `"ttl"`) to `python_wrap.events` listeners, with the function's qualified name. With `stale_ttl` they also emit `cache.stale` (a stale value was served and a background refresh started; the hit itself is reported as `cache.hit`) and `cache.refresh_failed` (field `exception`). With no listener registered this costs one list check. See [Metrics](../observability/metrics.md) for recipes. ## Testing -Use `python_wrap.testing.disable_caching()` to bypass every cache in tests. +Use `python_wrap.testing.disable_caching()` to bypass every cache in tests: nothing is read, written or refreshed. `ttl` and `stale_ttl` run on the package clock, so `python_wrap.testing.fake_clock()` makes entries expire or turn stale without waiting (see the example under [Serving stale values](#serving-stale-values-while-refreshing)). ## Gotchas @@ -241,10 +431,10 @@ Use `python_wrap.testing.disable_caching()` to bypass every cache in tests. The cache is guarded by a lock, but the function is never called while holding it. For sync functions, two threads that miss on the same key at the same moment may both compute the value, and the last result stored wins. If that is not acceptable, stack [`singleflight`](singleflight.md) under `memoize`. !!! warning "Methods keep instances alive" - `self` is part of the key and the cache holds a strong reference to every instance it has seen, which keeps them alive. Use `maxsize` or `ttl` (or a `key=` that ignores `self`) on methods of many short-lived objects. See [Limitations](../../project/limitations.md). + `self` is part of the key and the cache holds a strong reference to every instance it has seen, which keeps them alive. Use [`per_instance=True`](#per-instance-method-caches) on methods of many short-lived objects (or `maxsize`, `ttl`, or a `key=` that ignores `self`). See [Limitations](../../project/limitations.md). !!! warning "Unhashable instances" - A method of a class whose instances are unhashable (for example an `eq=True` dataclass) fails with a `TypeError` naming `self`. `id(self)` is deliberately not used as a fallback because ids are reused after garbage collection. Pass `key=` or use [`cached_property`](cached-property.md) instead. + A method of a class whose instances are unhashable (for example an `eq=True` dataclass) fails with a `TypeError` naming `self`. `id(self)` is deliberately not used as a fallback because ids are reused after garbage collection. Use [`per_instance=True`](#per-instance-method-caches) (keyed by identity through a weak reference), pass `key=`, or use [`cached_property`](cached-property.md) instead. ## See also diff --git a/docs/features/concurrency/index.md b/docs/features/concurrency/index.md index e877be5..876e236 100644 --- a/docs/features/concurrency/index.md +++ b/docs/features/concurrency/index.md @@ -34,6 +34,12 @@ Control how often a function runs, how many calls run at once, and how you run m Start a blocking call in a thread and get a `Future` back. +- :material-update:{ .lg .middle } **[Periodic tasks](periodic.md)** + + --- + + `periodic`: run a function every N seconds in the background, with clean shutdown. + ## Which one? @@ -48,6 +54,7 @@ Control how often a function runs, how many calls run at once, and how you run m | Run many awaitables or items with a bound | [Async helpers](async-helpers.md) | | Take the first successful result of several awaitables | [`race`](async-helpers.md#race) | | Call blocking code without waiting for it | [`run_in_thread`](run-in-thread.md) | +| Run something every N seconds in the background | [`periodic`](periodic.md) | | Make identical concurrent calls share one execution | [`singleflight`](../caching/singleflight.md) | A rate limit counts calls over **time**; a bulkhead counts calls **in flight**. Use both when a dependency has a quota and a connection limit. diff --git a/docs/features/concurrency/periodic.md b/docs/features/concurrency/periodic.md new file mode 100644 index 0000000..d56a3f3 --- /dev/null +++ b/docs/features/concurrency/periodic.md @@ -0,0 +1,116 @@ +# Periodic tasks + +Run a function every N seconds in the background, and stop it cleanly. `periodic` is a small loop, not a scheduler: no cron expressions, no persistence, no catching up on missed runs. + +```python +import time +from python_wrap.periodic import periodic + +beats: list[int] = [] + +@periodic(0.01, max_runs=3) +def heartbeat() -> None: + beats.append(1) + +with heartbeat.running_in_background(): + while heartbeat.runs < 3: + time.sleep(0.005) + +assert not heartbeat.running # stopped and joined on leaving the block +assert len(beats) == 3 +heartbeat() # still an ordinary function +``` + +## When to use it + +- Refresh a cache, flush a buffer, send a heartbeat, poll a queue, sweep expired rows. +- You want clean shutdown: no daemon thread left running after a test or after a `with` block. +- You need cron expressions, persistence or distributed locks: use a real scheduler. + +## What you get + +| | | +| --- | --- | +| `f.start(*args, **kwargs)` | Start the loop. The arguments are passed to every run. Returns `f`. Raises `RuntimeError` if already running. | +| `f.stop()` | Plain functions: stop and join the thread (`timeout=5.0`), returns `True` once it ended. `async def`: `await f.stop()` cancels the task and awaits it. Safe to call twice or before `start()`. | +| `f.running` | `True` while the thread or task is alive. | +| `f.runs` | Runs made since `start()`, failed ones included. | +| `f.last_error` | Exception of the most recent failed run, or `None`. | +| `f.running_in_background(*args, **kwargs)` | `with` (or `async with`) block that starts the loop and stops it on exit. | +| `f(*args)` | Calls the function directly, as before. | + +## Plain functions and async functions + +A plain function runs in a **daemon thread** (named `python-wrap-periodic:`), so it never keeps the interpreter alive. The caller's `contextvars` are copied into it. + +An `async def` function runs as a **task on the current event loop**, so `start()` must be called from inside a running loop; anywhere else it raises a `RuntimeError` saying so. `stop()` is a coroutine: it cancels the task, including a run in progress, and waits for it. If the task awaiting `stop()` is itself cancelled (or an `asyncio.timeout` around it expires), that `CancelledError` propagates at once while the loop finishes its cleanup in the background. Called from inside the function, `stop()` ends the loop as soon as that run returns, without waiting for the next tick. + +```python +import asyncio +from python_wrap.periodic import periodic + +@periodic(0.01) +async def refresh() -> None: + ... + +async def main() -> None: + async with refresh.running_in_background(): + await asyncio.sleep(0.05) + assert not refresh.running + +asyncio.run(main()) +``` + +A thread cannot be interrupted, so `stop()` on a plain function lets the run in progress finish. If it does not finish within `timeout`, `stop()` logs a warning and returns `False`. + +## Timing + +- **Start to start, no drift.** The n-th run is due at `first + n * interval`, whatever the runs cost. +- **Missed ticks are skipped.** If a run takes longer than the interval, the loop does not run again in a burst to catch up. It waits for the next tick on the grid. +- **`initial_delay`** delays the first run (default: run at once). +- **`jitter`** (a fraction, `0` up to but excluding `1`) moves each run by a uniformly random amount up to plus or minus `jitter * interval`. It spreads out many processes that started together. Ticks are computed without it, so it never accumulates. +- **`max_runs`** makes the loop end by itself after that many runs. + +## Errors + +| `on_error` | Effect of an exception in a run | +| --- | --- | +| `"log"` (default) | Logged through the package logger, loop continues. | +| `"stop"` | Logged, loop ends. | +| `"raise"` | Loop ends and the exception is re-raised once from `stop()` (or on leaving the `with` block), so it is never lost in a background thread. If `start()` is called again before `stop()` collected it, the old exception is logged at `ERROR` (with its traceback) and then dropped. | + +In every mode the exception is kept in `last_error` and published as a `periodic.failed` event. `logger=` and `level=` (default `logging.ERROR`) control the log line. + +## Methods + +Decorating a method in a class body is rejected with a `TypeError`, because the loop needs an instance. Decorate the bound method instead: + +```python +from python_wrap.periodic import periodic + +class Cache: + def __init__(self) -> None: + self.refreshed = 0 + self.job = periodic(60, name="cache-refresh")(self.refresh) + + def refresh(self) -> None: + self.refreshed += 1 + +cache = Cache() +cache.refresh() +assert cache.job.runs == 0 # not started yet; call cache.job.start() +``` + +## Events + +`periodic.run` (`run`, `seconds`) after each successful run and `periodic.failed` (`run`, `exception`) after each failure; `Event.function` is `name=` or the function's qualified name. See [Metrics and events](../observability/metrics.md). + +## Testing + +The schedule uses **real time**. `fast_mode()` and `fake_clock()` affect only backoff waits (see [what the overrides affect](../../testing/index.md#what-the-overrides-affect)); making a periodic loop instant would just spin the CPU. In tests, call the function directly, or use a tiny interval, `max_runs` and `with f.running_in_background():` so nothing outlives the test. + +## See also + +- [Throttle and debounce](throttle-debounce.md), [run_in_thread](run-in-thread.md) +- [Retry](../resilience/retry.md) for retrying a single failing run +- [API reference](../../api/periodic.md) diff --git a/docs/features/configuration.md b/docs/features/configuration.md index af6a56b..bd87cee 100644 --- a/docs/features/configuration.md +++ b/docs/features/configuration.md @@ -47,7 +47,7 @@ reset_config() | Key | Effect | | --- | --- | | `logger` | Logger (or name) used by every decorator that logs and was not given `logger=`. `None` restores the package logger. | -| `retry` | Dict of defaults for `retry` and `retrying`: `max_attempts`, `delay`, `backoff`, `max_delay`, `jitter`, `max_time`, `log_level`. | +| `retry` | Dict of defaults for `retry` and `retrying`: `max_attempts`, `delay`, `backoff`, `max_delay`, `jitter`, `max_time`, `log_level`, `budget` (one shared `RetryBudget`). | | `redact` | Replaces the built-in `DEFAULT_REDACT` list for `log_args` and `audit`. `None` restores it. | | `redact_extra` | Names **added** to the redaction list (the built-ins stay), for `log_args` and `audit`. | diff --git a/docs/features/observability/logging.md b/docs/features/observability/logging.md index 21b1982..b15c6e4 100644 --- a/docs/features/observability/logging.md +++ b/docs/features/observability/logging.md @@ -166,6 +166,23 @@ assert handler() == 1 Use it to answer "who is calling this?" while debugging. It defaults to `DEBUG`, so it is silent unless you enable that level. +### OpenTelemetry spans + +`trace(otel=True)` also opens an OpenTelemetry span around each call, named after the function (`span_name=` overrides it), using `opentelemetry.trace.get_tracer("python_wrap")`. The span is current for the whole call, also across `await`; it carries `code.function`, `code.namespace` and the fields of the active `log_context` as `python_wrap.context.` attributes, and on an exception it records the exception and sets the error status. + +python-wrap stays dependency-free: `opentelemetry-api` is imported only when `otel=True`, and a missing package raises `ImportError` when the function is decorated. Configure the SDK and exporter yourself, as for any OpenTelemetry code. + + +```python +from python_wrap import log_context, trace + +@trace(otel=True, span_name="charge-card") +def charge(amount: int) -> None: ... + +with log_context(request_id="r-42"): + charge(5) # span "charge-card", attribute python_wrap.context.request_id = "r-42" +``` + ## `log_context` Bind fields such as a request id once, and have every python-wrap log line made underneath carry them. It works as a context manager (`with` and `async with`) and as a decorator, and blocks nest. @@ -217,8 +234,8 @@ async def run() -> dict[str, object]: assert asyncio.run(run()) == {"job": "nightly"} ``` -!!! note "Threads start empty" - The fields live in a `contextvars.ContextVar`: every asyncio task and thread sees its own. A new thread starts **empty**; to carry the fields over, start it in a copied context with `contextvars.copy_context().run(...)` (`run_in_thread` and `asyncio.to_thread` already do this). +!!! note "New threads and the context" + The fields live in a `contextvars.ContextVar`: every asyncio task and thread sees its own. On a regular CPython build a new thread starts **empty**; to carry the fields over, start it in a copied context with `contextvars.copy_context().run(...)` (`run_in_thread` and `asyncio.to_thread` already do this). Free-threaded builds (3.14t) start threads in a copy of the caller's context by default (`sys.flags.thread_inherit_context`), so the fields carry over on their own. ## See also diff --git a/docs/features/observability/metrics.md b/docs/features/observability/metrics.md index 978af66..559962a 100644 --- a/docs/features/observability/metrics.md +++ b/docs/features/observability/metrics.md @@ -105,9 +105,10 @@ unsubscribe() | Event | Fields | | --- | --- | -| `retry.attempt_failed` | `attempt` (int, 1-based), `exception`, `next_delay` (seconds) | +| `retry.attempt_failed` | `attempt` (int, 1-based), `exception` (`None` for a rejected result), `next_delay` (seconds) | | `retry.succeeded` | `attempts` (total made) | -| `retry.gave_up` | `attempts`, `exception` | +| `retry.gave_up` | `attempts`, `exception` (`None` for a rejected result) | +| `retry.budget_exhausted` | `attempts`, `exception`: a shared `RetryBudget` refused the retry (`retry.gave_up` follows) | | `timeout.expired` | `seconds` (the limit) | | `circuit.state_changed` | `old`, `new` (`closed`, `open`, `half_open`) | | `circuit.rejected` | `retry_after` (seconds) | @@ -117,7 +118,11 @@ unsubscribe() | `cache.hit` | none | | `cache.miss` | none | | `cache.evicted` | `reason` (`lru` or `ttl`) | +| `cache.stale` | none (a stale value was served and a background refresh started) | +| `cache.refresh_failed` | `exception` (the stale value is kept) | | `bulkhead.rejected` | `active`, `waiting` | +| `periodic.run` | `run` (1-based), `seconds` (how long the run took) | +| `periodic.failed` | `run`, `exception` | ```python from python_wrap.events import EVENT_NAMES diff --git a/docs/features/resilience/index.md b/docs/features/resilience/index.md index 4f0fecc..1288275 100644 --- a/docs/features/resilience/index.md +++ b/docs/features/resilience/index.md @@ -8,7 +8,7 @@ Calls to networks, disks and other services fail, stall or get slow. These decor --- - Call again when it raises, with backoff, jitter, `Retry-After` and a time budget. + Call again when it raises or returns a bad value, with backoff, jitter, `Retry-After`, a time budget and a shared retry budget. - :material-electric-switch:{ .lg .middle } **[Circuit breaker](circuit-breaker.md)** diff --git a/docs/features/resilience/retry.md b/docs/features/resilience/retry.md index fde9967..99b5471 100644 --- a/docs/features/resilience/retry.md +++ b/docs/features/resilience/retry.md @@ -64,9 +64,11 @@ The wait before attempt `n + 1` is `min(delay * backoff ** (n - 1), max_delay) + | `jitter` | `0.0` | Up to this many random extra seconds, so many clients do not retry in lockstep. | | `max_time` | `None` | Total time budget in seconds from the first attempt. | | `on` | `Exception` | Exception type(s) that trigger a retry. Anything else propagates immediately. | -| `retry_if` | `None` | Extra predicate on the exception; retry only if it returns `True`. | -| `on_retry` | `None` | Called with a `RetryState` before each wait. | +| `retry_if` | `None` | Extra predicate on the exception; retry only if it returns `True`. May be `async def` for async functions. | +| `retry_if_result` | `None` | Predicate on the return value; `True` means "retry". The last value is returned when retries run out. May be `async def` for async functions. | +| `on_retry` | `None` | Called with a `RetryState` before each wait. May be `async def` for async functions. | | `delay_from` | `None` | `exc -> seconds or None`: exact wait for this attempt (e.g. `Retry-After`). | +| `budget` | `None` | A `RetryBudget` shared with other retried code, so retries stay a fraction of traffic. | | `logger` | package logger | Logger or logger name for the per-retry message. | | `log_level` | `WARNING` | Level of the per-retry message; `None` disables it. | | `log_exc_info` | `False` | Attach the traceback to the per-retry log record. | @@ -104,6 +106,137 @@ assert [s.attempt for s in seen] == [1, 2] assert isinstance(seen[0].exception, HttpError) ``` +## Retrying on a returned value + +Some APIs report "not ready yet" in the return value instead of raising: a job status of `"pending"`, an empty page, `None`. `retry_if_result` receives each returned value; when it returns `True` the attempt counts as failed and is retried with the usual backoff. + +```python +from python_wrap import retry + +statuses = iter(["pending", "pending", "done"]) + +@retry(max_attempts=5, delay=0.01, retry_if_result=lambda status: status == "pending") +def job_status() -> str: + return next(statuses) + +assert job_status() == "done" +``` + +When no retry is left (attempts, `max_time`, deadline or budget), the **last value is returned**, not an exception: the function did not fail, it just never gave the answer you hoped for. Check the value if it matters: + +```python +from python_wrap import retry + +@retry(max_attempts=3, delay=0.01, retry_if_result=lambda status: status == "pending") +def job_status() -> str: + return "pending" + +assert job_status() == "pending" # three attempts, then the last value +``` + +`retry_if_result` combines with `on`/`retry_if`: exceptions and bad values share the same attempt count. In the `RetryState` passed to `on_retry`, `exception` is `None` and `result` holds the rejected value; the `retry.attempt_failed` and `retry.gave_up` events carry `exception=None`. An exception raised by the predicate itself propagates at once. + +!!! note "Why not raise?" + tenacity raises `RetryError` when the result predicate never passes; the `backoff` library returns the last value. python-wrap returns it, so the decorated function keeps its return type and you never have to unwrap a wrapper exception. If you prefer an error, raise one yourself in the function instead of using `retry_if_result`. + +## Async hooks + +For `async def` functions, `retry_if`, `retry_if_result` and `on_retry` may be `async def` callables: they are awaited, so they can look something up or report without blocking the loop. Plain functions keep working. + +```python +import asyncio +from python_wrap import RetryState, retry + +reported: list[int] = [] + +async def is_transient(exc: BaseException) -> bool: + await asyncio.sleep(0) # e.g. ask a feature flag service + return isinstance(exc, ConnectionError) + +async def report(state: RetryState) -> None: + await asyncio.sleep(0) # e.g. send a metric + reported.append(state.attempt) + +errors = iter([ConnectionError("down")]) + +@retry(max_attempts=3, delay=0.01, retry_if=is_transient, on_retry=report) +async def fetch() -> str: + err = next(errors, None) + if err is not None: + raise err + return "payload" + +assert asyncio.run(fetch()) == "payload" +assert reported == [1] +``` + +Nothing can await an async hook around a plain function, so decorating a sync function with one raises `TypeError` straight away: + +```python +from python_wrap import retry + +async def is_transient(exc: BaseException) -> bool: + return True + +try: + @retry(retry_if=is_transient) + def fetch() -> str: + return "payload" +except TypeError as exc: + assert "retry_if= is an async function" in str(exc) +``` + +## Retry budgets + +`max_attempts=3` on every call looks harmless until the dependency goes down: every request now makes three, and the retries keep it down. A `RetryBudget` caps retries at a fraction of the traffic, shared by every function that uses it (the Finagle and Envoy approach): + +- every call's first attempt **deposits** into the budget, every retry **withdraws**; +- a retry is allowed while the retries in the last `window` seconds stay within `ratio` times the requests in that window, plus `min_per_second * window` so low-traffic code can still retry; +- when the budget refuses, `retry` stops at once: no sleep, the last exception is re-raised (or the last value returned with `retry_if_result`), and `retry.budget_exhausted` is emitted. + +```python +from python_wrap import retry +from python_wrap.retry import RetryBudget, retry_stats + +# Retries may add 10 % to traffic, and at least 1 retry per second is always allowed. +budget = RetryBudget(ratio=0.1, min_per_second=1, window=10) +calls = [] + +@retry(max_attempts=5, delay=0, on=ConnectionError, budget=budget) +def get_user() -> dict[str, str]: + calls.append(1) + raise ConnectionError("users service is down") + +@retry(max_attempts=5, delay=0, on=ConnectionError, budget=budget) +def get_orders() -> list[str]: + calls.append(1) + raise ConnectionError("orders service is down") + +for _ in range(10): + for func in (get_user, get_orders): + try: + func() + except ConnectionError: + pass + +# 20 calls; without the budget they would have made 100 attempts. +assert len(calls) == 20 + budget.stats.retries +assert budget.stats.retries <= 0.1 * 20 + 1 * 10 +stats = retry_stats(get_user) +assert stats is not None and stats.budget_exhausted > 0 +``` + +| Argument | Default | Meaning | +| --- | --- | --- | +| `ratio` | `0.1` | Retries allowed per request in the window. | +| `min_per_second` | `10` | Retries per second always allowed, whatever the traffic. | +| `window` | `10.0` | Length of the sliding window in seconds. | + +`budget.stats` returns a `RetryBudgetStats` with `requests` and `retries` (in the current window), `rejected` (refused since creation or the last `reset()`) and `allowed` (retries it would grant right now). One budget is thread-safe and can be shared across threads, event loops, `retry` decorators and `retrying` blocks; it never blocks or sleeps. To give every retry in the project the same budget, set it once with `configure(retry={"budget": RetryBudget()})` before decorating; pass `budget=None` to opt a function out. + +!!! tip "One budget per dependency" + Share a budget between the calls that hit the same service. A budget shared by unrelated services lets a failing one spend the retries the others need. + ## Honouring Retry-After Servers that answer 429 or 503 often say how long to wait. Pass `delay_from` a function that maps the exception to a number of seconds, or to `None` to fall back to the normal backoff. A number is the exact wait for that attempt (no backoff, no jitter), still capped by `max_delay` and `max_time`. @@ -217,11 +350,27 @@ assert 1 <= len(calls) <= 4 assert asyncio.run(main()) == [1, 2, 3] ``` -Use `async for` inside coroutines: a plain `for` would block the event loop while waiting. +Use `async for` inside coroutines: a plain `for` would block the event loop while waiting. With async hooks (`retry_if`, `retry_if_result` or `on_retry` defined with `async def`), also write `async with attempt:` so they can be awaited; a plain `with attempt:` raises `TypeError` for them. + +For `retry_if_result`, store the block's value in `attempt.result`. A rejected value starts another attempt; when no retry is left the loop ends normally and your variable holds the last value: + +```python +from python_wrap.retry import retrying + +pages = iter([[], [], ["row"]]) +rows: list[str] = [] + +for attempt in retrying(max_attempts=5, delay=0.01, retry_if_result=lambda r: not r): + with attempt: + rows = next(pages) + attempt.result = rows + +assert rows == ["row"] +``` ## Statistics -`retry_stats(func)` returns a `RetryStats` with thread-safe counters: `calls`, `attempts_total`, `retries`, `gave_up`, `total_sleep` and `last_exception`. Counters are per decorated function (shared by all instances for methods); `snapshot()` copies them and `reset()` clears them. +`retry_stats(func)` returns a `RetryStats` with thread-safe counters: `calls`, `attempts_total`, `retries`, `gave_up` (including calls that returned a rejected value), `budget_exhausted` (the part of `gave_up` caused by a `RetryBudget`), `total_sleep` and `last_exception`. Counters are per decorated function (shared by all instances for methods); `snapshot()` copies them and `reset()` clears them. ```python from python_wrap import retry @@ -244,7 +393,7 @@ assert (stats.calls, stats.attempts_total, stats.retries, stats.gave_up) == (1, ## Project-wide defaults -Set `max_attempts`, `delay`, `backoff`, `max_delay`, `jitter`, `max_time` and `log_level` once for the whole project with `python_wrap.config.configure(retry={...})`. See [Configuration](../configuration.md). +Set `max_attempts`, `delay`, `backoff`, `max_delay`, `jitter`, `max_time`, `log_level` and `budget` once for the whole project with `python_wrap.config.configure(retry={...})`. See [Configuration](../configuration.md). ## Logging @@ -254,15 +403,16 @@ Each retry logs one message at `log_level` (default `WARNING`) on `logger`. Use | Event | Fields | | --- | --- | -| `retry.attempt_failed` | `attempt` (1-based), `exception`, `next_delay` | +| `retry.attempt_failed` | `attempt` (1-based), `exception` (`None` for a rejected result), `next_delay` | | `retry.succeeded` | `attempts` | -| `retry.gave_up` | `attempts`, `exception` | +| `retry.gave_up` | `attempts`, `exception` (`None` for a rejected result) | +| `retry.budget_exhausted` | `attempts`, `exception`; emitted just before `retry.gave_up` when a `RetryBudget` refuses | Subscribe with `python_wrap.events`; see [Metrics](../observability/metrics.md). ## Testing -`fast_mode()` makes the backoff waits instant, and `fake_clock()` turns them into clock advances you can assert on. `disable_retries()` makes `retry` a single attempt, and the `max_attempts` override from `python_wrap.testing` caps the attempts of every `retry` and `retrying` at call time. See [Testing](../../testing/index.md). +`fast_mode()` makes the backoff waits instant, and `fake_clock()` turns them into clock advances you can assert on. A `RetryBudget` reads time from the same clock, so `fake_clock.advance()` slides its window. `disable_retries()` makes `retry` a single attempt (a budget still receives its per-call deposit, but nothing is withdrawn because no retry happens), and the `max_attempts` override from `python_wrap.testing` caps the attempts of every `retry` and `retrying` at call time. See [Testing](../../testing/index.md). ## Gotchas diff --git a/docs/features/resilience/timeout.md b/docs/features/resilience/timeout.md index 9057e93..f83c6dd 100644 --- a/docs/features/resilience/timeout.md +++ b/docs/features/resilience/timeout.md @@ -105,6 +105,39 @@ The object `timeout(...)` returns is also a context manager, so you can limit a The sync `with timeout(...)` form only exists on top of `SIGALRM`: it needs Unix and the main thread. Elsewhere (Windows, worker threads, `use_signals=False`) entering it raises `RuntimeError`, because a block of code cannot be moved to a worker thread. Decorate a function instead, which falls back to the thread strategy. +## Bounded cleanup with `cleanup_grace` + +Cancelling an `async def` at the limit lets its `finally` / `except CancelledError` cleanup run, but nothing bounds that cleanup. `cleanup_grace=` gives it a budget. + +```python +import asyncio +from python_wrap import TimeoutExpired, timeout + +cleaned = [] + +@timeout(0.02, cleanup_grace=0.5) +async def work() -> None: + try: + await asyncio.sleep(5) + finally: + await asyncio.sleep(0.02) # e.g. release a connection + cleaned.append("released") + +try: + asyncio.run(work()) +except TimeoutExpired: + pass +assert cleaned == ["released"] +``` + +As a **decorator**, the call runs as a child task. At the limit it is cancelled and given `cleanup_grace` more seconds to finish; then `TimeoutExpired` is raised (also when the cleanup finished in time). A task still running after the grace is **abandoned**: a warning is logged on the package logger and the task keeps running until it ends by itself, its result discarded. If the caller is cancelled from outside, the child is cancelled too and given the same grace before the caller's `CancelledError` propagates. + +Because the decorated call runs in a **separate task**, `contextvars` it sets are not visible to the caller afterwards, and `asyncio.current_task()` inside it is the child task, not the caller's. Without `cleanup_grace` the call runs in the caller's own task as usual. + +As **`async with timeout(5, cleanup_grace=1)`**, the block runs in your own task and cannot be moved, so it is not abandoned: after the grace the cleanup is cancelled a **second time** (with a warning). Cleanup that swallows cancellation cannot be stopped. Only the timer's own cancellations become `TimeoutExpired`: if your task is cancelled from outside during the grace, the block raises `CancelledError`, as `asyncio.timeout` does. + +Sync functions and plain `with` blocks do not support it and raise `TypeError`: signals and worker threads cannot give cleanup a bounded grace. The grace uses real time and is not changed by the test overrides. + ## Deadlines: one budget for many steps A `timeout` limits one call. A **deadline** limits everything inside a block (several calls, retries, sub-steps) to one shared budget. `deadline(seconds)` stores an absolute expiry in a context variable, so it is visible to every function called inside the block, sync or async, and to tasks started from it. Nested deadlines never extend an outer one: the earliest wins. diff --git a/docs/get-started/async-and-methods.md b/docs/get-started/async-and-methods.md index 7d7679e..0c92226 100644 --- a/docs/get-started/async-and-methods.md +++ b/docs/get-started/async-and-methods.md @@ -86,6 +86,47 @@ Decorators work on instance methods; `self` is an ordinary first argument. Two t Both are explained, with workarounds, in [Limitations](../project/limitations.md#shared-state-on-methods). To apply one decorator to every method of a class, see [`wrap_methods`](../features/devtools.md). +## Process pools and pickling + +Decorated functions pickle **by reference**, like plain functions: `pickle` stores the module and qualified name, and loading it finds the same object again. You can submit a decorated module-level function to a `multiprocessing` pool or a `ProcessPoolExecutor`, including with the `"spawn"` start method (the default on macOS and Windows). This also covers objects that are not functions, such as `periodic`, and classes decorated with `wrap_methods`. + +```python +import pickle +from python_wrap import count_calls, memoize, periodic, retry + +@memoize +@retry(max_attempts=3, delay=0.01) +def load(key: str) -> str: + return key.upper() + +@periodic(60) +def refresh() -> None: ... + +assert pickle.loads(pickle.dumps(load)) is load +assert pickle.loads(pickle.dumps(refresh)) is refresh +``` + + +```python +# jobs.py: workers re-import this module by name, so the function must live at module level +import concurrent.futures +import multiprocessing +from python_wrap import rate_limit, retry + +@retry(max_attempts=3, delay=0.1) +@rate_limit(10, 1) +def fetch(url: str) -> int: ... + +if __name__ == "__main__": + ctx = multiprocessing.get_context("spawn") + with concurrent.futures.ProcessPoolExecutor(4, mp_context=ctx) as pool: + sizes = list(pool.map(fetch, ["https://a.example", "https://b.example"])) +``` + +**Limiter, breaker and cache state is per process.** Each worker imports the module and builds its own decorated function, so it has its own cache, rate limiter, circuit breaker, bulkhead and call counters. Above, `rate_limit(10, 1)` allows 10 calls per second *in each worker*, not across the pool. Nothing is shared or sent back to the parent. + +Objects that hold that state refuse to pickle with a clear `TypeError`: `CircuitBreaker`, `RateLimiter`, `Bulkhead`, `RetryBudget`, `Transaction`, `Policy` and `testing.FakeClock`. Create them at module level so each process builds its own. Instances whose class uses `cached_property` pickle normally, but the cached values are not pickled: the copy computes them again on first access. See [Limitations](../project/limitations.md#pickling-and-process-pools). + ## Gotchas - **One event loop per `debounce` and `singleflight` burst.** Their state is tracked per loop; do not share a debounced coroutine across loops and expect them to coordinate. diff --git a/docs/project/limitations.md b/docs/project/limitations.md index 8b9a126..a0867f6 100644 --- a/docs/project/limitations.md +++ b/docs/project/limitations.md @@ -51,6 +51,7 @@ What to do: Mitigations: +- Use `per_instance=True`: each instance gets its own cache through a weak reference, so instances stay collectable (they must be weak-referenceable). See [Per-instance method caches](../features/caching/memoize.md#per-instance-method-caches). - Set `maxsize` and/or `ttl` so entries (and the instances they pin) are eventually dropped. - Use `key=` that ignores `self` (for results that do not depend on the instance), so the instance is not stored. - Call `cache_clear()` when a long-lived cache has served its purpose. @@ -70,6 +71,12 @@ It restores attribute values from a copy. In-place mutation of a container (`sel If your function has a `tx` parameter, it is passed as the keyword `tx=`. Put `tx` **last or keyword-only** so callers cannot supply it positionally by accident. A rollback callback that raises is logged, and the remaining callbacks still run; a commit callback that raises propagates and skips the remaining commit callbacks. +## Pickling and process pools + +Decorated functions pickle by reference, so the rules are those of plain functions: the function must be reachable by its module and qualified name. Functions defined inside another function, lambdas, and functions defined in `__main__` of an interactive session cannot be sent to a `"spawn"` worker. + +State does not cross processes. Each worker has its own cache, rate limiter, circuit breaker, bulkhead and counters; there is no pool-wide limit. `CircuitBreaker`, `RateLimiter`, `Bulkhead`, `RetryBudget`, `Transaction`, `Policy` and `testing.FakeClock` raise `TypeError` when pickled or copied. A `cached_property` value is not pickled with its instance and is computed again after unpickling. See [Process pools and pickling](../get-started/async-and-methods.md#process-pools-and-pickling). + ## Other things to know - **`debounce`** (sync) runs the function on a daemon timer thread, so it may be abandoned at interpreter exit with a call still pending. Call `.flush()` before shutting down if the last call matters. diff --git a/mkdocs.yml b/mkdocs.yml index eebada6..2d09ad1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,7 +1,7 @@ site_name: python-wrap site_description: Everyday Python decorators that just work. Retry, cache, time out, rate limit and more. Typed, async-aware, zero dependencies. site_author: Mahmoud Ahmed -site_url: https://python-wrap.readthedocs.io/ +site_url: https://pythonwrap.readthedocs.io/ repo_url: https://github.com/LitePy/PythonWrap repo_name: LitePy/PythonWrap edit_uri: edit/main/docs/ @@ -168,6 +168,7 @@ nav: - Bulkheads: features/concurrency/bulkheads.md - Async helpers: features/concurrency/async-helpers.md - Run in a thread: features/concurrency/run-in-thread.md + - Periodic tasks: features/concurrency/periodic.md - Observability: - features/observability/index.md - Logging: features/observability/logging.md @@ -206,6 +207,7 @@ nav: - Concurrency: api/concurrency.md - Async helpers: api/aio.md - Bulkhead: api/bulkhead.md + - Periodic: api/periodic.md - Observability: - Logging and audit: api/logs.md - Timing and profiling: api/timing.md diff --git a/pyproject.toml b/pyproject.toml index e48cc31..91265f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [] [project.urls] Homepage = "https://github.com/LitePy/PythonWrap" -Documentation = "https://python-wrap.readthedocs.io" +Documentation = "https://pythonwrap.readthedocs.io" Source = "https://github.com/LitePy/PythonWrap" Issues = "https://github.com/LitePy/PythonWrap/issues" Changelog = "https://github.com/LitePy/PythonWrap/blob/main/CHANGELOG.md" @@ -59,11 +59,13 @@ dev = [ "hypothesis>=6.100", "ruff>=0.8", "mypy>=1.13", + "opentelemetry-api>=1.45.1", + "opentelemetry-sdk>=1.45.1", ] docs = [ "mkdocs-material>=9.5", "mkdocs-redirects>=1.2.3", - "mkdocstrings[python]>=0.27", + "mkdocstrings[python]>=1.0.6", ] release = ["build>=1.2", "twine>=6.0"] @@ -78,6 +80,9 @@ exclude = ["**/__pycache__", "**/*.py[cod]", "**/.DS_Store"] include = [ "/src/python_wrap", "/tests", + "/examples", + "/benchmarks", + "/scripts/check_installed.py", "/README.md", "/LICENSE", "/CHANGELOG.md", diff --git a/scripts/check_installed.py b/scripts/check_installed.py new file mode 100644 index 0000000..1a4c679 --- /dev/null +++ b/scripts/check_installed.py @@ -0,0 +1,65 @@ +"""Check an installed distribution: python -I scripts/check_installed.py VERSION. + +Run with an isolated interpreter in a clean environment so the checkout cannot +mask missing wheel files. Uses only the standard library and python-wrap. +""" + +from __future__ import annotations + +import asyncio +import importlib +import importlib.metadata +import inspect +import pkgutil +import sys +from pathlib import Path + +import python_wrap +from python_wrap import memoize, retry + + +def main() -> None: + expected = sys.argv[1] + metadata = importlib.metadata.metadata("python-wrap") + assert metadata["Version"] == python_wrap.__version__ == expected + assert not metadata.get_all("Requires-Dist"), "Unexpected runtime dependencies" + assert metadata["Requires-Python"] == ">=3.10" + assert metadata["License-Expression"] == "MIT" + assert (Path(python_wrap.__file__).parent / "py.typed").is_file() + for module in pkgutil.iter_modules(python_wrap.__path__): + importlib.import_module(f"python_wrap.{module.name}") + for name in python_wrap.__all__: + assert hasattr(python_wrap, name), name + + attempts = 0 + + @retry(max_attempts=2, delay=0) + def flaky() -> str: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise ValueError("temporary") + return "ok" + + assert flaky() == "ok" and attempts == 2 + calls = 0 + + @memoize + def cached(value: int) -> int: + nonlocal calls + calls += 1 + return value * 2 + + assert cached(3) == cached(3) == 6 and calls == 1 + + @retry(max_attempts=2, delay=0) + async def asynchronous(value: int) -> int: + return value + 1 + + assert inspect.iscoroutinefunction(asynchronous) + assert asyncio.run(asynchronous(2)) == 3 + print(f"python-wrap {expected}: metadata, imports, typing marker, sync and async checks passed") + + +if __name__ == "__main__": + main() diff --git a/src/python_wrap/__init__.py b/src/python_wrap/__init__.py index f4aa547..480c28c 100644 --- a/src/python_wrap/__init__.py +++ b/src/python_wrap/__init__.py @@ -30,6 +30,7 @@ from .caching import ( CachedProperty, CacheInfo, + CacheStore, Memoized, Once, cache, @@ -107,12 +108,19 @@ Counted, count_calls, ) +from .periodic import ( + AsyncPeriodic, + Periodic, + periodic, +) from .policy import ( Policy, policy, ) from .retry import ( Attempt, + RetryBudget, + RetryBudgetStats, Retrying, RetryState, RetryStats, @@ -159,6 +167,7 @@ "ArgumentTypeError", "AsyncBatched", "AsyncDebounced", + "AsyncPeriodic", "Attempt", "AuditRecord", "BatchError", @@ -167,6 +176,7 @@ "Bulkhead", "BulkheadFull", "CacheInfo", + "CacheStore", "CachedProperty", "CallStats", "CircuitBreaker", @@ -181,11 +191,14 @@ "Memoized", "MemoryRecord", "Once", + "Periodic", "Policy", "PythonWrapError", "RaceFailed", "RateLimitExceeded", "RateLimiter", + "RetryBudget", + "RetryBudgetStats", "RetryState", "RetryStats", "Retrying", @@ -231,6 +244,7 @@ "mock_data", "no_debug", "once", + "periodic", "policy", "profile", "race", diff --git a/src/python_wrap/_core.py b/src/python_wrap/_core.py index 89caa6e..11dd2e2 100644 --- a/src/python_wrap/_core.py +++ b/src/python_wrap/_core.py @@ -16,6 +16,11 @@ and ``time.monotonic()``), so the test overrides and the fake clock never shorten it, and sync and async code behave the same. * Honour the process-wide test overrides (see below). +* Decorated functions pickle by reference (``module.qualname``), so they can be sent to + process pools. A decorator that returns an object instead of a function gives it a + ``__reduce__`` returning its ``__qualname__``. Objects holding shared runtime state + (locks, counters, timers) refuse to pickle through `refuse_pickle`: that state is + per process. Test overrides contract ----------------------- @@ -50,7 +55,7 @@ import time from collections.abc import Callable from dataclasses import dataclass, fields -from typing import Any, ParamSpec, TypeVar +from typing import Any, NoReturn, ParamSpec, TypeVar P = ParamSpec("P") R = TypeVar("R") @@ -151,6 +156,25 @@ def qualname(func: Callable[..., Any]) -> str: return getattr(func, "__qualname__", None) or getattr(func, "__name__", None) or repr(func) +def refuse_pickle(obj: object, advice: str | None = None) -> NoReturn: + """Raise a clear ``TypeError`` from ``__reduce__`` of an object with per-process state. + + Locks, counters and timers cannot be shared with another process, so pickling (and + copying) such an object fails here instead of deep inside ``pickle``. ``advice`` + replaces the default hint for objects that are not used as decorators. + """ + name = type(obj).__name__ + if advice is None: + advice = ( + f"Create one {name} per process instead (e.g. at module level, so each worker " + "builds its own); functions decorated with it pickle by reference." + ) + raise TypeError( + f"cannot pickle {name!r} object: its state (locks, counters, timers) belongs to " + f"one process. {advice}" + ) + + # ---------------------------------------------------------------------------- events # One process-wide hook for metrics. Decorators call ``_core.emit(...)`` at interesting # moments (a retry, a cache hit, a circuit opening); ``python_wrap.events`` is the public diff --git a/src/python_wrap/bulkhead.py b/src/python_wrap/bulkhead.py index e18f878..d382c8e 100644 --- a/src/python_wrap/bulkhead.py +++ b/src/python_wrap/bulkhead.py @@ -17,7 +17,7 @@ import weakref from collections.abc import Callable from types import TracebackType -from typing import Any, Concatenate, ParamSpec, Protocol, TypeVar, cast, overload +from typing import Any, Concatenate, NoReturn, ParamSpec, Protocol, TypeVar, cast, overload from python_wrap import _core from python_wrap._core import F, P, R @@ -130,6 +130,9 @@ async def main() -> None: ``` """ + def __reduce__(self) -> NoReturn: + _core.refuse_pickle(self) + def __init__( self, max_concurrent: int, diff --git a/src/python_wrap/caching.py b/src/python_wrap/caching.py index 54fc181..ffbe7dc 100644 --- a/src/python_wrap/caching.py +++ b/src/python_wrap/caching.py @@ -4,19 +4,33 @@ import asyncio import concurrent.futures +import contextlib +import contextvars import datetime import functools import inspect import threading +import time import weakref from collections import OrderedDict from collections.abc import Awaitable, Callable, Coroutine, Hashable -from typing import Any, Concatenate, Generic, NamedTuple, ParamSpec, Protocol, TypeVar, overload +from typing import ( + Any, + Concatenate, + Generic, + NamedTuple, + ParamSpec, + Protocol, + TypeVar, + overload, + runtime_checkable, +) from python_wrap import _core __all__ = [ "CacheInfo", + "CacheStore", "CachedProperty", "Memoized", "Once", @@ -214,8 +228,12 @@ def _freeze_arg(name: str, value: Any) -> Hashable: def _make_key_builder( - func: Callable[..., Any], typed: bool, key: Callable[..., Hashable] | None + func: Callable[..., Any], + typed: bool, + key: Callable[..., Hashable] | None, + skip_first: bool = False, ) -> Callable[[tuple[Any, ...], dict[str, Any]], Hashable]: + """Build cache keys; ``skip_first`` leaves the first argument (``self``) out.""" if key is not None: user_key = key @@ -229,14 +247,16 @@ def custom(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Hashable: except (TypeError, ValueError): sig = None + skip = 1 if skip_first else 0 + def build(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Hashable: if sig is not None: bound = sig.bind(*args, **kwargs) bound.apply_defaults() - named = list(bound.arguments.items()) + named = list(bound.arguments.items())[skip:] frozen = tuple(_freeze_arg(n, v) for n, v in named) else: - named = [(f"#{i}", v) for i, v in enumerate(args)] + named = [(f"#{i}", v) for i, v in enumerate(args)][skip:] frozen = tuple(_freeze_arg(n, v) for n, v in named) if kwargs: # Keep the names (and a marker), or f(1, b=2) would equal f(1, 2). @@ -305,7 +325,7 @@ def _start( coro: Coroutine[Any, Any, Any], shared: asyncio.Future[Any], on_done: Callable[[asyncio.Future[Any] | None], None], -) -> None: +) -> asyncio.Future[Any]: """Run ``coro`` as a task and copy its outcome into ``shared``. Call it WITHOUT holding a lock: under an eager task factory (Python 3.12+) the @@ -330,6 +350,7 @@ def finish(t: asyncio.Future[Any]) -> None: _copy_outcome(t, shared) task.add_done_callback(finish) + return task def _ttl_seconds(ttl: float | datetime.timedelta | None) -> float | None: @@ -369,10 +390,137 @@ def _fast_arity(func: Callable[..., Any], typed: bool, key: Callable[..., Hashab return len(params) -# lookup() returns a stored entry ``(value, expires, is_exception)``, ``None`` (absent) or -# this falsy marker (the entry had expired and was just removed). +# fetch() returns a stored entry, ``None`` (absent) or this falsy marker (the entry had +# expired and was just removed). An entry is ``(value, expires, is_exception, fresh_until)``: +# ``expires`` is when it is dropped (None: never), ``fresh_until`` when it turns stale +# (equal to ``expires`` unless ``stale_ttl`` is set). _EXPIRED: Any = () +# Deadlines in entries given to a custom `CacheStore` are Unix-epoch seconds, so processes +# sharing one store agree on them. They are still read through ``_core.monotonic()`` (plus +# this constant offset) so ``fake_clock`` drives them too. +_WALL_OFFSET = time.time() - time.monotonic() + + +@runtime_checkable +class CacheStore(Protocol): + """Where `memoize` keeps its entries when given ``store=...``. + + Implement these five methods on any object (no base class needed): + + * ``get(key)`` returns the entry stored under ``key``, or ``None`` if there is none. + * ``set(key, entry, ttl)`` stores ``entry``. ``ttl`` is the number of seconds after + which the entry is useless and may be discarded (``None``: keep it until deleted). + * ``delete(key)`` removes the entry and returns True if there was one. + * ``clear()`` removes every entry. + * ``__len__()`` is the number of stored entries (``cache_info().currsize``). + + Contract: + + * Entries are opaque: return exactly what was stored (a store that serialises, such + as a Redis adapter, must round-trip them, e.g. with `pickle`). They are never None. + They hold their own deadlines, as Unix timestamps, and `memoize` checks them, so a + store may forget entries at any time (that is just a miss) and need not expire them. + * Keys are hashable: normally tuples of the (normalised) arguments, or whatever + ``key=`` returns. They are **not** strings, so a store that needs strings or bytes + must serialise them itself (``repr`` or ``pickle`` plus a per-function prefix). + * The store must be thread-safe: `memoize` does not hold its lock while calling it, + so a slow store (a network round trip) never blocks other cached functions. Plain + ``dict`` operations are atomic in CPython, so a dict-backed store qualifies. + * Methods are synchronous; for ``async def`` functions they run on the event loop, + so keep them fast. + * Exceptions raised by the store propagate to the caller. + + Example: + >>> class DictStore: + ... def __init__(self): self.data = {} + ... def get(self, key): return self.data.get(key) + ... def set(self, key, entry, ttl): self.data[key] = entry + ... def delete(self, key): return self.data.pop(key, None) is not None + ... def clear(self): self.data.clear() + ... def __len__(self): return len(self.data) + >>> @memoize(store=DictStore()) + ... def double(n): + ... return n * 2 + >>> double(2) + 4 + """ + + def get(self, key: Hashable, /) -> Any: + """Return the entry stored under ``key``, or None.""" + + def set(self, key: Hashable, entry: Any, ttl: float | None, /) -> None: + """Store ``entry`` under ``key``; it may be discarded after ``ttl`` seconds.""" + + def delete(self, key: Hashable, /) -> bool: + """Remove the entry under ``key``; True if there was one.""" + + def clear(self) -> None: + """Remove every entry.""" + + def __len__(self) -> int: + """The number of stored entries.""" + + +class _LocalStore: + """The built-in store: an ordered dict with an optional LRU limit and TTL sweeps. + + Not thread-safe by itself: the decorator holds its lock around every call. + """ + + __slots__ = ("data", "expiring", "inserts", "live", "maxsize") + + def __init__(self, maxsize: int | None, expiring: bool) -> None: + self.data: OrderedDict[Hashable, tuple[Any, float | None, bool, float | None]] = ( + OrderedDict() + ) + self.maxsize = maxsize + self.expiring = expiring + self.inserts = 0 + self.live = 0 + + def get(self, k: Hashable) -> Any: + return self.data.get(k) + + def delete(self, k: Hashable) -> bool: + return self.data.pop(k, _MISSING) is not _MISSING + + def clear(self) -> None: + self.data.clear() + self.inserts = self.live = 0 + + def __len__(self) -> int: + return len(self.data) + + def sweep(self) -> int: + """Drop every expired entry; returns how many.""" + now = _core.monotonic() + data = self.data + dead = [k for k, e in data.items() if e[1] is not None and now >= e[1]] + for k in dead: + del data[k] + return len(dead) + + def put(self, k: Hashable, entry: tuple[Any, float | None, bool, float | None]) -> int: + """Store an entry; returns ``ttl_evicted * 2**32 + lru_evicted`` (cheap to pass).""" + data = self.data + data[k] = entry + ttl_evicted = lru_evicted = 0 + if self.maxsize is not None: + data.move_to_end(k) + if self.expiring: + # Amortised O(1): a full sweep only after at least + # max(64, entries left by the previous sweep) inserts. + self.inserts += 1 + if self.inserts >= max(64, self.live): + ttl_evicted = self.sweep() + self.inserts, self.live = 0, len(data) + if self.maxsize is not None: + while len(data) > self.maxsize: + data.popitem(last=False) + lru_evicted += 1 + return (ttl_evicted << 32) | lru_evicted + @overload def memoize(func: Callable[P, R], /) -> Memoized[P, R]: ... @@ -383,10 +531,14 @@ def memoize( *, maxsize: int | None = None, ttl: float | datetime.timedelta | Callable[[Any], float | None] | None = None, + stale_ttl: float | datetime.timedelta | None = None, typed: bool = False, key: Callable[..., Hashable] | None = None, cache_exceptions: type[BaseException] | tuple[type[BaseException], ...] = (), exception_ttl: float | datetime.timedelta | None = None, + store: CacheStore | None = None, + per_instance: bool = False, + logger: _core.LoggerLike = None, ) -> Callable[[Callable[P, R]], Memoized[P, R]]: ... def memoize( func: Callable[..., Any] | None = None, @@ -394,10 +546,14 @@ def memoize( *, maxsize: int | None = None, ttl: float | datetime.timedelta | Callable[[Any], float | None] | None = None, + stale_ttl: float | datetime.timedelta | None = None, typed: bool = False, key: Callable[..., Hashable] | None = None, cache_exceptions: type[BaseException] | tuple[type[BaseException], ...] = (), exception_ttl: float | datetime.timedelta | None = None, + store: CacheStore | None = None, + per_instance: bool = False, + logger: _core.LoggerLike = None, ) -> Any: """Cache a function's results per argument values. @@ -415,14 +571,22 @@ def memoize( result is cached and concurrent awaits of the same missing key share a single in-flight task per event loop, so it runs once. - A method of a class whose instances are unhashable (for example an ``eq=True`` - dataclass) fails with a ``TypeError`` naming ``self``: ``id(self)`` is deliberately not - used as a fallback because ids are reused after garbage collection. Pass ``key=`` or - use `cached_property` instead. - - Methods: ``self`` is part of the key, so results are per instance. The cache holds a - strong reference to every instance it has seen, keeping it alive; use ``maxsize`` or - ``ttl`` (or ``key=`` ignoring ``self``) on methods of many short-lived objects. + Stale-while-revalidate: with ``stale_ttl`` an entry older than ``ttl`` is *stale* but + still served, immediately, for ``stale_ttl`` more seconds; the call that finds it + stale starts one background refresh (``async def``: a task on the running event + loop, shared with concurrent misses; sync: a daemon thread running in a copy of the + caller's `contextvars` context, at most one per key). A failed refresh keeps the + stale value, is logged at WARNING and lets the next call try again. Cached + exceptions are never served stale. + + Methods: ``self`` is part of the key, so results are per instance, and the cache + holds a strong reference to every instance it has seen, keeping it alive. A class + whose instances are unhashable (for example an ``eq=True`` dataclass) fails with a + ``TypeError`` naming ``self``. ``per_instance=True`` solves both: each instance gets + its own cache, found by identity through a weak reference, so instances stay + collectable (their entries go with them) and need not be hashable. They must be + weak-referenceable (a ``__slots__`` class must list ``'__weakref__'``), and a cached + result that references its instance keeps it alive. The wrapper has ``cache_clear()``, ``cache_info()`` (a `CacheInfo`), ``cache_delete(*args, **kwargs)``, ``cache_contains(*args, **kwargs)``, @@ -433,28 +597,42 @@ def memoize( While ``python_wrap.testing.disable_caching()`` is active the cache is neither read nor written. Listeners registered with `python_wrap.events` receive ``cache.hit``, - ``cache.miss`` and ``cache.evicted`` (field ``reason``: ``"lru"`` or ``"ttl"``). + ``cache.miss``, ``cache.evicted`` (field ``reason``: ``"lru"`` or ``"ttl"``), + ``cache.stale`` (a stale value was served and a refresh started) and + ``cache.refresh_failed`` (field ``exception``). Args: func: The function (when used bare). maxsize: Maximum entries; least recently used are evicted. None is unbounded. + With ``per_instance`` the limit applies to each instance. ttl: Seconds (or a ``timedelta``) an entry stays valid. None never expires. May also be a callable ``ttl(result) -> float | None`` evaluated when a result is stored (None: never expires; zero or negative: not stored). + stale_ttl: Seconds (or a ``timedelta``) after ``ttl`` during which the stale + value is still served while a background refresh runs. Requires ``ttl``. typed: If True, ``f(1)`` and ``f(1.0)`` are cached separately. - key: Custom ``key(*args, **kwargs)`` returning a hashable cache key. + key: Custom ``key(*args, **kwargs)`` returning a hashable cache key. It receives + every argument (``self`` too; with ``per_instance`` it should leave it out). cache_exceptions: Exception class (or tuple of classes) whose instances are cached and re-raised like results. ``asyncio.CancelledError`` never is. exception_ttl: Seconds a cached exception stays valid. Defaults to ``ttl``; it must be given when ``ttl`` is a callable and ``cache_exceptions`` is set. + store: A `CacheStore` to keep entries in instead of the built-in LRU dict. + per_instance: For methods: one cache per instance, held through a weak + reference, keyed by the arguments after ``self``. + logger: Logger (or name) for failed background refreshes. Returns: The wrapped function, or a decorator when called with options. Raises: - ValueError: If ``maxsize`` is below 1 or a ttl is not positive. - TypeError: For a non-exception in ``cache_exceptions``; at call time, for an - unhashable argument (see above). + ValueError: If ``maxsize`` is below 1, a ttl is not positive, or ``stale_ttl`` + is given without ``ttl``. + TypeError: For a non-exception in ``cache_exceptions``, an object that is not a + `CacheStore`, ``store`` combined with ``maxsize`` or ``per_instance``, or + ``per_instance`` on a function without parameters. At call time, for an + unhashable argument (see above) or, with ``per_instance``, an instance that + cannot be weakly referenced. Example: >>> @memoize(maxsize=128, ttl=60) @@ -467,75 +645,141 @@ def memoize( """ if maxsize is not None and maxsize < 1: raise ValueError("maxsize must be a positive integer or None") + if store is not None: + if maxsize is not None: + raise TypeError("maxsize only applies to the built-in store; not with store=") + if per_instance: + raise TypeError("per_instance=True uses the built-in store; not with store=") + if not isinstance(store, CacheStore): + raise TypeError("store must implement CacheStore: get, set, delete, clear and __len__") ttl_fn: Callable[[Any], float | None] | None = None ttl_s: float | None = None if callable(ttl): ttl_fn = ttl else: ttl_s = _ttl_seconds(ttl) + if stale_ttl is not None and ttl is None: + raise ValueError("stale_ttl requires ttl") + stale_s = _ttl_seconds(stale_ttl) exc_types = _exception_types(cache_exceptions) exc_ttl_s = _ttl_seconds(exception_ttl) if exception_ttl is not None else ttl_s if exc_types and ttl_fn is not None and exception_ttl is None: raise ValueError("exception_ttl is required when ttl is a callable") lru = maxsize is not None + expiring = ttl_s is not None or ttl_fn is not None or exc_ttl_s is not None + builtin = store is None ov = _core.overrides + if builtin: + + def now() -> float: + return _core.monotonic() + else: + + def now() -> float: + return _core.monotonic() + _WALL_OFFSET + def decorate(f: Callable[..., Any]) -> Any: - build_key = _make_key_builder(f, typed, key) + build_key = _make_key_builder(f, typed, key, skip_first=per_instance) fast_n = _fast_arity(f, typed, key) + if per_instance and fast_n == 0: + raise TypeError("per_instance=True needs a method (a function taking self)") name = _core.qualname(f) - store: OrderedDict[Hashable, tuple[Any, float | None, bool]] = OrderedDict() + main: Any = _LocalStore(maxsize, expiring) if builtin else store + # per_instance: id(instance) -> (weak reference, that instance's store) + instances: dict[int, tuple[weakref.ref[Any], _LocalStore]] = {} # In-flight tasks are per event loop: a task cannot be awaited from another loop. inflight: weakref.WeakKeyDictionary[ asyncio.AbstractEventLoop, dict[Hashable, asyncio.Future[Any]] ] = weakref.WeakKeyDictionary() - store_get = store.get + refreshing: set[Hashable] = set() # sync keys being refreshed in a thread + background: set[asyncio.Future[Any]] = set() # strong refs to refresh tasks lock = threading.Lock() - hits = misses = gen = inserts = live = 0 + guard: Any = lock if builtin else contextlib.nullcontext() + hits = misses = gen = 0 def key_of(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Hashable: if len(args) == fast_n and not kwargs: + k = args[1:] if per_instance else args try: - hash(args) + hash(k) except TypeError: pass else: - return args + return k return build_key(args, kwargs) - def sweep() -> int: - # Caller holds the lock. Drop every expired entry; returns how many. - now = _core.monotonic() - dead = [k for k, (_, exp, _e) in store.items() if exp is not None and now >= exp] - for k in dead: - del store[k] - return len(dead) - - def lookup(k: Hashable) -> Any: - # Caller holds the lock. - entry = store.get(k) + def bucket_of(args: tuple[Any, ...], create: bool = True) -> Any: + if not per_instance: + return main + if not args: + if not create: + return None + raise TypeError( + f"memoize(per_instance=True): call {name} with the instance as its " + "first positional argument" + ) + obj = args[0] + i = id(obj) + slot = instances.get(i) + if slot is not None and slot[0]() is obj: + return slot[1] + if not create: + return None + + def forget(ref: weakref.ref[Any], i: int = i) -> None: + # Runs while the instance is being destroyed, maybe in a thread that holds + # ``lock``: so no locking here (dict operations are atomic). The id cannot + # be reused before this callback has run. + current = instances.get(i) + if current is not None and current[0] is ref: + instances.pop(i, None) + + try: + ref = weakref.ref(obj, forget) + except TypeError: + raise TypeError( + f"memoize(per_instance=True) needs weak-referenceable instances, but " + f"{type(obj).__name__!r} is not (a __slots__ class must also list " + "'__weakref__')" + ) from None + with lock: + slot = instances.get(i) + if slot is not None and slot[0]() is obj: + return slot[1] + bucket = _LocalStore(maxsize, expiring) + instances[i] = (ref, bucket) + return bucket + + def live_buckets() -> list[_LocalStore]: + # Caller holds the lock. A collected instance may drop its slot meanwhile. + while True: + try: + slots = list(instances.values()) + except RuntimeError: # pragma: no cover - changed size during iteration + continue + return [b for ref, b in slots if ref() is not None] + + def fetch(b: Any, k: Hashable) -> Any: + # Caller holds ``guard``. + entry = b.get(k) if entry is None: return None expires = entry[1] - if expires is not None and _core.monotonic() >= expires: - del store[k] + if expires is not None and now() >= expires: + b.delete(k) return _EXPIRED if lru: - store.move_to_end(k) + b.data.move_to_end(k) return entry - def peek(k: Hashable) -> tuple[Any, float | None, bool] | None: - with lock: - entry = store.get(k) - if entry is None or (entry[1] is not None and _core.monotonic() >= entry[1]): - return None - return entry + def is_stale(entry: Any) -> bool: + return entry[1] is not None and not entry[2] and now() >= entry[3] def report_expired() -> None: _core.emit("cache.evicted", name, reason="ttl") - def save(k: Hashable, value: Any, g: int, is_exc: bool = False) -> None: - nonlocal inserts, live + def save(b: Any, k: Hashable, value: Any, g: int, is_exc: bool = False) -> None: if is_exc: seconds = exc_ttl_s elif ttl_fn is not None: @@ -544,58 +788,157 @@ def save(k: Hashable, value: Any, g: int, is_exc: bool = False) -> None: return # not worth storing else: seconds = ttl_s - ttl_evicted = lru_evicted = 0 - with lock: + keep = seconds + if seconds is not None and stale_s is not None and not is_exc: + keep = seconds + stale_s + evicted = 0 + with guard: if g != gen: return # cache_clear() ran while computing - expires = None if seconds is None else _core.monotonic() + seconds - store[k] = (value, expires, is_exc) - if lru: - store.move_to_end(k) - if ttl_s is not None or ttl_fn is not None or exc_ttl_s is not None: - # Amortised O(1): a full sweep only after at least - # max(64, entries left by the previous sweep) inserts. - inserts += 1 - if inserts >= max(64, live): - ttl_evicted = sweep() - inserts, live = 0, len(store) - if maxsize is not None: - while len(store) > maxsize: - store.popitem(last=False) - lru_evicted += 1 - if _core.listeners: - for _ in range(ttl_evicted): + t = now() + fresh = None if seconds is None else t + seconds + expires = None if keep is None else t + keep + entry = (value, expires, is_exc, fresh) + if builtin: + evicted = b.put(k, entry) + else: + b.set(k, entry, keep) + if evicted and _core.listeners: + for _ in range(evicted >> 32): report_expired() - for _ in range(lru_evicted): + for _ in range(evicted & 0xFFFFFFFF): _core.emit("cache.evicted", name, reason="lru") + def refresh_failed(exc: BaseException) -> None: + _core.get_logger(logger).warning( + "memoize: background refresh of %s failed; keeping the stale value", + name, + exc_info=exc, + ) + _core.emit("cache.refresh_failed", name, exception=exc) + + def refresh_in_thread( + b: Any, k: Hashable, args: tuple[Any, ...], kwargs: dict[str, Any] + ) -> None: + rk = (id(b), k) if per_instance else k + with lock: + if rk in refreshing: + return + refreshing.add(rk) + g = gen + + def run() -> None: + try: + save(b, k, f(*args, **kwargs), g) + except Exception as exc: + refresh_failed(exc) + finally: + with lock: + refreshing.discard(rk) + + thread = threading.Thread( + target=contextvars.copy_context().run, + args=(run,), + name=f"python_wrap-refresh-{name}", + daemon=True, + ) + try: + thread.start() + except Exception as exc: # e.g. "can't start new thread": keep serving stale + with lock: + refreshing.discard(rk) + refresh_failed(exc) + return + if _core.listeners: + _core.emit("cache.stale", name) + + def refresh_in_task( + b: Any, k: Hashable, args: tuple[Any, ...], kwargs: dict[str, Any] + ) -> None: + rk = (id(b), k) if per_instance else k + with lock: + loop = asyncio.get_running_loop() + loop_inflight = inflight.setdefault(loop, {}) + if rk in loop_inflight: + return # a refresh (or a miss) for this key is already running + g = gen + shared_fut = loop_inflight[rk] = _shared_future() + + async def run() -> Any: + try: + result = await f(*args, **kwargs) + save(b, k, result, g) + except Exception as exc: + refresh_failed(exc) + raise + return result + + def done(_t: asyncio.Future[Any] | None) -> None: + with lock: + if loop_inflight.get(rk) is shared_fut: + del loop_inflight[rk] + + task = _start(run(), shared_fut, done) + background.add(task) + task.add_done_callback(background.discard) + if _core.listeners: + _core.emit("cache.stale", name) + def cache_clear() -> None: - nonlocal hits, misses, inserts, live, gen + nonlocal hits, misses, gen with lock: - store.clear() + if builtin: + main.clear() + instances.clear() inflight.clear() - hits = misses = inserts = live = 0 + hits = misses = 0 gen += 1 + if not builtin: + main.clear() def cache_info() -> CacheInfo: + expired = 0 with lock: - expired = sweep() - info = CacheInfo(hits, misses, maxsize, len(store)) + if per_instance: + buckets = live_buckets() + for b in buckets: + expired += b.sweep() + size = sum(len(b) for b in buckets) + elif builtin: + expired = main.sweep() + size = len(main) + info = (hits, misses) + if not builtin: + size = len(main) if _core.listeners: for _ in range(expired): report_expired() - return info + return CacheInfo(info[0], info[1], maxsize, size) + + def peek(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + b = bucket_of(args, create=False) + if b is None: + return None + k = key_of(args, kwargs) + with guard: + entry = b.get(k) + if entry is None or (entry[1] is not None and now() >= entry[1]): + return None + return entry def cache_delete(*args: Any, **kwargs: Any) -> bool: + b = bucket_of(args, create=False) + if b is None: + return False k = key_of(args, kwargs) - with lock: - return store.pop(k, _MISSING) is not _MISSING + with guard: + return bool(b.delete(k)) def cache_contains(*args: Any, **kwargs: Any) -> bool: - return peek(key_of(args, kwargs)) is not None + return peek(args, kwargs) is not None def cache_get(*args: Any, **kwargs: Any) -> Any: - entry = peek(key_of(args, kwargs)) + entry = peek(args, kwargs) if entry is None: raise KeyError("no cached entry for these arguments") if entry[2]: @@ -606,75 +949,130 @@ def cache_parameters() -> dict[str, Any]: return { "maxsize": maxsize, "ttl": ttl_fn if ttl_fn is not None else ttl_s, + "stale_ttl": stale_s, "typed": typed, "cache_exceptions": exc_types, "exception_ttl": exc_ttl_s, + "store": store, + "per_instance": per_instance, } - if _core.is_async(f): + data: Any = main.data if builtin else None + data_get: Any = data.get if builtin else None + + def fetch_main(k: Hashable) -> Any: + # `fetch` for the default store, inlined for speed. Caller holds the lock. + entry = data_get(k) + if entry is None: + return None + expires = entry[1] + if expires is not None and _core.monotonic() >= expires: + del data[k] + return _EXPIRED + if lru: + data.move_to_end(k) + return entry + + async def amiss(b: Any, k: Hashable, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + # The async miss path: share one in-flight task per key and event loop. + nonlocal hits, misses + rk = (id(b), k) if per_instance else k + with lock: + loop = asyncio.get_running_loop() + loop_inflight = inflight.setdefault(loop, {}) + shared_fut = loop_inflight.get(rk) + if shared_fut is None: + misses += 1 + g = gen + shared = False + shared_fut = loop_inflight[rk] = _shared_future() + else: + hits += 1 + shared = True + if not shared: + + async def run() -> Any: + try: + result = await f(*args, **kwargs) + except exc_types as exc: + if not isinstance(exc, asyncio.CancelledError): + save(b, k, exc, g, True) + raise + save(b, k, result, g) + return result + + def done(_t: asyncio.Future[Any] | None) -> None: + with lock: + if loop_inflight.get(rk) is shared_fut: + del loop_inflight[rk] + + _start(run(), shared_fut, done) + if _core.listeners: + _core.emit("cache.hit" if shared else "cache.miss", name) + # one cancelled caller must not cancel the shared computation + return await _await_shared(shared_fut) + + if _core.is_async(f) and builtin and not per_instance: @functools.wraps(f) async def awrapper(*args: Any, **kwargs: Any) -> Any: - nonlocal hits, misses + nonlocal hits if ov.disable_caching: return await f(*args, **kwargs) k = args if len(args) == fast_n and not kwargs else build_key(args, kwargs) try: with lock: - entry = lookup(k) + entry = fetch_main(k) + if entry: + hits += 1 except TypeError: if k is not args: raise k = build_key(args, kwargs) with lock: - entry = lookup(k) + entry = fetch_main(k) + if entry: + hits += 1 if entry: - with lock: - hits += 1 if _core.listeners: _core.emit("cache.hit", name) + if stale_s is not None and is_stale(entry): + refresh_in_task(main, k, args, kwargs) if entry[2]: raise entry[0] return entry[0] if _core.listeners and entry is not None: report_expired() - with lock: - loop = asyncio.get_running_loop() - loop_inflight = inflight.setdefault(loop, {}) - shared_fut = loop_inflight.get(k) - if shared_fut is None: - misses += 1 - g = gen - shared = False - shared_fut = loop_inflight[k] = _shared_future() - else: - hits += 1 - shared = True - if not shared: + return await amiss(main, k, args, kwargs) - async def run() -> Any: - try: - result = await f(*args, **kwargs) - except exc_types as exc: - if not isinstance(exc, asyncio.CancelledError): - save(k, exc, g, True) - raise - save(k, result, g) - return result - - def done(_t: asyncio.Future[Any] | None, k: Hashable = k) -> None: - with lock: - if loop_inflight.get(k) is shared_fut: - del loop_inflight[k] + wrapper: Any = awrapper + elif _core.is_async(f): - _start(run(), shared_fut, done) - if _core.listeners: - _core.emit("cache.hit" if shared else "cache.miss", name) - # one cancelled caller must not cancel the shared computation - return await _await_shared(shared_fut) + @functools.wraps(f) + async def agwrapper(*args: Any, **kwargs: Any) -> Any: + nonlocal hits + if ov.disable_caching: + return await f(*args, **kwargs) + b = bucket_of(args) + k = key_of(args, kwargs) + with guard: + entry = fetch(b, k) + if entry: + with lock: + hits += 1 + if _core.listeners: + _core.emit("cache.hit", name) + if stale_s is not None and is_stale(entry): + refresh_in_task(b, k, args, kwargs) + if entry[2]: + raise entry[0] + return entry[0] + if _core.listeners and entry is not None: + report_expired() + return await amiss(b, k, args, kwargs) - wrapper: Any = awrapper - else: + wrapper = agwrapper + elif builtin and not per_instance: @functools.wraps(f) def wrapper(*args: Any, **kwargs: Any) -> Any: @@ -684,14 +1082,14 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: k = args if len(args) == fast_n and not kwargs else build_key(args, kwargs) try: with lock: - entry = store_get(k) + entry = data_get(k) if entry is not None and entry[1] is None: # hot path: a live entry that never expires if lru: - store.move_to_end(k) + data.move_to_end(k) hits += 1 else: - entry = lookup(k) + entry = fetch_main(k) if entry: hits += 1 else: @@ -702,7 +1100,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: raise k = build_key(args, kwargs) with lock: - entry = lookup(k) + entry = fetch_main(k) if entry: hits += 1 else: @@ -711,6 +1109,45 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: if entry: if _core.listeners: _core.emit("cache.hit", name) + if stale_s is not None and is_stale(entry): + refresh_in_thread(main, k, args, kwargs) + if entry[2]: + raise entry[0] + return entry[0] + if _core.listeners: + if entry is not None: + report_expired() + _core.emit("cache.miss", name) + try: + result = f(*args, **kwargs) + except exc_types as exc: + save(main, k, exc, g, True) + raise + save(main, k, result, g) + return result + + else: + + @functools.wraps(f) + def wrapper(*args: Any, **kwargs: Any) -> Any: + nonlocal hits, misses + if ov.disable_caching: + return f(*args, **kwargs) + b = bucket_of(args) + k = key_of(args, kwargs) + with guard: + entry = fetch(b, k) + with lock: + if entry: + hits += 1 + else: + misses += 1 + g = gen + if entry: + if _core.listeners: + _core.emit("cache.hit", name) + if stale_s is not None and is_stale(entry): + refresh_in_thread(b, k, args, kwargs) if entry[2]: raise entry[0] return entry[0] @@ -721,9 +1158,9 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: try: result = f(*args, **kwargs) except exc_types as exc: - save(k, exc, g, True) + save(b, k, exc, g, True) raise - save(k, result, g) + save(b, k, result, g) return result wrapper.cache_clear = cache_clear diff --git a/src/python_wrap/circuit.py b/src/python_wrap/circuit.py index 6fa5eb9..b328439 100644 --- a/src/python_wrap/circuit.py +++ b/src/python_wrap/circuit.py @@ -7,7 +7,7 @@ import threading from collections.abc import Callable, Coroutine from dataclasses import dataclass -from typing import Any, Concatenate, ParamSpec, Protocol, TypeVar, cast, overload +from typing import Any, Concatenate, NoReturn, ParamSpec, Protocol, TypeVar, cast, overload from python_wrap import _core from python_wrap._core import F, LoggerLike, P, R @@ -129,6 +129,9 @@ def charge(fail: bool) -> str: ``` """ + def __reduce__(self) -> NoReturn: + _core.refuse_pickle(self) + def __init__( self, *, diff --git a/src/python_wrap/concurrency.py b/src/python_wrap/concurrency.py index 704d2df..2f6eb56 100644 --- a/src/python_wrap/concurrency.py +++ b/src/python_wrap/concurrency.py @@ -36,6 +36,7 @@ Concatenate, Final, Generic, + NoReturn, ParamSpec, Protocol, TypeVar, @@ -283,6 +284,18 @@ def _call_in_worker( return cast(R, job.result) +# Message of the cancellations a timer sends itself. Python 3.10 has no +# ``Task.cancelling()`` counter, so there the message is how a timer tells its own +# cancellation apart from one sent by someone else (best effort: two cancellations that +# arrive before the task resumes are delivered as one, as in the 3.10 stdlib). +_TIMER_CANCEL = "python_wrap.timeout expired" + + +def _is_timer_cancel(exc: BaseException | None) -> bool: + """True if ``exc`` is the CancelledError a timer sent (3.10 path only).""" + return isinstance(exc, asyncio.CancelledError) and exc.args[:1] == (_TIMER_CANCEL,) + + if sys.version_info >= (3, 11): _async_timer = asyncio.timeout @@ -309,7 +322,7 @@ async def __aenter__(self) -> _CancelTimer: def fire() -> None: self._fired = True - task.cancel() + task.cancel(_TIMER_CANCEL) self._handle = asyncio.get_running_loop().call_later(self._seconds, fire) return self @@ -321,12 +334,129 @@ async def __aexit__( tb: TracebackType | None, ) -> None: self._handle.cancel() - if self._fired and exc_type is asyncio.CancelledError: + if self._fired and _is_timer_cancel(exc): raise TimeoutError from exc _async_timer = _CancelTimer +class _GraceTimer: + """Cancel the current task at the deadline, and again after ``grace`` seconds. + + The first cancellation lets the block run its ``finally`` / ``except CancelledError`` + cleanup; if the cleanup is still awaiting when the grace period ends, a second + cancellation interrupts it (with a warning). Only cancellations caused by this timer + are turned into ``TimeoutError``. + """ + + def __init__(self, seconds: float, grace: float, label: str) -> None: + self._seconds = seconds + self._grace = grace + self._label = label + self._fired = False + self._cancels = 0 + self._cancelling = 0 + self._handles: list[asyncio.TimerHandle] = [] + + def expired(self) -> bool: + return self._fired + + async def __aenter__(self) -> _GraceTimer: + task = asyncio.current_task() + if task is None: # pragma: no cover - asyncio always runs coroutines in a task + raise RuntimeError("timeout() must be used inside a task") + loop = asyncio.get_running_loop() + + def second() -> None: + self._cancels += 1 + _core.get_logger(None).warning( + "%s: cleanup still running %ss after the timeout, cancelling it again", + self._label, + self._grace, + ) + task.cancel(_TIMER_CANCEL) + + def fire() -> None: + self._fired = True + self._cancels += 1 + task.cancel(_TIMER_CANCEL) + self._handles.append(loop.call_later(self._grace, second)) + + self._task = task + # Like asyncio.timeout: remember the cancel requests already pending on entry, so + # on exit only the timer's own ones are converted (3.11+). + cancelling = getattr(task, "cancelling", None) + self._cancelling = cancelling() if cancelling is not None else 0 + self._handles.append(loop.call_later(self._seconds, fire)) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + for handle in self._handles: + handle.cancel() + if not self._fired or exc_type is not asyncio.CancelledError: + return + uncancel = getattr(self._task, "uncancel", None) + if uncancel is not None: # 3.11+: withdraw our own requests, then count the rest + remaining: int = getattr(self._task, "cancelling")() # noqa: B009 - 3.11+ only + for _ in range(self._cancels): + if remaining <= self._cancelling: + break + remaining = uncancel() + if remaining <= self._cancelling: + raise TimeoutError from exc + elif _is_timer_cancel(exc): # pragma: no cover - exercised by the 3.10 CI job + raise TimeoutError from exc + + +async def _await_with_grace(awaitable: Awaitable[T], label: str, seconds: float, grace: float) -> T: + """Run ``awaitable`` as a child task; on expiry cancel it and wait ``grace`` for cleanup.""" + task = asyncio.ensure_future(awaitable) + try: + await asyncio.wait({task}, timeout=seconds) + except BaseException: + # Cancelled from outside: cancel the child too, give its cleanup the same grace, + # then re-raise the caller's cancellation. Never leave its outcome unretrieved. + task.cancel() + try: + await asyncio.wait({task}, timeout=grace) + finally: + _consume_when_done(task) + raise + if task.done(): + return task.result() + task.cancel() + try: + await asyncio.wait({task}, timeout=grace) + except BaseException: + _consume_when_done(task) # cancelled from outside during the grace period + raise + if not task.done(): + _core.get_logger(None).warning( + "%s: still running %ss after the timeout; abandoning it", label, grace + ) + _consume_when_done(task) + raise _expired(label, seconds) + + +def _consume_when_done(task: asyncio.Future[Any]) -> None: + """Retrieve ``task``'s outcome now if it is done, otherwise once it is.""" + if task.done(): + _consume_result(task) + else: + task.add_done_callback(_consume_result) + + +def _consume_result(task: asyncio.Future[Any]) -> None: + """Mark an abandoned task's outcome as retrieved so asyncio does not log it.""" + if not task.cancelled(): + task.exception() + + async def _await_with_deadline(awaitable: Awaitable[T], label: str, seconds: float) -> T: """Await under a timer; convert only *our* expiry to TimeoutExpired.""" timer = _async_timer(seconds) @@ -349,11 +479,22 @@ class Timeout: Attributes: seconds: The configured limit. use_signals: The configured sync strategy (`None` means automatic). + cleanup_grace: Seconds async code may spend cleaning up after the limit (`None`: no + separate bound). """ - def __init__(self, seconds: float, *, use_signals: bool | None = None) -> None: + def __init__( + self, + seconds: float, + *, + use_signals: bool | None = None, + cleanup_grace: float | None = None, + ) -> None: if seconds <= 0: raise ValueError(f"timeout seconds must be positive, got {seconds!r}") + if cleanup_grace is not None and cleanup_grace < 0: + raise ValueError(f"cleanup_grace must be >= 0, got {cleanup_grace!r}") + self.cleanup_grace = cleanup_grace if use_signals and not _HAS_SETITIMER: raise RuntimeError("use_signals=True needs signal.setitimer, unavailable here") self.seconds = seconds @@ -377,12 +518,21 @@ def __call__(self, func: F, /) -> F: """Apply the limit to every call of ``func`` (sync or ``async def``).""" label = _core.qualname(func) use_signals = self.use_signals + grace = self.cleanup_grace + + if grace is not None and not _core.is_async(func): + raise TypeError( + f"timeout(cleanup_grace=...) only supports async def functions; {label} is " + "synchronous (signals and worker threads cannot give cleanup a bounded grace)" + ) if _core.is_async(func): @functools.wraps(func) async def async_wrapper(*args: Any, **kwargs: Any) -> Any: seconds = self._limit(label) + if grace is not None: + return await _await_with_grace(func(*args, **kwargs), label, seconds, grace) return await _await_with_deadline(func(*args, **kwargs), label, seconds) return cast(F, async_wrapper) @@ -404,6 +554,8 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: # -- sync context manager ----------------------------------------------------------- def __enter__(self) -> Timeout: + if self.cleanup_grace is not None: + raise TypeError("timeout(cleanup_grace=...) is only supported with `async with`") if self.use_signals is False or not _signals_usable(): raise RuntimeError( "`with timeout(...)` needs SIGALRM (Unix) and the main thread, because a " @@ -430,7 +582,11 @@ async def __aenter__(self) -> Timeout: if task is None: # pragma: no cover - asyncio always runs coroutines in a task raise RuntimeError("`async with timeout(...)` must run inside an asyncio task") seconds = self._limit(_BLOCK) - timer = _async_timer(seconds) + timer: Any = ( + _async_timer(seconds) + if self.cleanup_grace is None + else _GraceTimer(seconds, self.cleanup_grace, _BLOCK) + ) await timer.__aenter__() self._timers.setdefault(task, []).append((timer, seconds)) return self @@ -455,10 +611,15 @@ async def __aexit__( raise _expired(_BLOCK, seconds) from error def __repr__(self) -> str: - return f"Timeout(seconds={self.seconds!r}, use_signals={self.use_signals!r})" + return ( + f"Timeout(seconds={self.seconds!r}, use_signals={self.use_signals!r}, " + f"cleanup_grace={self.cleanup_grace!r})" + ) -def timeout(seconds: float, *, use_signals: bool | None = None) -> Timeout: +def timeout( + seconds: float, *, use_signals: bool | None = None, cleanup_grace: float | None = None +) -> Timeout: """Raise `TimeoutExpired` if a call or block runs longer than ``seconds``. ``TimeoutExpired`` subclasses the built-in ``TimeoutError``. The returned `Timeout` @@ -517,13 +678,28 @@ def timeout(seconds: float, *, use_signals: bool | None = None) -> Timeout: use_signals: ``True`` forces the signal strategy, ``False`` forces the thread fallback (and so rules out ``with timeout(...)``), ``None`` (default) picks signals when they are usable. + cleanup_grace: Async only. Seconds the code may spend in its ``finally`` / + ``except CancelledError`` cleanup after the limit is hit. As a decorator, the + call runs as a child task: at the limit it is cancelled and given + ``cleanup_grace`` more seconds to finish; then ``TimeoutExpired`` is raised + (even if the cleanup finished), and a task still running is abandoned with a + warning logged to the package logger (it keeps running until it ends on its own). + As ``async with timeout(5, cleanup_grace=1)`` the block cannot be moved to + another task, so after the grace period the cleanup is cancelled a second time + (with a warning) rather than abandoned; cleanup that suppresses cancellation + cannot be stopped. Sync functions and ``with`` blocks are not supported and + raise ``TypeError`` (signals and worker threads cannot bound cleanup). + ``None`` (default) keeps the plain behaviour. Real time: not affected by the + test overrides. Returns: A `Timeout`: a decorator that preserves the wrapped function's type, and a sync and async context manager. Raises: - ValueError: If ``seconds`` is not positive. + ValueError: If ``seconds`` is not positive or ``cleanup_grace`` is negative. + TypeError: If ``cleanup_grace`` is set and the target is a sync function or a + plain ``with`` block. RuntimeError: If ``use_signals=True`` but ``signal.setitimer`` does not exist on this platform (at creation time); if ``use_signals=True`` and the call is not made from the main thread (at call time); or on entering ``with`` where @@ -552,7 +728,7 @@ async def main() -> None: asyncio.run(main()) ``` """ - return Timeout(seconds, use_signals=use_signals) + return Timeout(seconds, use_signals=use_signals, cleanup_grace=cleanup_grace) # --------------------------------------------------------------------------------------- @@ -711,6 +887,9 @@ def get_user(name: str) -> str: ``` """ + def __reduce__(self) -> NoReturn: + _core.refuse_pickle(self) + def __init__( self, calls: int | None = None, diff --git a/src/python_wrap/config.py b/src/python_wrap/config.py index 17a2863..b076862 100644 --- a/src/python_wrap/config.py +++ b/src/python_wrap/config.py @@ -31,6 +31,7 @@ "jitter", "max_time", "log_level", + "budget", ) _lock = threading.RLock() @@ -63,8 +64,11 @@ def _validate(key: str, value: Any) -> Any: for name in value: if name not in _RETRY_KEYS: raise _hint(str(name), _RETRY_KEYS, "retry default") - from python_wrap.retry import _validate as validate_retry # avoids an import cycle + # Imported here to avoid an import cycle. + from python_wrap.retry import _check_budget + from python_wrap.retry import _validate as validate_retry + _check_budget(value.get("budget")) validate_retry( value.get("max_attempts", 3), value.get("delay", 1.0), @@ -93,14 +97,16 @@ def configure(**defaults: Any) -> None: - `logger`: logger (or name) used by every decorator that logs when it is not given `logger=`. `None` restores the package logger. - `retry`: dict of defaults for `retry`/`retrying`; keys `max_attempts`, - `delay`, `backoff`, `max_delay`, `jitter`, `max_time`, `log_level`. + `delay`, `backoff`, `max_delay`, `jitter`, `max_time`, `log_level`, + `budget` (a `RetryBudget` shared by every retry applied afterwards). - `redact`: default redaction fragments for `log_args` and `audit` (replaces `DEFAULT_REDACT`); `None` restores it. - `redact_extra`: fragments added to the redaction list (the built-ins stay), for `log_args` and `audit`. Raises: - TypeError: For an unknown key (with a "did you mean" hint) or a wrongly typed value. + TypeError: For an unknown key (with a "did you mean" hint) or a wrongly typed value + (including a `budget` that is not a `RetryBudget`). ValueError: If a retry default is out of range. Example: diff --git a/src/python_wrap/events.py b/src/python_wrap/events.py index 30c8787..4fd7402 100644 --- a/src/python_wrap/events.py +++ b/src/python_wrap/events.py @@ -25,11 +25,18 @@ { "retry.attempt_failed": ( "An attempt failed and another will follow. Fields: attempt (int, 1-based), " - "exception (BaseException), next_delay (float seconds before the next attempt)." + "exception (BaseException, or None for a value rejected by retry_if_result), " + "next_delay (float seconds before the next attempt)." ), "retry.succeeded": "The call finally succeeded. Fields: attempts (int, total made).", + "retry.budget_exhausted": ( + "A shared RetryBudget refused another retry; the call stops (retry.gave_up " + "follows). Fields: attempts (int), exception (BaseException, or None when " + "the last attempt returned a value rejected by retry_if_result)." + ), "retry.gave_up": ( - "Attempts were exhausted. Fields: attempts (int), exception (BaseException)." + "Attempts were exhausted. Fields: attempts (int), exception (BaseException, or " + "None when the last attempt returned a value rejected by retry_if_result)." ), "timeout.expired": "A call exceeded its time limit. Fields: seconds (float, the limit).", "circuit.state_changed": ( @@ -49,9 +56,25 @@ "cache.hit": "A cached value was returned. No fields.", "cache.miss": "No cached value; the function ran. No fields.", "cache.evicted": "An entry left the cache. Fields: reason (str): 'lru' or 'ttl'.", + "cache.stale": ( + "A stale value was served (stale-while-revalidate) and a background refresh " + "started. No fields." + ), + "cache.refresh_failed": ( + "A background refresh failed; the stale value is kept. " + "Fields: exception (BaseException)." + ), "bulkhead.rejected": ( "A call was refused because the bulkhead is full. Fields: active (int), waiting (int)." ), + "periodic.run": ( + "A background run of a periodic function finished. function is its name. " + "Fields: run (int, 1-based), seconds (float, how long it took)." + ), + "periodic.failed": ( + "A background run of a periodic function raised. Fields: run (int), " + "exception (BaseException)." + ), } ) diff --git a/src/python_wrap/logs.py b/src/python_wrap/logs.py index b5d372b..1fe982d 100644 --- a/src/python_wrap/logs.py +++ b/src/python_wrap/logs.py @@ -336,7 +336,9 @@ def log_context(**fields: Any) -> _LogContext: and ``async def`` functions. Blocks nest: inner fields add to, and override, outer ones. The fields live in a `contextvars.ContextVar`, so each thread and asyncio task sees its own; a new thread starts empty unless you run it in a copied context - (``contextvars.copy_context().run(...)``; `run_in_thread` does this for you). + (``contextvars.copy_context().run(...)``; `run_in_thread` does this for you), or + unless ``sys.flags.thread_inherit_context`` is set (the default on free-threaded + builds). `log_args`, `log_return`, `audit`, `trace` and `timing` append `` [key=value ...]`` to their message and attach the dict as ``LogRecord.context``; `AuditRecord.context` @@ -632,6 +634,50 @@ def finish( # ------------------------------------------------------------------------------ trace +def _otel_hooks( + f: Callable[..., Any], span_name: str | None +) -> tuple[Callable[[], Any], Callable[[Any, bool, Any], None]]: + """Import OpenTelemetry now and return ``(open_span, close_span)`` hooks for ``trace``.""" + try: + from opentelemetry import context as otel_context + from opentelemetry import trace as otel_trace + except ImportError as exc: + raise ImportError( + "trace(otel=True) needs the 'opentelemetry-api' package: pip install opentelemetry-api" + ) from exc + tracer = otel_trace.get_tracer("python_wrap") + name = span_name or _core.qualname(f) + module = getattr(f, "__module__", None) + qual = getattr(f, "__qualname__", name) + + def open_span() -> Any: + attributes: dict[str, Any] = {"code.function": qual.rpartition(".")[2]} + if module: + attributes["code.namespace"] = module + for key, value in current_log_context().items(): + attributes[f"python_wrap.context.{key}"] = ( + value if isinstance(value, (str, bool, int, float)) else str(value) + ) + span = tracer.start_span(name, attributes=attributes) + token = otel_context.attach(otel_trace.set_span_in_context(span)) + return span, token + + def close_span(state: Any, ok: bool, value: Any) -> None: + span, token = state + try: + if not ok: + if isinstance(value, Exception): + span.record_exception(value) + span.set_status( + otel_trace.Status(otel_trace.StatusCode.ERROR, type(value).__name__) + ) + finally: + otel_context.detach(token) + span.end() + + return open_span, close_span + + @overload def trace(func: F, /) -> F: ... @overload @@ -642,6 +688,8 @@ def trace( logger: LoggerLike = None, level: int = logging.DEBUG, limit: int | None = None, + otel: bool = False, + span_name: str | None = None, ) -> Callable[[F], F]: ... def trace( func: Callable[..., Any] | None = None, @@ -650,21 +698,37 @@ def trace( logger: LoggerLike = None, level: int = logging.DEBUG, limit: int | None = None, + otel: bool = False, + span_name: str | None = None, ) -> Any: """Log entry, the call stack that led to the call, and exit. Three messages per call: ``Entering f``, the call stack (python_wrap's own frames are left out, outermost first), and ``Exiting f`` (or ``Exiting f (raised ValueError)``). + With ``otel=True`` the call is also wrapped in an OpenTelemetry span (see Args). + Args: func: The function, when used as a bare ``@trace``. logger: A ``logging.Logger``, a logger name, or ``None`` for the package logger. level: Log level. Defaults to ``DEBUG``. limit: Keep only this many innermost frames; ``None`` keeps all. + otel: Open an OpenTelemetry span around every call, using the tracer + ``opentelemetry.trace.get_tracer("python_wrap")``. The span is current for the + whole call (also across ``await``), carries ``code.function``, + ``code.namespace`` and the fields of the active `log_context` as + ``python_wrap.context.`` attributes, and on an exception records it and + sets the error status. ``opentelemetry-api`` is imported only when this is + true; python-wrap itself never depends on it. + span_name: Span name; defaults to the function's qualified name. Returns: The decorated function, with its signature preserved. + Raises: + ImportError: At decoration time, if ``otel=True`` and ``opentelemetry-api`` is not + installed. + Example: >>> @trace(limit=5) ... def handler(): ... @@ -673,10 +737,12 @@ def trace( def decorate(f: Callable[P, R]) -> Callable[P, R]: lg = _core.get_logger(logger) name = _core.qualname(f) + open_span, close_span = _otel_hooks(f, span_name) if otel else (None, None) - def start(args: tuple[Any, ...], kwargs: dict[str, Any]) -> None: + def start(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + span_state = open_span() if open_span is not None else None if not lg.isEnabledFor(level): - return + return span_state _log(lg, level, "Entering %s", name) frame = _external_frame() if frame is not None: @@ -685,12 +751,16 @@ def start(args: tuple[Any, ...], kwargs: dict[str, Any]) -> None: entries = entries[-limit:] if limit > 0 else [] text = "".join(traceback.format_list(entries)).rstrip() _log(lg, level, "Call stack for %s:\n%s", name, text, suffix=False) + return span_state - def finish(state: None, ok: bool, value: Any) -> None: - if not lg.isEnabledFor(level): - return - suffix = "" if ok else f" (raised {type(value).__name__})" - _log(lg, level, "Exiting %s%s", name, suffix) + def finish(state: Any, ok: bool, value: Any) -> None: + try: + if lg.isEnabledFor(level): + suffix = "" if ok else f" (raised {type(value).__name__})" + _log(lg, level, "Exiting %s%s", name, suffix) + finally: + if close_span is not None: + close_span(state, ok, value) return _instrument(f, start, finish) diff --git a/src/python_wrap/periodic.py b/src/python_wrap/periodic.py new file mode 100644 index 0000000..1ea086f --- /dev/null +++ b/src/python_wrap/periodic.py @@ -0,0 +1,491 @@ +"""Run a function every N seconds in the background. + +`periodic` is a tiny background loop, not a scheduler: no cron expressions, no +persistence, no catch-up. The decorated function stays directly callable and gains +``start()``, ``stop()`` and a few read-only attributes. + +Timing rules: + +* The schedule is **real time**. ``fast_mode()`` and ``fake_clock()`` never shorten + it (they only affect backoff waits, see `python_wrap.testing`); a periodic loop that + ran "instantly" would spin. Test with tiny intervals, or call the function directly. +* Intervals are measured **start to start** and do not drift: the n-th tick is due at + ``first + n * interval``. If a run is still going when the next tick is due, the missed + ticks are **skipped** (never run in a burst) and the loop resumes at the next tick. +* ``jitter`` moves each run by a uniformly random offset of up to ``± jitter * interval`` + around its tick. Jitter never accumulates, because ticks are computed without it. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import contextvars +import functools +import inspect +import logging +import math +import random +import threading +import time +from collections.abc import AsyncIterator, Callable, Coroutine, Iterator +from typing import Any, Generic, Literal, ParamSpec, Protocol, TypeVar, overload + +from python_wrap import _core +from python_wrap._core import LoggerLike + +__all__ = ["AsyncPeriodic", "Periodic", "periodic"] + +P = ParamSpec("P") +R = TypeVar("R") + +OnError = Literal["log", "stop", "raise"] +_ON_ERROR = ("log", "stop", "raise") + +# Patched by tests; the schedule is real time on purpose (see the module docstring). +_clock: Callable[[], float] = time.monotonic +_uniform: Callable[[float, float], float] = random.uniform + + +def _next_tick(due: float, now: float, interval: float) -> float: + """The tick after ``due``, skipping every tick that is already in the past.""" + nxt = due + interval + if nxt <= now: + nxt += (math.floor((now - nxt) / interval) + 1) * interval + return nxt + + +def _delay_until(due: float, now: float, interval: float, jitter: float) -> float: + """Seconds to wait until ``due``, moved by the jitter and never negative.""" + offset = _uniform(-jitter, jitter) * interval if jitter else 0.0 + return max(0.0, due + offset - now) + + +class _Base(Generic[P, R]): + """State and policy shared by the sync and async variants.""" + + def __init__( + self, + func: Callable[..., Any], + interval: float, + jitter: float, + initial_delay: float, + max_runs: int | None, + on_error: OnError, + name: str | None, + logger: LoggerLike, + level: int, + ) -> None: + self._func = func + self._interval = interval + self._jitter = jitter + self._initial_delay = initial_delay + self._max_runs = max_runs + self._on_error = on_error + self._name = name or _core.qualname(func) + self._logger = logger + self._level = level + self._runs = 0 + self._last_error: BaseException | None = None + self._pending: BaseException | None = None # on_error="raise", surfaced by stop() + self._lock = threading.Lock() + functools.update_wrapper(self, func) # type: ignore[arg-type] + + def __reduce__(self) -> str: + # Pickle by reference, like a function: the object at ``module.qualname`` is + # this one. The background loop and its counters stay in this process. + return str(self.__dict__["__qualname__"]) # set by update_wrapper + + @property + def runs(self) -> int: + """How many times the function has been run since `start` (failed runs count).""" + return self._runs + + @property + def last_error(self) -> BaseException | None: + """The exception of the most recent failed run, or ``None``.""" + return self._last_error + + def _ran(self, seconds: float) -> None: + _core.emit("periodic.run", self._name, run=self._runs, seconds=seconds) + + def _failed(self, exc: Exception) -> bool: + """Record a failed run. Returns True if the loop should continue.""" + self._last_error = exc + _core.emit("periodic.failed", self._name, run=self._runs, exception=exc) + if self._on_error == "log": + _core.get_logger(self._logger).log( + self._level, "periodic %s failed (run %d)", self._name, self._runs, exc_info=exc + ) + return True + if self._on_error == "stop": + _core.get_logger(self._logger).log( + self._level, + "periodic %s failed (run %d); stopping", + self._name, + self._runs, + exc_info=exc, + ) + else: + self._pending = exc + return False + + def _done(self) -> bool: + return self._max_runs is not None and self._runs >= self._max_runs + + def _surface(self) -> None: + exc, self._pending = self._pending, None + if exc is not None: + raise exc + + def _reset(self) -> None: + if self._pending is not None: + # on_error="raise", but stop() never collected it: do not let a restart lose it. + _core.get_logger(self._logger).error( + "periodic %s: restarted before stop() raised the error that ended the " + "previous loop; dropping it", + self._name, + exc_info=self._pending, + ) + self._runs = 0 + self._last_error = None + self._pending = None + + +class Periodic(_Base[P, R]): + """A function that runs in a background daemon thread; see `periodic`. + + Calling the object calls the wrapped function directly. Attributes: `runs`, + `last_error`, `running`. + """ + + def __init__(self, func: Callable[P, R], *config: Any) -> None: + super().__init__(func, *config) + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R: + return self._func(*args, **kwargs) # type: ignore[no-any-return] + + @property + def running(self) -> bool: + """True while the background thread is alive.""" + thread = self._thread + return thread is not None and thread.is_alive() + + def start(self, *args: P.args, **kwargs: P.kwargs) -> Periodic[P, R]: + """Start the background thread; ``args`` and ``kwargs`` go to every run. + + Returns: + This object, so ``f.start()`` can be chained. + + Raises: + RuntimeError: If it is already running. + """ + with self._lock: + if self.running: + raise RuntimeError(f"periodic {self._name} is already running") + self._reset() + self._stop_event = threading.Event() + ctx = contextvars.copy_context() + thread = threading.Thread( + target=ctx.run, + args=(self._loop, self._stop_event, args, kwargs), + name=f"python-wrap-periodic:{self._name}", + daemon=True, + ) + self._thread = thread + thread.start() + return self + + def stop(self, timeout: float | None = 5.0) -> bool: + """Stop the loop and wait up to ``timeout`` seconds for the thread to end. + + A run that is in progress is allowed to finish (a thread cannot be + interrupted). Calling it again, or on a loop that never started, is harmless. + Called from inside the function itself it only requests the stop. + + Args: + timeout: Seconds to wait for the thread; ``None`` waits forever. + + Returns: + True if the thread has ended, False if it is still finishing a run. + + Raises: + Exception: With ``on_error="raise"``, the exception that ended the loop + (raised once). + """ + self._stop_event.set() + thread = self._thread + if thread is not None and thread is not threading.current_thread(): + thread.join(timeout) + if thread.is_alive(): + _core.get_logger(self._logger).warning( + "periodic %s did not stop within %s seconds", self._name, timeout + ) + return False + self._surface() + return True + + @contextlib.contextmanager + def running_in_background(self, *args: P.args, **kwargs: P.kwargs) -> Iterator[Periodic[P, R]]: + """Start the loop for the duration of a ``with`` block, then `stop` it.""" + self.start(*args, **kwargs) + try: + yield self + finally: + self.stop() + + def _loop( + self, + stop: threading.Event, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> None: + due = _clock() + self._initial_delay + if stop.wait(self._initial_delay): + return + while True: + began = _clock() + self._runs += 1 + try: + self._func(*args, **kwargs) + except Exception as exc: + if not self._failed(exc): + return + else: + self._ran(_clock() - began) + if self._done() or stop.is_set(): + return + now = _clock() + due = _next_tick(due, now, self._interval) + if stop.wait(_delay_until(due, now, self._interval, self._jitter)): + return + + +class AsyncPeriodic(_Base[P, R]): + """An ``async def`` function that runs as a task on the current loop; see `periodic`. + + Calling the object calls the coroutine function directly. Attributes: `runs`, + `last_error`, `running`. + """ + + def __init__(self, func: Callable[P, Coroutine[Any, Any, R]], *config: Any) -> None: + super().__init__(func, *config) + self._task: asyncio.Task[None] | None = None + self._stopping = False + mark = getattr(inspect, "markcoroutinefunction", None) # Python 3.12+ + if mark is not None: + mark(self) + + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> Coroutine[Any, Any, R]: + return self._func(*args, **kwargs) # type: ignore[no-any-return] + + @property + def running(self) -> bool: + """True while the background task has not finished.""" + task = self._task + return task is not None and not task.done() + + def start(self, *args: P.args, **kwargs: P.kwargs) -> AsyncPeriodic[P, R]: + """Start the background task on the running loop; arguments go to every run. + + Returns: + This object, so ``f.start()`` can be chained. + + Raises: + RuntimeError: If called outside a running event loop, or if already running. + """ + try: + loop = asyncio.get_running_loop() + except RuntimeError: + raise RuntimeError( + f"periodic {self._name}: start() must be called from a running event loop " + "(inside an async function). Use a plain def function for a thread-based loop." + ) from None + if self.running: + raise RuntimeError(f"periodic {self._name} is already running") + self._reset() + self._stopping = False + self._task = loop.create_task(self._loop(args, kwargs), name=f"periodic:{self._name}") + return self + + async def stop(self) -> bool: + """Cancel the task (including a run in progress) and wait for it to end. + + Calling it again, or on a loop that never started, is harmless. Called from + inside the function itself it ends the loop as soon as that run returns. If the + task awaiting it is cancelled, the cancellation propagates at once and the loop + task finishes in the background. + + Returns: + True once the task has ended. + + Raises: + Exception: With ``on_error="raise"``, the exception that ended the loop + (raised once). + """ + task = self._task + if task is not None and not task.done(): + if task is asyncio.current_task(): + self._stopping = True + return True + self._stopping = True # also ends a loop whose run swallows the cancel + task.cancel() + try: + # Not ``await task``: a CancelledError from it could not be told apart + # from one aimed at the caller, so the caller's would be swallowed. + await asyncio.wait({task}) + except BaseException: + task.add_done_callback(_retrieve) # the caller was cancelled: re-raise + raise + if not task.cancelled(): + task.result() # an exception the loop itself raised (not a cancel) + self._surface() + return True + + @contextlib.asynccontextmanager + async def running_in_background( + self, *args: P.args, **kwargs: P.kwargs + ) -> AsyncIterator[AsyncPeriodic[P, R]]: + """Start the loop for the duration of an ``async with`` block, then `stop` it.""" + self.start(*args, **kwargs) + try: + yield self + finally: + await self.stop() + + async def _loop(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> None: + due = _clock() + self._initial_delay + await asyncio.sleep(self._initial_delay) + while not self._stopping: + began = _clock() + self._runs += 1 + try: + await self._func(*args, **kwargs) + except Exception as exc: + if not self._failed(exc): + return + else: + self._ran(_clock() - began) + if self._done() or self._stopping: + return + now = _clock() + due = _next_tick(due, now, self._interval) + await asyncio.sleep(_delay_until(due, now, self._interval, self._jitter)) + + +def _retrieve(task: asyncio.Task[None]) -> None: + """Mark a finished loop task's outcome as retrieved so asyncio does not log it.""" + if not task.cancelled(): + task.exception() + + +class _Decorator(Protocol): + @overload + def __call__( # type: ignore[overload-overlap] + self, func: Callable[P, Coroutine[Any, Any, R]], / + ) -> AsyncPeriodic[P, R]: ... + @overload + def __call__(self, func: Callable[P, R], /) -> Periodic[P, R]: ... + + +def periodic( + interval: float, + *, + jitter: float = 0.0, + initial_delay: float = 0.0, + max_runs: int | None = None, + on_error: OnError = "log", + name: str | None = None, + logger: LoggerLike = None, + level: int = logging.ERROR, +) -> _Decorator: + """Run the decorated function every ``interval`` seconds in the background. + + The function stays directly callable. The decorated object adds ``start()``, + ``stop()``, ``running``, ``runs``, ``last_error`` and ``running_in_background()`` + (a context manager that starts and stops cleanly). A plain function runs in a daemon + thread; an ``async def`` function runs as a task on the current event loop + (``start()`` must be called inside it, and ``await f.stop()`` ends it). ``start()`` + arguments are passed to every run, and the caller's ``contextvars`` are copied. + + Runs start every ``interval`` seconds measured start to start, without drift. A run + that outlasts the interval makes the loop skip the missed ticks instead of running + in a burst. The schedule is real time: `fast_mode` and `fake_clock` do not shorten it. + + To use it on a method, decorate the bound method: ``periodic(5)(self.refresh)``. + Decorating a function that takes ``self`` or ``cls`` first is rejected. + + Args: + interval: Seconds between run starts. Must be greater than zero. + jitter: Fraction of ``interval`` (``0`` to less than ``1``) by which each run is + randomly moved earlier or later, e.g. ``0.1`` is +/-10 percent. + initial_delay: Seconds before the first run (no jitter). Default: run at once. + max_runs: Stop by itself after this many runs (failed runs count). Default: forever. + on_error: What an exception in a run does. ``"log"`` logs it and carries on; + ``"stop"`` logs it and ends the loop; ``"raise"`` ends the loop and re-raises + the exception, once, from `stop()` (so it never vanishes in a thread; if + `start()` is called again first, it is logged at ``ERROR`` instead). In every + mode it is kept in ``last_error`` and published as a ``periodic.failed`` event. + name: Name for logs, events and the thread. Default: the function's qualified name. + logger: Logger, logger name or ``None`` for the package logger. + level: Level for failure logs. Default: ``logging.ERROR``. + + Returns: + A decorator producing a `Periodic` (plain function) or `AsyncPeriodic` + (``async def``). + + Raises: + ValueError: For an invalid ``interval``, ``jitter``, ``initial_delay``, + ``max_runs`` or ``on_error``. + TypeError: When decorating something that is not callable, or an unbound method. + + Events: + ``periodic.run`` (fields ``run``, ``seconds``) and ``periodic.failed`` (fields + ``run``, ``exception``). + + Example: + ```python + from python_wrap.periodic import periodic + + @periodic(0.01, max_runs=3) + def tick() -> None: + ... + + with tick.running_in_background(): + pass + ``` + """ + if not isinstance(interval, (int, float)) or isinstance(interval, bool) or not interval > 0: + raise ValueError("interval must be a number greater than 0") + if not isinstance(jitter, (int, float)) or not 0 <= jitter < 1: + raise ValueError("jitter must be at least 0 and less than 1") + if not isinstance(initial_delay, (int, float)) or not initial_delay >= 0: + raise ValueError("initial_delay must be a number >= 0") + if max_runs is not None and (not isinstance(max_runs, int) or max_runs < 1): + raise ValueError("max_runs must be None or an integer >= 1") + if on_error not in _ON_ERROR: + raise ValueError(f"on_error must be one of {', '.join(map(repr, _ON_ERROR))}") + + def decorate(func: Callable[..., Any]) -> Any: + if not callable(func): + raise TypeError("periodic() decorates a function") + _reject_unbound_method(func) + config = (interval, jitter, initial_delay, max_runs, on_error, name, logger, level) + if _core.is_async(func): + return AsyncPeriodic(func, *config) + return Periodic(func, *config) + + return decorate + + +def _reject_unbound_method(func: Callable[..., Any]) -> None: + if not inspect.isfunction(func): + return + params = list(inspect.signature(func).parameters) + if params and params[0] in ("self", "cls"): + raise TypeError( + f"periodic cannot decorate the method {func.__qualname__} in a class body: " + "decorate the bound method instead, e.g. periodic(5)(self.refresh)" + ) diff --git a/src/python_wrap/policy.py b/src/python_wrap/policy.py index d985cbe..0fc1d7b 100644 --- a/src/python_wrap/policy.py +++ b/src/python_wrap/policy.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Callable, Mapping, Sequence -from typing import Any, TypeVar, cast +from typing import Any, NoReturn, TypeVar, cast F = TypeVar("F", bound=Callable[..., Any]) @@ -79,6 +79,13 @@ def fetch() -> str: ... _inner: tuple[_Decorator, ...] _roles: dict[str, tuple[_Decorator, ...]] + def __reduce__(self) -> NoReturn: + raise TypeError( + "cannot pickle a Policy: it holds decorator closures. Define the policy at " + "module level so each process builds it on import; functions decorated " + "with it pickle by reference." + ) + def __init__( self, *, diff --git a/src/python_wrap/retry.py b/src/python_wrap/retry.py index 22e8968..89f36db 100644 --- a/src/python_wrap/retry.py +++ b/src/python_wrap/retry.py @@ -5,6 +5,7 @@ import asyncio import datetime import functools +import inspect import logging import math import random @@ -12,10 +13,10 @@ import threading import time import warnings -from collections.abc import AsyncIterator, Callable, Coroutine, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator from dataclasses import dataclass from types import TracebackType -from typing import Any, Protocol, TypeVar, cast, overload +from typing import Any, NoReturn, Protocol, TypeVar, cast, overload from python_wrap import _core, config from python_wrap._core import F, LoggerLike, P, R @@ -28,6 +29,8 @@ __all__ = [ "Attempt", + "RetryBudget", + "RetryBudgetStats", "RetryState", "RetryStats", "Retrying", @@ -43,22 +46,29 @@ _MISSING: Any = object() _UNSET: Any = config.UNSET # resolved by `_resolve_defaults` +ExceptionPredicate = Callable[[BaseException], bool] | Callable[[BaseException], Awaitable[bool]] +ResultPredicate = Callable[[Any], bool] | Callable[[Any], Awaitable[bool]] +RetryCallback = Callable[["RetryState"], None] | Callable[["RetryState"], Awaitable[None]] + @dataclass(frozen=True) class RetryState: - """Snapshot passed to the ``on_retry`` callback before each wait. + """Snapshot passed to the `on_retry` callback before each wait. Attributes: attempt: Number of the attempt that just failed (1-based). - exception: The exception that attempt raised. + exception: The exception that attempt raised, or `None` when the attempt + returned a value that `retry_if_result` rejected. next_delay: Seconds about to be slept before the next attempt (jitter included). elapsed: Seconds since the first attempt started. + result: The rejected return value when `exception` is `None`, else `None`. """ attempt: int - exception: BaseException + exception: BaseException | None next_delay: float elapsed: float + result: Any = None def _validate( @@ -104,8 +114,11 @@ class RetryStats: calls: Times the wrapped function was called (each call may make several attempts). attempts_total: Individual attempts made across all calls. retries: Times an attempt failed and another one was scheduled. - gave_up: Calls that ended by re-raising a retryable exception (attempts, time - budget or `retry_if` said stop). + gave_up: Calls that ended by re-raising a retryable exception, or by returning + a result `retry_if_result` rejected (attempts, time budget, retry budget or + `retry_if` said stop). + budget_exhausted: Calls among `gave_up` that stopped because the shared + `RetryBudget` refused another retry. total_sleep: Seconds spent waiting between attempts. last_exception: The most recent exception a retried attempt raised, if any. """ @@ -116,6 +129,7 @@ def __init__(self) -> None: self.attempts_total = 0 self.retries = 0 self.gave_up = 0 + self.budget_exhausted = 0 self.total_sleep = 0.0 self.last_exception: BaseException | None = None @@ -127,6 +141,7 @@ def snapshot(self) -> dict[str, Any]: "attempts_total": self.attempts_total, "retries": self.retries, "gave_up": self.gave_up, + "budget_exhausted": self.budget_exhausted, "total_sleep": self.total_sleep, "last_exception": self.last_exception, } @@ -135,6 +150,7 @@ def reset(self) -> None: """Set every counter back to zero.""" with self._lock: self.calls = self.attempts_total = self.retries = self.gave_up = 0 + self.budget_exhausted = 0 self.total_sleep = 0.0 self.last_exception = None @@ -142,7 +158,8 @@ def __repr__(self) -> str: s = self.snapshot() return ( f"RetryStats(calls={s['calls']}, attempts_total={s['attempts_total']}, " - f"retries={s['retries']}, gave_up={s['gave_up']}, total_sleep={s['total_sleep']:g})" + f"retries={s['retries']}, gave_up={s['gave_up']}, " + f"budget_exhausted={s['budget_exhausted']}, total_sleep={s['total_sleep']:g})" ) @@ -171,6 +188,162 @@ def retry_stats(func: Callable[..., Any]) -> RetryStats | None: return stats if isinstance(stats, RetryStats) else None +@dataclass(frozen=True) +class RetryBudgetStats: + """Snapshot of a `RetryBudget`, returned by `RetryBudget.stats`. + + Attributes: + requests: First attempts recorded in the current window. + retries: Retries granted in the current window. + rejected: Retries refused since creation (or the last `reset`). + allowed: Retries the budget would still grant right now. + """ + + requests: int + retries: int + rejected: int + allowed: int + + +_BUDGET_SLOTS = 10 + + +class RetryBudget: + """A retry allowance shared by many retried functions (Finagle/Envoy style). + + Every call's first attempt deposits into the budget and every retry withdraws from + it. A retry is allowed while the retries made in the last `window` seconds stay + within `ratio` times the requests made in that window, plus a reserve of + `min_per_second * window` retries so that low-traffic code can still retry. When a + dependency fails for everyone, retries are capped at about `ratio` of the traffic + instead of multiplying it by `max_attempts`. + + One budget can be passed to any number of `retry` decorators and `retrying` blocks; + it is thread-safe and works across threads and event loops (it never blocks). Time + comes from `python_wrap._core.monotonic`, so `fake_clock` controls it in tests. + + Args: + ratio: Retries allowed per request in the window (`0.1` = 10 %). `>= 0`. + min_per_second: Retries per second always allowed regardless of traffic. `>= 0`. + window: Length of the sliding window in seconds. `> 0`. + + Raises: + ValueError: If an argument is out of range. + + Example: + ```python + from python_wrap.retry import RetryBudget, retry + + budget = RetryBudget(ratio=0.2, min_per_second=1, window=10) + + @retry(max_attempts=3, delay=0, budget=budget) + def fetch() -> str: + return "ok" + + assert fetch() == "ok" + assert budget.stats.requests == 1 + ``` + """ + + def __reduce__(self) -> NoReturn: + _core.refuse_pickle(self) + + def __init__( + self, ratio: float = 0.1, min_per_second: float = 10, window: float = 10.0 + ) -> None: + if not ratio >= 0 or math.isinf(ratio): + raise ValueError(f"ratio must be a finite number >= 0, got {ratio!r}") + if not min_per_second >= 0 or math.isinf(min_per_second): + raise ValueError(f"min_per_second must be a finite number >= 0, got {min_per_second!r}") + if not window > 0 or math.isinf(window): + raise ValueError(f"window must be a finite number > 0, got {window!r}") + self.ratio = float(ratio) + self.min_per_second = float(min_per_second) + self.window = float(window) + self._reserve = self.min_per_second * self.window + self._width = self.window / _BUDGET_SLOTS + self._lock = threading.Lock() + self._epochs = [-1] * _BUDGET_SLOTS + self._requests = [0] * _BUDGET_SLOTS + self._retries = [0] * _BUDGET_SLOTS + self._rejected = 0 + + def _slot(self) -> tuple[int, int]: + """Index and epoch of the current slot; clears it if it held an older epoch.""" + epoch = int(_core.monotonic() // self._width) + index = epoch % _BUDGET_SLOTS + if self._epochs[index] != epoch: + self._epochs[index] = epoch + self._requests[index] = 0 + self._retries[index] = 0 + return index, epoch + + def _totals(self, epoch: int) -> tuple[int, int]: + oldest = epoch - _BUDGET_SLOTS + requests = retries = 0 + for i in range(_BUDGET_SLOTS): + if self._epochs[i] > oldest: + requests += self._requests[i] + retries += self._retries[i] + return requests, retries + + def _available(self, requests: int, retries: int) -> float: + return self.ratio * requests + self._reserve - retries + + def deposit(self) -> None: + """Record one request (a first attempt). `retry` calls this for you.""" + with self._lock: + index, _ = self._slot() + self._requests[index] += 1 + + def try_withdraw(self) -> bool: + """Take one retry from the budget. + + `retry` calls this before each retry; call it yourself only in a hand-written + loop. + + Returns: + `True` if the retry is allowed (and was recorded), `False` if the budget is + exhausted (recorded as rejected). + """ + with self._lock: + index, epoch = self._slot() + if self._available(*self._totals(epoch)) >= 1: + self._retries[index] += 1 + return True + self._rejected += 1 + return False + + @property + def stats(self) -> RetryBudgetStats: + """A consistent `RetryBudgetStats` snapshot of the current window.""" + with self._lock: + _, epoch = self._slot() + requests, retries = self._totals(epoch) + allowed = max(math.floor(self._available(requests, retries)), 0) + return RetryBudgetStats(requests, retries, self._rejected, allowed) + + def reset(self) -> None: + """Forget all recorded requests, retries and rejections.""" + with self._lock: + self._epochs = [-1] * _BUDGET_SLOTS + self._requests = [0] * _BUDGET_SLOTS + self._retries = [0] * _BUDGET_SLOTS + self._rejected = 0 + + def __repr__(self) -> str: + return ( + f"RetryBudget(ratio={self.ratio:g}, min_per_second={self.min_per_second:g}, " + f"window={self.window:g})" + ) + + +async def _maybe_await(value: Any) -> Any: + if inspect.isawaitable(value): + return await value + return value + + def _parse_seconds(value: Any) -> float | None: if value is None or isinstance(value, bool): return None @@ -251,9 +424,15 @@ def _resolve_defaults(**given: Any) -> dict[str, Any]: "jitter": 0.0, "max_time": None, "log_level": logging.WARNING, + "budget": None, } +def _async_hooks(**hooks: Callable[..., Any] | None) -> list[str]: + """Names of the given hooks that are async callables.""" + return [name for name, hook in hooks.items() if hook is not None and _core.is_async(hook)] + + class _Engine: """The retry state machine shared by `retry` and `retrying`.""" @@ -268,15 +447,18 @@ def __init__( jitter: float, max_time: float | None, on: ExceptionTypes, - retry_if: Callable[[BaseException], bool] | None, - on_retry: Callable[[RetryState], None] | None, + retry_if: ExceptionPredicate | None, + retry_if_result: ResultPredicate | None, + on_retry: RetryCallback | None, delay_from: Callable[[BaseException], float | None] | None, + budget: RetryBudget | None, logger: LoggerLike, log_level: int | None, log_exc_info: bool, stats: RetryStats, ) -> None: _validate(max_attempts, delay, backoff, max_delay, jitter, max_time) + _check_budget(budget) self.name = name self.max_attempts = max_attempts self.delay = delay @@ -286,12 +468,19 @@ def __init__( self.max_time = max_time self.on = on self.retry_if = retry_if + self.retry_if_result = retry_if_result self.on_retry = on_retry self.delay_from = delay_from + self.budget = budget self.log = _core.get_logger(logger) self.log_level = log_level self.log_exc_info = log_exc_info self.stats = stats + if budget is not None: # keeps the budget check off the no-budget hot path + self.start_call = self._start_call_with_budget # type: ignore[method-assign] + self.async_hooks = _async_hooks( + retry_if=retry_if, retry_if_result=retry_if_result, on_retry=on_retry + ) def limit(self) -> int | None: """Attempts allowed for this call, honouring the test override at call time.""" @@ -305,6 +494,12 @@ def start_call(self) -> float: self.stats.calls += 1 return _core.monotonic() + def _start_call_with_budget(self) -> float: + with self.stats._lock: + self.stats.calls += 1 + self.budget.deposit() # type: ignore[union-attr] # only installed with a budget + return _core.monotonic() + def start_attempt(self) -> None: with self.stats._lock: self.stats.attempts_total += 1 @@ -316,8 +511,18 @@ def slept(self, seconds: float) -> None: with self.stats._lock: self.stats.total_sleep += seconds - def _wait_for(self, attempt: int, exc: BaseException) -> float: - if self.delay_from is not None: + def check_sync(self, *hooks: str) -> None: + """Refuse async hooks where nothing can await them.""" + bad = [h for h in hooks if h in self.async_hooks] + if bad: + raise TypeError( + f"{self.name}: {', '.join(f'{h}=' for h in bad)} is an async function, which " + "only an async retry can await: decorate an `async def` function, or use " + "`async for attempt in retrying(...)` with `async with attempt:`" + ) + + def _wait_for(self, attempt: int, exc: BaseException | None) -> float: + if self.delay_from is not None and exc is not None: try: hint = _parse_hint(self.delay_from(exc)) except Exception: @@ -327,46 +532,134 @@ def _wait_for(self, attempt: int, exc: BaseException) -> float: return hint if self.max_delay is None else min(hint, self.max_delay) return _compute_delay(attempt, self.delay, self.backoff, self.max_delay, self.jitter) - def failure(self, attempt: int, exc: BaseException, started: float) -> float | None: - """Decide after a failed attempt: the wait in seconds, or None to re-raise.""" - if isinstance(exc, _NEVER_RETRY): - return None - with self.stats._lock: - self.stats.last_exception = exc + def _plan( + self, attempt: int, exc: BaseException | None, wanted: bool, started: float + ) -> float | None: + """The wait before the next attempt, or None to stop (counted and emitted).""" wait: float | None = None + denied = False limit = self.limit() - if (self.retry_if is None or self.retry_if(exc)) and (limit is None or attempt < limit): + if wanted and (limit is None or attempt < limit): wait = self._wait_for(attempt, exc) if self.max_time is not None and _core.monotonic() - started + wait > self.max_time: wait = None left = _deadline_remaining() if wait is not None and left is not None and wait > left: wait = None # never sleep past the surrounding `deadline` + if wait is not None and self.budget is not None and not self.budget.try_withdraw(): + wait = None + denied = True if wait is None: with self.stats._lock: self.stats.gave_up += 1 + if denied: + self.stats.budget_exhausted += 1 + if denied: + _core.emit("retry.budget_exhausted", self.name, attempts=attempt, exception=exc) _core.emit("retry.gave_up", self.name, attempts=attempt, exception=exc) return None with self.stats._lock: self.stats.retries += 1 - if self.on_retry is not None: - self.on_retry(RetryState(attempt, exc, wait, _core.monotonic() - started)) + return wait + + def _state( + self, attempt: int, exc: BaseException | None, result: Any, wait: float, started: float + ) -> RetryState: + return RetryState(attempt, exc, wait, _core.monotonic() - started, result) + + def _announce(self, attempt: int, exc: BaseException | None, result: Any, wait: float) -> None: if self.log_level is not None: + limit = self.limit() + what = repr(exc) if exc is not None else f"returned {result!r}" self.log.log( self.log_level, - "%s failed (attempt %d/%s): %r; retrying in %.3fs", + "%s failed (attempt %d/%s): %s; retrying in %.3fs", self.name, attempt, "inf" if limit is None else limit, - exc, + what, wait, - exc_info=(type(exc), exc, exc.__traceback__) if self.log_exc_info else None, + exc_info=( + (type(exc), exc, exc.__traceback__) + if self.log_exc_info and exc is not None + else None + ), ) _core.emit( "retry.attempt_failed", self.name, attempt=attempt, exception=exc, next_delay=wait ) + + def _record(self, exc: BaseException) -> None: + with self.stats._lock: + self.stats.last_exception = exc + + # -- sync decisions ------------------------------------------------------------------- + def failure(self, attempt: int, exc: BaseException, started: float) -> float | None: + """Decide after a failed attempt: the wait in seconds, or None to re-raise.""" + if isinstance(exc, _NEVER_RETRY): + return None + self.check_sync("retry_if", "on_retry") + self._record(exc) + wanted = self.retry_if is None or bool(self.retry_if(exc)) + wait = self._plan(attempt, exc, wanted, started) + if wait is not None: + if self.on_retry is not None: + self.on_retry(self._state(attempt, exc, None, wait, started)) + self._announce(attempt, exc, None, wait) + return wait + + def rejects(self, result: Any) -> bool: + """True if `retry_if_result` asks for another attempt (sync callers).""" + if self.retry_if_result is None: + return False + self.check_sync("retry_if_result") + return bool(self.retry_if_result(result)) + + def result_failure(self, attempt: int, result: Any, started: float) -> float | None: + """Decide after a rejected result: the wait, or None to return it as is.""" + self.check_sync("on_retry") + wait = self._plan(attempt, None, True, started) + if wait is not None: + if self.on_retry is not None: + self.on_retry(self._state(attempt, None, result, wait, started)) + self._announce(attempt, None, result, wait) + return wait + + # -- async decisions ------------------------------------------------------------------ + async def afailure(self, attempt: int, exc: BaseException, started: float) -> float | None: + """`failure` for async callers: async hooks are awaited.""" + if isinstance(exc, _NEVER_RETRY): + return None + self._record(exc) + wanted = self.retry_if is None or bool(await _maybe_await(self.retry_if(exc))) + wait = self._plan(attempt, exc, wanted, started) + if wait is not None: + if self.on_retry is not None: + await _maybe_await(self.on_retry(self._state(attempt, exc, None, wait, started))) + self._announce(attempt, exc, None, wait) return wait + async def arejects(self, result: Any) -> bool: + """`rejects` for async callers.""" + if self.retry_if_result is None: + return False + return bool(await _maybe_await(self.retry_if_result(result))) + + async def aresult_failure(self, attempt: int, result: Any, started: float) -> float | None: + """`result_failure` for async callers.""" + wait = self._plan(attempt, None, True, started) + if wait is not None: + if self.on_retry is not None: + state = self._state(attempt, None, result, wait, started) + await _maybe_await(self.on_retry(state)) + self._announce(attempt, None, result, wait) + return wait + + +def _check_budget(budget: Any) -> None: + if budget is not None and not isinstance(budget, RetryBudget): + raise TypeError(f"budget must be a RetryBudget or None, got {budget!r}") + @overload def retry( @@ -380,9 +673,11 @@ def retry( jitter: float = ..., max_time: float | None = ..., on: ExceptionTypes = ..., - retry_if: Callable[[BaseException], bool] | None = ..., - on_retry: Callable[[RetryState], None] | None = ..., + retry_if: ExceptionPredicate | None = ..., + retry_if_result: ResultPredicate | None = ..., + on_retry: RetryCallback | None = ..., delay_from: Callable[[BaseException], float | None] | None = ..., + budget: RetryBudget | None = ..., logger: LoggerLike = ..., log_level: int | None = ..., log_exc_info: bool = ..., @@ -399,9 +694,11 @@ def retry( jitter: float = ..., max_time: float | None = ..., on: ExceptionTypes = ..., - retry_if: Callable[[BaseException], bool] | None = ..., - on_retry: Callable[[RetryState], None] | None = ..., + retry_if: ExceptionPredicate | None = ..., + retry_if_result: ResultPredicate | None = ..., + on_retry: RetryCallback | None = ..., delay_from: Callable[[BaseException], float | None] | None = ..., + budget: RetryBudget | None = ..., logger: LoggerLike = ..., log_level: int | None = ..., log_exc_info: bool = ..., @@ -417,14 +714,16 @@ def retry( jitter: float = _UNSET, max_time: float | None = _UNSET, on: ExceptionTypes = Exception, - retry_if: Callable[[BaseException], bool] | None = None, - on_retry: Callable[[RetryState], None] | None = None, + retry_if: ExceptionPredicate | None = None, + retry_if_result: ResultPredicate | None = None, + on_retry: RetryCallback | None = None, delay_from: Callable[[BaseException], float | None] | None = None, + budget: RetryBudget | None = _UNSET, logger: LoggerLike = None, log_level: int | None = _UNSET, log_exc_info: bool = False, ) -> Any: - """Retry a function when it raises, optionally with exponential backoff. + """Retry a function when it raises (or returns a bad value), with optional backoff. Works bare (`@retry`), with options (`@retry(max_attempts=5)`) and as a plain call (`retry(func, max_attempts=5)`). Async functions are awaited and slept @@ -433,7 +732,15 @@ def retry( The wait before attempt `n + 1` is `min(delay * backoff ** (n - 1), max_delay) + random.uniform(0, jitter)`. The last exception is re-raised unchanged, with its original traceback, once - attempts or the time budget are exhausted. + attempts, the time budget or the retry budget are exhausted. + + With `retry_if_result`, a returned value for which the predicate is true is treated + as a failure and retried. When no retry is left, that last value is **returned** + (no exception is raised), so check it if it matters. + + For `async def` functions, `retry_if`, `retry_if_result` and `on_retry` may be + `async def` callables; they are awaited. A sync function given an async callable + raises `TypeError` when decorated. `asyncio.CancelledError`, `KeyboardInterrupt`, `SystemExit` and `GeneratorExit` are never retried, even with `on=BaseException` or when listed in `on`. Cancelling an @@ -445,8 +752,9 @@ def retry( wait would outlast the deadline the last exception is re-raised instead. Statistics are available through `retry_stats`. The events `retry.attempt_failed`, - `retry.succeeded` and `retry.gave_up` are emitted for `python_wrap.events`. Under - `python_wrap.testing`, an `overrides.max_attempts` cap lowers `max_attempts`. + `retry.succeeded`, `retry.gave_up` and `retry.budget_exhausted` are emitted for + `python_wrap.events`. Under `python_wrap.testing`, an `overrides.max_attempts` cap + lowers `max_attempts`. Args: func: The function to wrap (when used without parentheses). @@ -459,11 +767,17 @@ def retry( max_time: Total time budget in seconds, measured from the first attempt. If the next wait would run past it, the last exception is re-raised instead. on: Exception type(s) that trigger a retry; anything else propagates at once. - retry_if: Extra predicate; the exception is retried only if it returns True. - on_retry: Called with a `RetryState` before each wait. + retry_if: Extra predicate on the exception; it is retried only if this returns + True. May be async for async functions. + retry_if_result: Predicate on the return value; True means "retry". When no + retry is left the last value is returned. May be async for async functions. + on_retry: Called with a `RetryState` before each wait. May be async for async + functions. delay_from: Called with the exception; a number is the exact wait for this attempt (no backoff, no jitter; still capped by `max_delay` and `max_time`), `None` means use the normal backoff. See `retry_after_header`. + budget: A `RetryBudget` shared with other retried code. When it refuses a retry + the last exception is re-raised (or the last result returned) at once. logger: Logger (or logger name) for the per-retry message. log_level: Level of the per-retry message, or `None` to not log. log_exc_info: Attach the traceback to the per-retry log record. @@ -473,6 +787,8 @@ def retry( Raises: ValueError: If an option is out of range (raised when decorating). + TypeError: If `budget` is not a `RetryBudget`, or a sync function is given an + async `retry_if`, `retry_if_result` or `on_retry` (raised when decorating). Example: >>> @retry(max_attempts=5, delay=0.5, backoff=2, on=ConnectionError) @@ -486,6 +802,7 @@ def retry( jitter=jitter, max_time=max_time, log_level=log_level, + budget=budget, ) max_attempts, delay, backoff = ds["max_attempts"], ds["delay"], ds["backoff"] max_delay, jitter, max_time, log_level = ( @@ -494,8 +811,10 @@ def retry( ds["max_time"], ds["log_level"], ) + budget = ds["budget"] # Validate eagerly so a bad option fails at decoration time, even with no function yet. _validate(max_attempts, delay, backoff, max_delay, jitter, max_time) + _check_budget(budget) def decorate(f: Callable[P, R]) -> Callable[P, R]: stats = RetryStats() @@ -509,13 +828,16 @@ def decorate(f: Callable[P, R]) -> Callable[P, R]: max_time=max_time, on=on, retry_if=retry_if, + retry_if_result=retry_if_result, on_retry=on_retry, delay_from=delay_from, + budget=budget, logger=logger, log_level=log_level, log_exc_info=log_exc_info, stats=stats, ) + check_result = retry_if_result is not None if _core.is_async(f): af = cast("Callable[..., Coroutine[Any, Any, Any]]", f) @@ -529,12 +851,16 @@ async def async_wrapper(*args: Any, **kwargs: Any) -> Any: try: result = await af(*args, **kwargs) except on as exc: - wait = engine.failure(attempt, exc, started) + wait = await engine.afailure(attempt, exc, started) if wait is None: raise else: - engine.success(attempt) - return result + if not check_result or not await engine.arejects(result): + engine.success(attempt) + return result + wait = await engine.aresult_failure(attempt, result, started) + if wait is None: + return result await _core.async_sleep(wait) engine.slept(wait) attempt += 1 @@ -542,6 +868,8 @@ async def async_wrapper(*args: Any, **kwargs: Any) -> Any: setattr(async_wrapper, _STATS_ATTR, stats) return cast("Callable[P, R]", async_wrapper) + engine.check_sync(*engine.async_hooks) + @functools.wraps(f) def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: started = engine.start_call() @@ -555,8 +883,12 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: if wait is None: raise else: - engine.success(attempt) - return result + if not check_result or not engine.rejects(result): + engine.success(attempt) + return result + wait = engine.result_failure(attempt, result, started) + if wait is None: + return result _core.sleep(wait) engine.slept(wait) attempt += 1 @@ -578,7 +910,13 @@ class Attempt: Exceptions that `retry` would retry are swallowed by `with attempt:` (the loop then waits and continues); other exceptions, and the failure of the final attempt, - propagate. + propagate. In an `async for` loop, use `async with attempt:` when `retry_if`, + `retry_if_result` or `on_retry` are async callables (plain `with` works otherwise). + + With `retry_if_result`, store the value the block produced in `attempt.result`; the + predicate is checked when the block ends and a rejected value starts another + attempt. When no retry is left the loop simply ends, leaving the last value in + your variable (and in `attempt.result`). Attributes: number: 1-based number of this attempt. @@ -588,6 +926,17 @@ def __init__(self, number: int, run: _Run) -> None: self.number = number self._run = run self._wait: float | None = None + self._gave_up = False + self._result: Any = _MISSING + + @property + def result(self) -> Any: + """The value checked by `retry_if_result` (`None` until you set it).""" + return None if self._result is _MISSING else self._result + + @result.setter + def result(self, value: Any) -> None: + self._result = value def __enter__(self) -> Attempt: self._run.engine.start_attempt() @@ -600,11 +949,38 @@ def __exit__( tb: TracebackType | None, ) -> bool: engine = self._run.engine - if exc is None or not isinstance(exc, engine.on): + if exc is None: + if self._result is not _MISSING and engine.rejects(self._result): + self._wait = engine.result_failure(self.number, self._result, self._run.started) + self._gave_up = self._wait is None + return False + if not isinstance(exc, engine.on): return False self._wait = engine.failure(self.number, exc, self._run.started) return self._wait is not None + async def __aenter__(self) -> Attempt: + self._run.engine.start_attempt() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> bool: + engine = self._run.engine + if exc is None: + if self._result is not _MISSING and await engine.arejects(self._result): + started = self._run.started + self._wait = await engine.aresult_failure(self.number, self._result, started) + self._gave_up = self._wait is None + return False + if not isinstance(exc, engine.on): + return False + self._wait = await engine.afailure(self.number, exc, self._run.started) + return self._wait is not None + class Retrying: """Iterable of `Attempt` objects returned by `retrying`. @@ -629,7 +1005,8 @@ def __iter__(self) -> Iterator[Attempt]: yield attempt wait = attempt._wait if wait is None: - run.engine.success(number) + if not attempt._gave_up: + run.engine.success(number) return _core.sleep(wait) run.engine.slept(wait) @@ -643,7 +1020,8 @@ async def __aiter__(self) -> AsyncIterator[Attempt]: yield attempt wait = attempt._wait if wait is None: - run.engine.success(number) + if not attempt._gave_up: + run.engine.success(number) return await _core.async_sleep(wait) run.engine.slept(wait) @@ -659,9 +1037,11 @@ def retrying( jitter: float = _UNSET, max_time: float | None = _UNSET, on: ExceptionTypes = Exception, - retry_if: Callable[[BaseException], bool] | None = None, - on_retry: Callable[[RetryState], None] | None = None, + retry_if: ExceptionPredicate | None = None, + retry_if_result: ResultPredicate | None = None, + on_retry: RetryCallback | None = None, delay_from: Callable[[BaseException], float | None] | None = None, + budget: RetryBudget | None = _UNSET, logger: LoggerLike = None, log_level: int | None = _UNSET, log_exc_info: bool = False, @@ -670,12 +1050,18 @@ def retrying( """Retry an inline block of code instead of a whole function. Takes the same options as `retry` and behaves identically (backoff, jitter, - `max_time`, `delay_from`, cancellation rules, events, test overrides). Put the - risky code in `with attempt:`: a retryable exception is swallowed and the loop + `max_time`, `delay_from`, `budget`, cancellation rules, events, test overrides). Put + the risky code in `with attempt:`: a retryable exception is swallowed and the loop waits and continues; success ends the loop; the final failure propagates. Use `for` in sync code and `async for` in async code. Do not use `for` inside a - coroutine, it would block the event loop while waiting. + coroutine, it would block the event loop while waiting. Async `retry_if`, + `retry_if_result` or `on_retry` callables need `async for` and `async with attempt:`; + with plain `with attempt:` they raise `TypeError`. + + For `retry_if_result`, assign the block's value to `attempt.result`; a rejected + value is retried, and when no retry is left the loop ends normally with the last + value (like `retry`, nothing is raised). Args: max_attempts: Total attempts allowed, or `None` for no limit. @@ -685,9 +1071,11 @@ def retrying( jitter: Up to this many random extra seconds are added to each wait. max_time: Total time budget in seconds from the first attempt. on: Exception type(s) that trigger a retry. - retry_if: Extra predicate; retry only if it returns True. + retry_if: Extra predicate on the exception; retry only if it returns True. + retry_if_result: Predicate on `attempt.result`; True means "retry". on_retry: Called with a `RetryState` before each wait. delay_from: Exact wait derived from the exception, or `None` for backoff. + budget: A shared `RetryBudget`; when it refuses, the loop stops retrying. logger: Logger (or logger name) for the per-retry message. log_level: Level of the per-retry message, or `None` to not log. log_exc_info: Attach the traceback to the per-retry log record. @@ -698,6 +1086,7 @@ def retrying( Raises: ValueError: If an option is out of range. + TypeError: If `budget` is not a `RetryBudget`. Example: >>> for attempt in retrying(max_attempts=3, delay=0.1, on=ConnectionError): @@ -712,6 +1101,7 @@ def retrying( jitter=jitter, max_time=max_time, log_level=log_level, + budget=budget, ) max_attempts, delay, backoff = ds["max_attempts"], ds["delay"], ds["backoff"] max_delay, jitter, max_time, log_level = ( @@ -731,8 +1121,10 @@ def retrying( max_time=max_time, on=on, retry_if=retry_if, + retry_if_result=retry_if_result, on_retry=on_retry, delay_from=delay_from, + budget=ds["budget"], logger=logger, log_level=log_level, log_exc_info=log_exc_info, diff --git a/src/python_wrap/state.py b/src/python_wrap/state.py index 29bdc10..d0b36c3 100644 --- a/src/python_wrap/state.py +++ b/src/python_wrap/state.py @@ -10,7 +10,7 @@ import threading from collections.abc import Callable, Iterable from types import TracebackType -from typing import Any, Final, ParamSpec, TypeVar, cast, overload +from typing import Any, Final, NoReturn, ParamSpec, TypeVar, cast, overload from python_wrap import _core from python_wrap._core import F, LoggerLike @@ -274,6 +274,9 @@ class Transaction: ... tx.on_rollback(print, "undo") """ + def __reduce__(self) -> NoReturn: + _core.refuse_pickle(self, "Open the transaction inside the process that runs the work.") + def __init__(self, logger: LoggerLike = None) -> None: self._logger = _core.get_logger(logger) self._commit: list[tuple[Callable[..., Any], tuple[Any, ...], dict[str, Any]]] = [] diff --git a/src/python_wrap/testing.py b/src/python_wrap/testing.py index a0615ab..67c25c3 100644 --- a/src/python_wrap/testing.py +++ b/src/python_wrap/testing.py @@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Iterator from contextlib import contextmanager from dataclasses import dataclass, field -from typing import Any, Generic, ParamSpec, TypeVar, overload +from typing import Any, Generic, NoReturn, ParamSpec, TypeVar, overload from python_wrap import _core @@ -264,6 +264,9 @@ class FakeClock: default_factory=threading.Lock, init=False, repr=False, compare=False ) + def __reduce__(self) -> NoReturn: + _core.refuse_pickle(self, "Install a fake clock in each process that needs one.") + def __post_init__(self) -> None: self.now = self.start diff --git a/tests/_pickle_helpers.py b/tests/_pickle_helpers.py new file mode 100644 index 0000000..2128a14 --- /dev/null +++ b/tests/_pickle_helpers.py @@ -0,0 +1,298 @@ +"""Module-level decorated functions for ``test_pickling.py``. + +They live in their own importable module because pickle stores functions by reference +(``module.qualname``): a process started with the "spawn" method re-imports this module +by name to find them. +""" + +from __future__ import annotations + +import asyncio +import warnings +from typing import Any + +import python_wrap as pw + +breaker = pw.CircuitBreaker(failure_threshold=3, name="pickle-helpers") +bulkhead = pw.Bulkhead(4) +shared_policy = pw.policy(retry=pw.retry(max_attempts=2, delay=0), timeout=pw.timeout(5)) + + +@pw.retry(max_attempts=2, delay=0) +def f_retry(x: int) -> int: + return x + 1 + + +@pw.fallback(-1) +def f_fallback(x: int) -> int: + return x + 1 + + +@pw.suppress(ValueError) +def f_suppress(x: int) -> int: + return x + 1 + + +@pw.circuit_breaker(failure_threshold=3) +def f_circuit_breaker(x: int) -> int: + return x + 1 + + +@breaker +def f_breaker_object(x: int) -> int: + return x + 1 + + +@pw.timeout(5) +def f_timeout(x: int) -> int: + return x + 1 + + +@pw.memoize +def f_memoize(x: int) -> int: + return x + 1 + + +@pw.memoize(maxsize=8, ttl=60) +def f_memoize_options(x: int) -> int: + return x + 1 + + +@pw.cache(60) +def f_cache(x: int) -> int: + return x + 1 + + +@pw.once +def f_once(x: int) -> int: + return x + 1 + + +@pw.rate_limit(100, 1) +def f_rate_limit(x: int) -> int: + return x + 1 + + +@pw.throttle(0.001) +def f_throttle(x: int) -> int: + return x + 1 + + +@pw.debounce(0.01) +def f_debounce(x: int) -> int: + return x + 1 + + +@pw.singleflight +def f_singleflight(x: int) -> int: + return x + 1 + + +@pw.run_in_thread +def f_run_in_thread(x: int) -> int: + return x + 1 + + +@pw.limit_concurrency(4) +def f_limit_concurrency(x: int) -> int: + return x + 1 + + +@bulkhead +def f_bulkhead(x: int) -> int: + return x + 1 + + +@pw.batch_calls(max_size=10, max_wait=0.001) +def f_batch_calls(xs: list[int]) -> list[int]: + return [x + 1 for x in xs] + + +@pw.hedge(5.0) +def f_hedge(x: int) -> int: + return x + 1 + + +@pw.periodic(60) +def f_periodic(x: int) -> int: + return x + 1 + + +@pw.log_args +def f_log_args(x: int) -> int: + return x + 1 + + +@pw.log_return +def f_log_return(x: int) -> int: + return x + 1 + + +@pw.audit +def f_audit(x: int) -> int: + return x + 1 + + +@pw.trace +def f_trace(x: int) -> int: + return x + 1 + + +@pw.timing +def f_timing(x: int) -> int: + return x + 1 + + +@pw.measure_memory +def f_measure_memory(x: int) -> int: + return x + 1 + + +@pw.profile +def f_profile(x: int) -> int: + return x + 1 + + +@pw.validate_args(x=lambda v: v >= 0) +def f_validate_args(x: int) -> int: + return x + 1 + + +@pw.check_type(x=int) +def f_check_type(x: int) -> int: + return x + 1 + + +@pw.enforce_types +def f_enforce_types(x: int) -> int: + return x + 1 + + +@pw.revert_on_failure +def f_revert_on_failure(x: int) -> int: + return x + 1 + + +@pw.transactional +def f_transactional(x: int) -> int: + return x + 1 + + +@pw.deprecate +def f_deprecate(x: int) -> int: + return x + 1 + + +@pw.mock_data(None, enabled=False) +def f_mock_data(x: int) -> int: + return x + 1 + + +@pw.compose(pw.timing, pw.retry(max_attempts=2, delay=0)) +def f_compose(x: int) -> int: + return x + 1 + + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) # no_debug warns when applied + + @pw.no_debug + def f_no_debug(x: int) -> int: + return x + 1 + + +@pw.count_calls +def f_count_calls(x: int) -> int: + return x + 1 + + +@shared_policy +def f_policy(x: int) -> int: + return x + 1 + + +@pw.memoize +@pw.retry(max_attempts=2, delay=0) +@pw.count_calls +def f_stacked(x: int) -> int: + return x + 1 + + +@pw.retry(max_attempts=2, delay=0) +async def af_retry(x: int) -> int: + return x + 1 + + +@pw.memoize +async def af_memoize(x: int) -> int: + return x + 1 + + +@pw.periodic(60) +async def af_periodic(x: int) -> int: + return x + 1 + + +@pw.debounce(0.01) +async def af_debounce(x: int) -> int: + return x + 1 + + +@pw.batch_calls(max_size=10, max_wait=0.001) +async def af_batch_calls(xs: list[int]) -> list[int]: + return [x + 1 for x in xs] + + +@pw.wrap_methods(pw.retry(max_attempts=2, delay=0), pw.count_calls) +class Service: + """A class whose methods were decorated by `wrap_methods`.""" + + def __init__(self, base: int = 0) -> None: + self.base = base + + def add(self, x: int) -> int: + return self.base + x + + +class Model: + """An instance that uses `cached_property`, `memoize` methods and decorated methods.""" + + def __init__(self, n: int) -> None: + self.n = n + self.computed = 0 + + @pw.cached_property + def square(self) -> int: + self.computed += 1 + return self.n * self.n + + @pw.memoize + def times(self, k: int) -> int: + return self.n * k + + @pw.memoize(per_instance=True) + def plus(self, k: int) -> int: + return self.n + k + + @pw.retry(max_attempts=2, delay=0) + def retried(self, k: int) -> int: + return self.n - k + + @pw.count_calls + def counted(self) -> int: + return self.n + + +def call_in_child(func: Any, *args: Any) -> Any: + """Run in a worker process: call a function that arrived there by pickle.""" + return func(*args) + + +def run_async(func: Any, *args: Any) -> Any: + """Run in a worker process: await an ``async def`` function that arrived by pickle.""" + return asyncio.run(func(*args)) + + +def count_calls_seen() -> int: + """Run in a worker process: how many calls this process's counter has seen.""" + return f_count_calls.stats.calls diff --git a/tests/test_caching.py b/tests/test_caching.py index 3daf4a7..7d85478 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -481,13 +481,10 @@ def f(x: int) -> int: for i in range(1000): f(i) fake_clock.advance(2) - sweep = next( + store = next( c.cell_contents for c in f.cache_info.__closure__ # type: ignore[attr-defined] - if getattr(c.cell_contents, "__name__", "") == "sweep" - ) - store = next( - c.cell_contents for c in sweep.__closure__ if hasattr(c.cell_contents, "move_to_end") + if type(c.cell_contents).__name__ == "_LocalStore" ) assert len(store) <= 130 @@ -722,6 +719,9 @@ def f(x: int) -> int: "typed": True, "cache_exceptions": (), "exception_ttl": 2.0, + "stale_ttl": None, + "store": None, + "per_instance": False, } diff --git a/tests/test_caching_extras.py b/tests/test_caching_extras.py new file mode 100644 index 0000000..36193b1 --- /dev/null +++ b/tests/test_caching_extras.py @@ -0,0 +1,657 @@ +"""memoize extras: stale-while-revalidate, pluggable stores and per-instance caches.""" + +from __future__ import annotations + +import asyncio +import contextvars +import dataclasses +import datetime +import gc +import logging +import pickle +import threading +import weakref +from collections.abc import Hashable, Iterator +from typing import Any + +import pytest + +from conftest import FakeClock +from python_wrap import _core +from python_wrap.caching import CacheInfo, CacheStore, memoize +from python_wrap.testing import disable_caching + + +@pytest.fixture +def events() -> Iterator[list[_core.Event]]: + seen: list[_core.Event] = [] + _core.listeners.append(seen.append) + try: + yield seen + finally: + _core.listeners.remove(seen.append) + + +def names(events: list[_core.Event], prefix: str = "cache.") -> list[str]: + return [e.name for e in events if e.name.startswith(prefix)] + + +def join_refreshes() -> None: + for t in threading.enumerate(): + if t.name.startswith("python_wrap-refresh-"): + t.join(2) + + +async def settle() -> None: + for _ in range(5): + await asyncio.sleep(0) + + +class DictStore: + def __init__(self) -> None: + self.data: dict[Hashable, Any] = {} + self.ttls: list[float | None] = [] + + def get(self, key: Hashable) -> Any: + return self.data.get(key) + + def set(self, key: Hashable, entry: Any, ttl: float | None) -> None: + self.ttls.append(ttl) + self.data[key] = entry + + def delete(self, key: Hashable) -> bool: + return self.data.pop(key, None) is not None + + def clear(self) -> None: + self.data.clear() + + def __len__(self) -> int: + return len(self.data) + + +class PickleStore(DictStore): + """Serialises keys and entries, as a Redis adapter would.""" + + def get(self, key: Hashable) -> Any: + raw = self.data.get(pickle.dumps(key)) + return None if raw is None else pickle.loads(raw) + + def set(self, key: Hashable, entry: Any, ttl: float | None) -> None: + self.data[pickle.dumps(key)] = pickle.dumps(entry) + + def delete(self, key: Hashable) -> bool: + return self.data.pop(pickle.dumps(key), None) is not None + + +# ------------------------------------------------------------ stale-while-revalidate + + +def test_stale_value_is_served_and_refreshed_in_background( + fake_clock: FakeClock, events: list[_core.Event] +) -> None: + version = [1] + + @memoize(ttl=10, stale_ttl=30) + def load(key: str) -> int: + return version[0] + + assert load("a") == 1 + version[0] = 2 + fake_clock.advance(5) + assert load("a") == 1 # fresh + fake_clock.advance(10) + assert load("a") == 1 # stale: served immediately, refresh started + join_refreshes() + assert load("a") == 2 + assert names(events) == ["cache.miss", "cache.hit", "cache.hit", "cache.stale", "cache.hit"] + assert load.cache_info().hits == 3 + assert load.cache_parameters()["stale_ttl"] == 30.0 + + +def test_past_the_stale_window_is_a_plain_miss(fake_clock: FakeClock) -> None: + calls = [] + + @memoize(ttl=datetime.timedelta(seconds=10), stale_ttl=datetime.timedelta(seconds=5)) + def load() -> int: + calls.append(1) + return len(calls) + + assert load() == 1 + fake_clock.advance(16) + assert load() == 2 # computed in the caller, not in the background + assert len(calls) == 2 + + +def test_one_refresh_thread_per_key(fake_clock: FakeClock) -> None: + release = threading.Event() + calls = [] + + @memoize(ttl=1, stale_ttl=100) + def load(key: int) -> int: + calls.append(key) + if len(calls) > 2: + release.wait(2) + return len(calls) + + load(1) + load(2) + fake_clock.advance(2) + results = [load(1) for _ in range(5)] + [load(2)] + assert results == [1, 1, 1, 1, 1, 2] + release.set() + join_refreshes() + assert sorted(calls) == [1, 1, 2, 2] # one refresh for each key + + +def test_refresh_thread_sees_the_callers_context(fake_clock: FakeClock) -> None: + var: contextvars.ContextVar[str] = contextvars.ContextVar("var", default="unset") + seen = [] + + @memoize(ttl=1, stale_ttl=10) + def load() -> str: + seen.append(var.get()) + return var.get() + + load() + fake_clock.advance(2) + var.set("request-42") + load() + join_refreshes() + assert seen == ["unset", "request-42"] + + +def test_failed_refresh_keeps_the_stale_value( + fake_clock: FakeClock, events: list[_core.Event], caplog: pytest.LogCaptureFixture +) -> None: + fail = [False] + calls = [] + + @memoize(ttl=1, stale_ttl=10) + def load() -> int: + calls.append(1) + if fail[0]: + raise ConnectionError("down") + return len(calls) + + assert load() == 1 + fake_clock.advance(2) + fail[0] = True + with caplog.at_level(logging.WARNING, logger="python_wrap"): + assert load() == 1 + join_refreshes() + assert "background refresh" in caplog.text + failed = [e for e in events if e.name == "cache.refresh_failed"] + assert len(failed) == 1 and isinstance(failed[0].fields["exception"], ConnectionError) + fail[0] = False + assert load() == 1 # still stale: the next call retries + join_refreshes() + assert load() == 3 + assert len(calls) == 3 + + +def test_cached_exceptions_are_not_served_stale(fake_clock: FakeClock) -> None: + calls = [] + + @memoize(ttl=1, stale_ttl=10, cache_exceptions=KeyError) + def load() -> int: + calls.append(1) + raise KeyError("x") + + for _ in range(2): + with pytest.raises(KeyError): + load() + fake_clock.advance(2) + with pytest.raises(KeyError): + load() + assert len(calls) == 2 + + +def test_stale_with_callable_ttl(fake_clock: FakeClock) -> None: + version = [1] + + @memoize(ttl=lambda value: None if value == 0 else 5, stale_ttl=5) + def load(n: int) -> int: + return n * version[0] + + load(0) + load(1) + version[0] = 10 + fake_clock.advance(6) + assert load(0) == 0 # ttl None: never stale + assert load(1) == 1 # stale + join_refreshes() + assert load(1) == 10 + + +def test_stale_ttl_validation() -> None: + with pytest.raises(ValueError, match="requires ttl"): + memoize(stale_ttl=5) + with pytest.raises(ValueError): + memoize(ttl=5, stale_ttl=0) + + +def test_stale_respects_disable_caching(fake_clock: FakeClock) -> None: + calls = [] + + @memoize(ttl=1, stale_ttl=10) + def load() -> int: + calls.append(1) + return len(calls) + + load() + fake_clock.advance(2) + with disable_caching(): + assert load() == 2 + assert load() == 3 + join_refreshes() + assert load() == 1 # the stale entry is untouched; its refresh starts now + join_refreshes() + assert load() == 4 + + +def test_cache_clear_discards_a_running_refresh(fake_clock: FakeClock) -> None: + started, release = threading.Event(), threading.Event() + calls = [] + + @memoize(ttl=1, stale_ttl=10) + def load() -> int: + calls.append(1) + if len(calls) == 2: + started.set() + release.wait(2) + return len(calls) + + load() + fake_clock.advance(2) + load() + assert started.wait(2) + load.cache_clear() + release.set() + join_refreshes() + assert load.cache_info().currsize == 0 + + +def test_async_stale_refresh_runs_once(fake_clock: FakeClock, events: list[_core.Event]) -> None: + calls = [] + + @memoize(ttl=1, stale_ttl=10) + async def load(key: str) -> int: + calls.append(key) + await asyncio.sleep(0) + return len(calls) + + async def main() -> None: + assert await load("a") == 1 + fake_clock.advance(2) + assert await asyncio.gather(load("a"), load("a"), load(key="a")) == [1, 1, 1] + await settle() + assert await load("a") == 2 + + asyncio.run(main()) + assert calls == ["a", "a"] + assert names(events).count("cache.stale") == 1 + + +def test_async_failed_refresh_keeps_stale(fake_clock: FakeClock, events: list[_core.Event]) -> None: + fail = [False] + + @memoize(ttl=1, stale_ttl=10) + async def load() -> int: + if fail[0]: + raise ConnectionError("down") + return 1 + + async def main() -> None: + await load() + fake_clock.advance(2) + fail[0] = True + assert await load() == 1 + await settle() + assert await load() == 1 + await settle() + + asyncio.run(main()) + assert names(events).count("cache.refresh_failed") == 2 + + +# ------------------------------------------------------------------- custom stores + + +def test_custom_store_round_trip(fake_clock: FakeClock, events: list[_core.Event]) -> None: + store = DictStore() + calls = [] + + @memoize(store=store, ttl=10, stale_ttl=5) + def double(n: int) -> int: + calls.append(n) + return n * 2 + + assert isinstance(store, CacheStore) + assert double(2) == double(n=2) == 4 + assert calls == [2] + assert store.ttls == [15.0] # the store may discard the entry after ttl + stale_ttl + assert double.cache_info() == CacheInfo(hits=1, misses=1, maxsize=None, currsize=1) + assert double.cache_contains(2) and double.cache_get(2) == 4 + assert double.cache_delete(2) is True and double.cache_delete(2) is False + double(3) + fake_clock.advance(20) + assert not double.cache_contains(3) + double(3) + assert names(events)[-2:] == ["cache.evicted", "cache.miss"] + double.cache_clear() + assert len(store) == 0 and double.cache_info() == (0, 0, None, 0) + assert double.cache_parameters()["store"] is store + + +def test_store_entries_survive_serialisation(fake_clock: FakeClock) -> None: + store = PickleStore() + calls = [] + + @memoize(store=store, ttl=10, cache_exceptions=LookupError) + def find(items: list[int], missing: bool = False) -> int: + calls.append(1) + if missing: + raise LookupError("nope") + return sum(items) + + assert find([1, 2]) == find(items=[1, 2]) == 3 + for _ in range(2): + with pytest.raises(LookupError): + find([1], missing=True) + assert len(calls) == 2 + assert all(isinstance(k, bytes) for k in store.data) + fake_clock.advance(11) + find([1, 2]) + assert len(calls) == 3 + + +def test_custom_store_async() -> None: + store = DictStore() + calls = [] + + @memoize(store=store) + async def load(n: int) -> int: + calls.append(n) + await asyncio.sleep(0) + return n + + async def main() -> None: + assert await asyncio.gather(load(1), load(1)) == [1, 1] + assert await load(1) == 1 + + asyncio.run(main()) + assert calls == [1] + assert load.cache_info().hits == 2 + + +def test_custom_store_unhashable_argument_raises() -> None: + @memoize(store=DictStore()) + def f(x: Any) -> Any: + return x + + with pytest.raises(TypeError, match="not hashable"): + f(object.__new__(type("U", (), {"__hash__": None}))) + + +def test_custom_store_threads() -> None: + store = DictStore() + + @memoize(store=store) + def f(n: int) -> int: + return n * n + + def worker() -> None: + for i in range(200): + assert f(i % 20) == (i % 20) ** 2 + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + info = f.cache_info() + assert info.hits + info.misses == 1600 and info.currsize == 20 + + +def test_store_option_errors() -> None: + with pytest.raises(TypeError, match="maxsize"): + memoize(store=DictStore(), maxsize=3) + with pytest.raises(TypeError, match="per_instance"): + memoize(store=DictStore(), per_instance=True) + with pytest.raises(TypeError, match="CacheStore"): + memoize(store=object()) # type: ignore[arg-type] + + +# ----------------------------------------------------------------- per_instance + + +@dataclasses.dataclass +class Point: # eq=True: instances are unhashable + x: int + + @memoize(per_instance=True, maxsize=2) + def scaled(self, k: int) -> int: + calls.append((self.x, k)) + return self.x * k + + +calls: list[tuple[int, int]] = [] + + +def test_per_instance_unhashable_identity_and_gc() -> None: + calls.clear() + Point.scaled.cache_clear() + a, b = Point(2), Point(2) # equal but distinct: separate caches + assert a.scaled(3) == a.scaled(k=3) == b.scaled(3) == 6 + assert calls == [(2, 3), (2, 3)] + assert Point.scaled.cache_info() == CacheInfo(hits=1, misses=2, maxsize=2, currsize=2) + ref = weakref.ref(a) + del a + gc.collect() + assert ref() is None + assert Point.scaled.cache_info().currsize == 1 + + +def test_per_instance_maxsize_and_controls() -> None: + calls.clear() + Point.scaled.cache_clear() + p = Point(1) + for k in (1, 2, 3): + p.scaled(k) + assert Point.scaled.cache_info().currsize == 2 # LRU per instance + assert not p.scaled.cache_contains(p, 1) + assert p.scaled.cache_get(p, 3) == 3 + assert Point.scaled.cache_delete(p, 3) is True + other = Point(9) + assert not Point.scaled.cache_contains(other, 3) + assert Point.scaled.cache_delete(other, 3) is False + with pytest.raises(KeyError): + Point.scaled.cache_get(other, 3) + Point.scaled.cache_clear() + assert Point.scaled.cache_info() == (0, 0, 2, 0) + p.scaled(2) + assert calls[-1] == (1, 2) + + +def test_per_instance_needs_weakref() -> None: + class Slotted: + __slots__ = ("v",) + + @memoize(per_instance=True) + def get(self) -> int: + return 1 + + class SlottedWeak: + __slots__ = ("__weakref__",) + + @memoize(per_instance=True) + def get(self) -> int: + return 2 + + with pytest.raises(TypeError, match="__weakref__"): + Slotted().get() + assert SlottedWeak().get() == 2 + + +def test_per_instance_argument_errors() -> None: + with pytest.raises(TypeError, match="method"): + + @memoize(per_instance=True) + def nothing() -> None: ... + + @memoize(per_instance=True) + def anything(*args: Any) -> int: + return len(args) + + with pytest.raises(TypeError, match="first positional"): + anything() + assert anything.cache_delete() is False + + +def test_per_instance_ttl_key_typed_and_classmethod(fake_clock: FakeClock) -> None: + seen = [] + + class Svc: + @memoize(per_instance=True, ttl=5, typed=True) + def get(self, n: Any) -> str: + return f"{n!r}" + + @memoize(per_instance=True, key=lambda self, n, verbose=False: n) + def label(self, n: int, verbose: bool = False) -> int: + seen.append(n) + return n + + @classmethod + @memoize(per_instance=True) + def make(cls, n: int) -> int: + return n + + s = Svc() + assert s.get(1) == "1" and s.get(1.0) == "1.0" + fake_clock.advance(6) + assert Svc.get.cache_info().currsize == 0 # swept + s.label(1) + s.label(1, verbose=True) + assert seen == [1] + assert Svc.make(3) == Svc.make(3) == 3 + + +def test_per_instance_without_signature() -> None: + class NoSignature: + __signature__ = "not a signature" + + def __call__(self, *args: Any, **kwargs: Any) -> int: + return len(args) + len(kwargs) + + f = memoize(per_instance=True)(NoSignature()) + + class Owner: + pass + + o = Owner() + assert f(o, 1, b=2) == 3 + assert f.cache_contains(o, 1, b=2) + + +def test_per_instance_async_and_threads() -> None: + runs = [] + + class Repo: + @memoize(per_instance=True) + async def load(self, key: int) -> int: + runs.append(key) + await asyncio.sleep(0) + return key + + @memoize(per_instance=True) + def sync_load(self, key: int) -> int: + return key * 2 + + r1, r2 = Repo(), Repo() + + async def main() -> None: + assert await asyncio.gather(r1.load(1), r1.load(1), r2.load(1)) == [1, 1, 1] + assert await r1.load(1) == 1 + + asyncio.run(main()) + assert runs == [1, 1] + + def worker() -> None: + for i in range(100): + assert r1.sync_load(i % 10) == (i % 10) * 2 + + threads = [threading.Thread(target=worker) for _ in range(6)] + for t in threads: + t.start() + for t in threads: + t.join() + assert Repo.sync_load.cache_info().currsize == 10 + + +def test_per_instance_with_stale_refresh(fake_clock: FakeClock) -> None: + class Feed: + def __init__(self) -> None: + self.version = 1 + + @memoize(per_instance=True, ttl=1, stale_ttl=10) + def latest(self) -> int: + return self.version + + @memoize(per_instance=True, ttl=1, stale_ttl=10) + async def alatest(self) -> int: + return self.version + + feed = Feed() + assert feed.latest() == 1 + feed.version = 2 + fake_clock.advance(2) + assert feed.latest() == 1 + join_refreshes() + assert feed.latest() == 2 + + async def main() -> None: + assert await feed.alatest() == 2 + feed.version = 3 + fake_clock.advance(2) + assert await feed.alatest() == 2 + await settle() + assert await feed.alatest() == 3 + + asyncio.run(main()) + + +def test_per_instance_disable_caching() -> None: + n = [] + + class C: + @memoize(per_instance=True) + def f(self) -> int: + n.append(1) + return len(n) + + c = C() + with disable_caching(): + assert c.f() == 1 and c.f() == 2 + assert C.f.cache_info().currsize == 0 + + +def test_refresh_thread_start_failure_keeps_the_stale_value( + fake_clock: FakeClock, monkeypatch: pytest.MonkeyPatch, events: list[_core.Event] +) -> None: + @memoize(ttl=1, stale_ttl=10) + def load() -> int: + return 1 + + load() + fake_clock.advance(2) + + def boom(self: threading.Thread) -> None: + raise RuntimeError("can't start new thread") + + with monkeypatch.context() as m: + m.setattr(threading.Thread, "start", boom) + assert load() == 1 + assert names(events)[-1] == "cache.refresh_failed" + assert load() == 1 # the key is not stuck as "refreshing" + assert names(events)[-1] == "cache.stale" + join_refreshes() diff --git a/tests/test_concurrency.py b/tests/test_concurrency.py index c0bad59..b2f5073 100644 --- a/tests/test_concurrency.py +++ b/tests/test_concurrency.py @@ -5,11 +5,13 @@ import contextlib import gc import inspect +import logging import signal import sys import threading import time import types +from typing import Any import pytest @@ -1798,3 +1800,229 @@ def late(): with pytest.raises(TimeoutExpired): late() assert [e.name for e in events] == ["timeout.expired"] + + +# ------------------------------------------------------------------ timeout(cleanup_grace=) + + +def test_cleanup_grace_validation() -> None: + with pytest.raises(ValueError): + timeout(1, cleanup_grace=-1) + with pytest.raises(TypeError): + + @timeout(1, cleanup_grace=1) + def sync() -> None: ... + + with pytest.raises(TypeError), timeout(1, cleanup_grace=1): + pass + + +def test_cleanup_grace_decorator_cleanup_finishes() -> None: + cleaned: list[str] = [] + + @timeout(0.02, cleanup_grace=0.5) + async def work() -> None: + try: + await asyncio.sleep(5) + finally: + await asyncio.sleep(0.02) + cleaned.append("done") + + with pytest.raises(TimeoutExpired): + asyncio.run(work()) + assert cleaned == ["done"] + + +def test_cleanup_grace_decorator_abandons_slow_cleanup( + caplog: pytest.LogCaptureFixture, +) -> None: + finished: list[str] = [] + + @timeout(0.02, cleanup_grace=0.02) + async def work() -> None: + try: + await asyncio.sleep(5) + finally: + await asyncio.sleep(0.2) + finished.append("late") + + async def main() -> None: + with pytest.raises(TimeoutExpired): + await work() + assert finished == [] + await asyncio.sleep(0.3) # the abandoned task keeps running + assert finished == ["late"] + + with caplog.at_level(logging.WARNING, logger="python_wrap"): + asyncio.run(main()) + assert "abandoning" in caplog.text + + +def test_cleanup_grace_decorator_passthrough_and_outer_cancel() -> None: + @timeout(1, cleanup_grace=0.1) + async def quick() -> int: + return 7 + + @timeout(1, cleanup_grace=0.1) + async def fails() -> None: + raise KeyError("x") + + async def main() -> None: + assert await quick() == 7 + with pytest.raises(KeyError): + await fails() + started = asyncio.Event() + stopped: list[bool] = [] + + @timeout(5, cleanup_grace=0.1) + async def slow() -> None: + started.set() + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + stopped.append(True) + raise + + t = asyncio.ensure_future(slow()) + await started.wait() + t.cancel() + with pytest.raises(asyncio.CancelledError): + await t + await asyncio.sleep(0) + assert stopped == [True] + + asyncio.run(main()) + + +def test_cleanup_grace_block() -> None: + cleaned: list[str] = [] + + async def main() -> None: + with pytest.raises(TimeoutExpired): + async with timeout(0.02, cleanup_grace=0.5): + try: + await asyncio.sleep(5) + finally: + await asyncio.sleep(0.02) + cleaned.append("done") + async with timeout(1, cleanup_grace=0.1): + pass + + asyncio.run(main()) + assert cleaned == ["done"] + + +def test_cleanup_grace_block_second_cancel(caplog: pytest.LogCaptureFixture) -> None: + async def main() -> None: + with pytest.raises(TimeoutExpired): + async with timeout(0.02, cleanup_grace=0.02): + try: + await asyncio.sleep(5) + finally: + await asyncio.sleep(5) + + with caplog.at_level(logging.WARNING, logger="python_wrap"): + asyncio.run(main()) + assert "cancelling it again" in caplog.text + + +# ------------------------------------------------------------------ cleanup_grace + outer cancel + + +def test_cleanup_grace_block_outer_cancel_during_grace_stays_cancelled() -> None: + in_cleanup = asyncio.Event() + + async def block() -> None: + async with timeout(0.01, cleanup_grace=5): + try: + await asyncio.sleep(5) + finally: + in_cleanup.set() + await asyncio.sleep(5) # interrupted by the outside cancel below + + async def main() -> None: + t = asyncio.ensure_future(block()) + await in_cleanup.wait() # the timer has fired + t.cancel() # the application cancels the task during the grace period + with pytest.raises(asyncio.CancelledError): + await t + + asyncio.run(main()) + + +def test_cleanup_grace_block_keeps_cancel_pending_on_entry() -> None: + async def main() -> None: + task = asyncio.current_task() + assert task is not None + if not hasattr(task, "cancelling"): # pragma: no cover - 3.10 has no counter + return + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.sleep(1) + assert task.cancelling() == 1 # a cancel request is still outstanding + with pytest.raises(TimeoutExpired): + async with timeout(0.01, cleanup_grace=0.5): + await asyncio.sleep(5) + assert task.cancelling() == 1 # only the timer's own requests were withdrawn + task.uncancel() + + asyncio.run(main()) + + +def test_cleanup_grace_decorator_outer_cancel_waits_for_and_consumes_child() -> None: + in_cleanup = asyncio.Event() + finished: list[bool] = [] + unhandled: list[dict[str, Any]] = [] + + @timeout(5, cleanup_grace=1) + async def work() -> None: + try: + await asyncio.sleep(5) + finally: + in_cleanup.set() + await asyncio.sleep(0.02) + finished.append(True) + raise ValueError("cleanup failed") + + async def main() -> None: + asyncio.get_running_loop().set_exception_handler(lambda _loop, ctx: unhandled.append(ctx)) + t = asyncio.ensure_future(work()) + await asyncio.sleep(0.01) + t.cancel() + with pytest.raises(asyncio.CancelledError): + await t + assert finished == [True] # the child's cleanup ran within the grace period + + asyncio.run(main()) + gc.collect() + assert unhandled == [] + + +def test_cleanup_grace_decorator_outer_cancel_during_grace_consumes_child() -> None: + in_cleanup = asyncio.Event() + release = asyncio.Event() + unhandled: list[dict[str, Any]] = [] + + @timeout(0.01, cleanup_grace=5) + async def work() -> None: + try: + await asyncio.sleep(5) + finally: + in_cleanup.set() + await release.wait() + raise ValueError("cleanup failed") + + async def main() -> None: + asyncio.get_running_loop().set_exception_handler(lambda _loop, ctx: unhandled.append(ctx)) + t = asyncio.ensure_future(work()) + await in_cleanup.wait() # the limit expired; the wrapper waits out the grace + t.cancel() + with pytest.raises(asyncio.CancelledError): + await t + release.set() + for _ in range(5): + await asyncio.sleep(0) + + asyncio.run(main()) + gc.collect() + assert unhandled == [] diff --git a/tests/test_events.py b/tests/test_events.py index 97d28a4..31941fd 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -86,4 +86,16 @@ def test_unsubscribe_during_emit_is_safe() -> None: def test_event_names_catalogue() -> None: assert "retry.attempt_failed" in EVENT_NAMES assert all(isinstance(v, str) and v for v in EVENT_NAMES.values()) - assert len(EVENT_NAMES) == 13 + # Every name the package emits is catalogued, and nothing else. + import pathlib + import re + + import python_wrap + + src = pathlib.Path(python_wrap.__file__).parent + emitted = { + name + for path in src.glob("*.py") + for name in re.findall(r"""emit\(\s*["']([a-z_]+\.[a-z_]+)["']""", path.read_text()) + } + assert emitted == set(EVENT_NAMES) diff --git a/tests/test_logs.py b/tests/test_logs.py index 8f73d14..7571cb9 100644 --- a/tests/test_logs.py +++ b/tests/test_logs.py @@ -4,6 +4,7 @@ import functools import inspect import logging +import sys from datetime import timedelta, timezone from typing import Any @@ -482,7 +483,9 @@ async def main() -> list[dict[str, object]]: t = threading.Thread(target=lambda: seen.update(copied=ctx.run(current_log_context))) t.start() t.join() - assert seen == {"plain": {}, "copied": {"a": 1}} + # Free-threaded 3.14 builds start threads in a copy of the caller's context. + inherits = getattr(sys.flags, "thread_inherit_context", 0) + assert seen == {"plain": {"a": 1} if inherits else {}, "copied": {"a": 1}} def test_log_context_restores_on_error() -> None: @@ -694,3 +697,81 @@ async def f(a: int, token: str) -> int: assert asyncio.run(f(2, "t")) == 2 assert records[0].arguments == {"a": 2, "token": "***"} + + +# ---------------------------------------------------------------------------- trace(otel=) + + +def test_trace_otel_missing_package_raises(monkeypatch: pytest.MonkeyPatch) -> None: + import sys + + monkeypatch.setitem(sys.modules, "opentelemetry", None) + with pytest.raises(ImportError, match="opentelemetry-api"): + + @trace(otel=True) + def f() -> None: ... + + +def _otel_exporter() -> Any: + pytest.importorskip("opentelemetry.sdk.trace") + from opentelemetry import trace as ot + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + # get_tracer is looked up at decoration time, so patch the global lazily + ot._TRACER_PROVIDER = provider # type: ignore[attr-defined] + ot._TRACER_PROVIDER_SET_ONCE._done = True # type: ignore[attr-defined] + return exporter + + +def test_trace_otel_span_attributes_and_error() -> None: + exporter = _otel_exporter() + from opentelemetry.trace import StatusCode + + from python_wrap.logs import log_context + + @trace(otel=True, span_name="custom") + def ok(x: int) -> int: + return x + + @trace(otel=True) + def bad() -> None: + raise ValueError("boom") + + with log_context(request_id="r1", n=3, obj=[1]): + assert ok(1) == 1 + with pytest.raises(ValueError): + bad() + first, second = exporter.get_finished_spans() + assert first.name == "custom" + assert first.attributes["code.function"] == "ok" + assert first.attributes["code.namespace"] == __name__ + assert first.attributes["python_wrap.context.request_id"] == "r1" + assert first.attributes["python_wrap.context.n"] == 3 + assert first.attributes["python_wrap.context.obj"] == "[1]" + assert second.name.endswith("bad") + assert second.status.status_code == StatusCode.ERROR + assert second.events[0].name == "exception" + + +def test_trace_otel_async_span_current_across_await() -> None: + exporter = _otel_exporter() + from opentelemetry import trace as ot + + seen: list[bool] = [] + + @trace(otel=True) + async def work() -> int: + await asyncio.sleep(0) + seen.append(ot.get_current_span().is_recording()) + return 1 + + assert asyncio.run(work()) == 1 + assert seen == [True] + assert inspect.iscoroutinefunction(work) + assert len(exporter.get_finished_spans()) == 1 + assert not ot.get_current_span().get_span_context().is_valid diff --git a/tests/test_periodic.py b/tests/test_periodic.py new file mode 100644 index 0000000..ef8463c --- /dev/null +++ b/tests/test_periodic.py @@ -0,0 +1,705 @@ +"""Tests for python_wrap.periodic.""" + +from __future__ import annotations + +import asyncio +import contextvars +import itertools +import logging +import sys +import threading +import time +from collections.abc import Iterator +from typing import Any + +import pytest + +from python_wrap import _core +from python_wrap.periodic import AsyncPeriodic, Periodic, _delay_until, _next_tick, periodic + +periodic_module = sys.modules["python_wrap.periodic"] + +TICK = 0.005 + + +def wait_for(predicate: Any, timeout: float = 2.0) -> None: + end = time.monotonic() + timeout + while not predicate(): + assert time.monotonic() < end, "timed out" + time.sleep(0.001) + + +def periodic_threads() -> list[threading.Thread]: + return [t for t in threading.enumerate() if t.name.startswith("python-wrap-periodic")] + + +@pytest.fixture(autouse=True) +def no_leaks() -> Iterator[None]: + yield + wait_for(lambda: not periodic_threads()) + + +# ------------------------------------------------------------------ validation + + +@pytest.mark.parametrize( + "kwargs", + [ + {"interval": 0}, + {"interval": -1}, + {"interval": True}, + {"interval": 1, "jitter": 1.0}, + {"interval": 1, "jitter": -0.1}, + {"interval": 1, "initial_delay": -1}, + {"interval": 1, "max_runs": 0}, + {"interval": 1, "on_error": "explode"}, + ], +) +def test_invalid_arguments(kwargs: dict[str, Any]) -> None: + with pytest.raises(ValueError): + periodic(**kwargs) + + +def test_rejects_non_callable_and_unbound_method() -> None: + with pytest.raises(TypeError): + periodic(1)(5) # type: ignore[call-overload] + + with pytest.raises(TypeError, match="bound method"): + + class C: + @periodic(1) + def refresh(self) -> None: ... + + +def test_bound_method_works() -> None: + class C: + def __init__(self) -> None: + self.n = 0 + + def refresh(self, step: int) -> None: + self.n += step + + c = C() + task = periodic(TICK, max_runs=3)(c.refresh) + task.start(2) + wait_for(lambda: not task.running) + assert c.n == 6 + + +# ------------------------------------------------------------------ sync + + +def test_direct_call_and_metadata() -> None: + @periodic(1) + def add(a: int, b: int) -> int: + """Doc.""" + return a + b + + assert isinstance(add, Periodic) + assert add(1, 2) == 3 + assert add.__name__ == "add" + assert add.__doc__ == "Doc." + assert not add.running + assert add.runs == 0 + assert add.last_error is None + assert add.stop() is True # never started: harmless + + +def test_runs_repeatedly_with_args_and_stops() -> None: + seen: list[tuple[int, str]] = [] + + @periodic(TICK) + def work(n: int, *, tag: str) -> None: + seen.append((n, tag)) + + assert work.start(1, tag="x") is work + assert work.running + wait_for(lambda: work.runs >= 3) + assert work.stop() is True + assert not work.running + count = work.runs + time.sleep(TICK * 4) + assert work.runs == count + assert set(seen) == {(1, "x")} + assert work.stop() is True # idempotent + + +def test_double_start_rejected_and_restart_resets() -> None: + @periodic(TICK, max_runs=2) + def work() -> None: ... + + work.start() + with pytest.raises(RuntimeError, match="already running"): + work.start() + wait_for(lambda: not work.running) + assert work.runs == 2 + work.start() + wait_for(lambda: not work.running) + assert work.runs == 2 + + +def test_max_runs_ends_by_itself() -> None: + @periodic(TICK, max_runs=3) + def work() -> None: ... + + work.start() + wait_for(lambda: not work.running) + assert work.runs == 3 + assert work.stop() is True + + +def test_initial_delay_and_stop_during_delay() -> None: + calls: list[int] = [] + + @periodic(TICK, initial_delay=30) + def work() -> None: + calls.append(1) + + work.start() + time.sleep(0.02) + assert work.runs == 0 + started = time.monotonic() + assert work.stop() is True + assert time.monotonic() - started < 1 + assert calls == [] + + +def test_context_manager_stops_and_joins() -> None: + @periodic(TICK) + def work() -> None: ... + + with work.running_in_background() as inner: + assert inner is work + assert work.running + wait_for(lambda: work.runs >= 1) + assert not work.running + assert periodic_threads() == [] + + +def test_context_manager_stops_on_exception() -> None: + @periodic(TICK) + def work() -> None: ... + + with pytest.raises(KeyError), work.running_in_background(): + raise KeyError("x") + assert not work.running + + +def test_thread_is_daemon_and_named() -> None: + seen: list[threading.Thread] = [] + + @periodic(TICK, max_runs=1, name="heartbeat") + def work() -> None: + seen.append(threading.current_thread()) + + work.start() + wait_for(lambda: seen) + assert seen[0].daemon + assert seen[0].name == "python-wrap-periodic:heartbeat" + + +def test_stop_times_out_while_run_in_progress() -> None: + release = threading.Event() + entered = threading.Event() + + @periodic(TICK) + def slow() -> None: + entered.set() + release.wait(5) + + slow.start() + entered.wait(2) + assert slow.stop(timeout=0.01) is False + assert slow.running + release.set() + assert slow.stop() is True + + +def test_stop_from_inside_function_does_not_deadlock() -> None: + @periodic(TICK) + def work() -> None: + work.stop() + + work.start() + wait_for(lambda: not work.running) + assert work.runs == 1 + + +def test_contextvars_are_copied() -> None: + var: contextvars.ContextVar[str] = contextvars.ContextVar("var", default="unset") + seen: list[str] = [] + + @periodic(TICK, max_runs=2) + def work() -> None: + seen.append(var.get()) + + var.set("caller") + work.start() + wait_for(lambda: not work.running) + assert seen == ["caller", "caller"] + + +# ------------------------------------------------------------------ errors + + +def boom_every_time() -> Any: + @periodic(TICK, max_runs=3, on_error="log") + def work() -> None: + raise ValueError("bad") + + return work + + +def test_on_error_log_continues(caplog: pytest.LogCaptureFixture) -> None: + work = boom_every_time() + with caplog.at_level(logging.ERROR, logger="python_wrap"): + work.start() + wait_for(lambda: not work.running) + assert work.runs == 3 + assert isinstance(work.last_error, ValueError) + assert sum("failed" in r.getMessage() for r in caplog.records) == 3 + assert work.stop() is True # nothing to re-raise in "log" mode + + +def test_on_error_stop_ends_loop_quietly(caplog: pytest.LogCaptureFixture) -> None: + @periodic(TICK, on_error="stop") + def work() -> None: + raise ValueError("bad") + + with caplog.at_level(logging.ERROR, logger="python_wrap"): + work.start() + wait_for(lambda: not work.running) + assert work.runs == 1 + assert isinstance(work.last_error, ValueError) + assert work.stop() is True + assert any("stopping" in r.getMessage() for r in caplog.records) + + +def test_on_error_raise_surfaces_from_stop_once() -> None: + @periodic(TICK, on_error="raise") + def work() -> None: + raise ValueError("bad") + + work.start() + wait_for(lambda: not work.running) + with pytest.raises(ValueError, match="bad"): + work.stop() + assert work.stop() is True + assert isinstance(work.last_error, ValueError) + + +def test_on_error_raise_via_context_manager() -> None: + @periodic(TICK, on_error="raise") + def work() -> None: + raise ValueError("bad") + + with pytest.raises(ValueError, match="bad"), work.running_in_background(): + wait_for(lambda: not work.running) + + +def test_custom_logger_name(caplog: pytest.LogCaptureFixture) -> None: + @periodic(TICK, max_runs=1, logger="mine", level=logging.WARNING) + def work() -> None: + raise ValueError("bad") + + with caplog.at_level(logging.WARNING, logger="mine"): + work.start() + wait_for(lambda: not work.running) + assert [r.name for r in caplog.records] == ["mine"] + assert caplog.records[0].levelno == logging.WARNING + + +# ------------------------------------------------------------------ events + + +def test_events() -> None: + events: list[_core.Event] = [] + _core.listeners.append(events.append) + try: + + @periodic(TICK, max_runs=2, name="job") + def ok() -> None: ... + + @periodic(TICK, max_runs=1, name="bad") + def bad() -> None: + raise ValueError("x") + + ok.start() + bad.start() + wait_for(lambda: not ok.running and not bad.running) + finally: + _core.listeners.remove(events.append) + runs = [e for e in events if e.name == "periodic.run"] + assert [e.fields["run"] for e in runs] == [1, 2] + assert all(e.function == "job" and e.fields["seconds"] >= 0 for e in runs) + (failed,) = [e for e in events if e.name == "periodic.failed"] + assert failed.function == "bad" + assert isinstance(failed.fields["exception"], ValueError) + + +# ------------------------------------------------------------------ schedule maths + + +def test_next_tick_has_no_drift_and_skips_missed() -> None: + # On time: simply due + interval, however late within the interval. + assert _next_tick(10.0, 10.4, 1.0) == 11.0 + # Run took 2.5 intervals: ticks 11 and 12 are skipped, resume at 13. + assert _next_tick(10.0, 12.5, 1.0) == 13.0 + # Exactly on a tick counts as missed. + assert _next_tick(10.0, 12.0, 1.0) == 13.0 + # Many consecutive slow runs never move the grid off first + n * interval. + due = 100.0 + for now in (100.9, 102.2, 102.9, 105.5, 106.0): + due = _next_tick(due, now, 1.0) + assert (due - 100.0) % 1.0 == pytest.approx(0.0, abs=1e-9) + assert due > now + + +def test_jitter_bounds_and_scale(monkeypatch: pytest.MonkeyPatch) -> None: + for _ in range(500): + delay = _delay_until(10.0, 0.0, 2.0, 0.25) + assert 9.5 <= delay <= 10.5 + assert _delay_until(10.0, 0.0, 2.0, 0.0) == 10.0 + assert _delay_until(1.0, 5.0, 2.0, 0.0) == 0.0 # never negative + seen: list[tuple[float, float]] = [] + monkeypatch.setattr(periodic_module, "_uniform", lambda a, b: seen.append((a, b)) or 0.0) + _delay_until(10.0, 0.0, 4.0, 0.1) + assert seen == [(-0.1, 0.1)] + + +def test_loop_real_time_start_to_start_without_drift(monkeypatch: pytest.MonkeyPatch) -> None: + # Check the ticks the loop schedules rather than measured gaps: CI runners can + # overshoot sleeps by tens of milliseconds, but the schedule must stay on the grid. + interval = 0.04 + dues: list[float] = [] + real_delay_until = periodic_module._delay_until + + def recording(due: float, now: float, interval: float, jitter: float) -> float: + dues.append(due) + return real_delay_until(due, now, interval, jitter) + + monkeypatch.setattr(periodic_module, "_delay_until", recording) + + @periodic(interval, max_runs=5) + def work() -> None: + time.sleep(0.015) # work time must not stretch the period + + work.start() + wait_for(lambda: not work.running) + assert len(dues) == 4 + assert all(b > a for a, b in itertools.pairwise(dues)) + # Drift would shift every tick by the work time; on the grid each is a whole + # number of intervals after the first (ticks may be skipped on a slow machine). + for due in dues: + steps = (due - dues[0]) / interval + assert abs(steps - round(steps)) < 1e-6 + + +def test_overrun_skips_missed_ticks_instead_of_bursting() -> None: + starts: list[float] = [] + interval = 0.02 + + @periodic(interval, max_runs=3) + def work() -> None: + starts.append(time.monotonic()) + if len(starts) == 1: + time.sleep(0.07) # misses ~3 ticks + + work.start() + wait_for(lambda: not work.running) + # A burst would start run 2 immediately after run 1 ends; skipping waits for the grid. + assert starts[1] - starts[0] >= 0.07 + + +# ------------------------------------------------------------------ async + + +async def spin_until(predicate: Any, timeout: float = 2.0) -> None: + end = time.monotonic() + timeout + while not predicate(): + assert time.monotonic() < end, "timed out" + await asyncio.sleep(0.001) + + +def test_async_start_outside_loop_is_clear_error() -> None: + @periodic(1) + async def work() -> None: ... + + with pytest.raises(RuntimeError, match="running event loop"): + work.start() + + +async def test_async_run_stop_and_direct_call() -> None: + seen: list[int] = [] + + @periodic(TICK) + async def work(n: int) -> int: + seen.append(n) + return n * 2 + + assert isinstance(work, AsyncPeriodic) + assert await work(4) == 8 + assert work.start(7) is work + assert work.running + with pytest.raises(RuntimeError, match="already running"): + work.start(7) + await spin_until(lambda: work.runs >= 3) + assert await work.stop() is True + assert not work.running + count = work.runs + await asyncio.sleep(TICK * 4) + assert work.runs == count + assert 7 in seen + assert await work.stop() is True + others = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] + assert others == [] + + +async def test_async_marked_as_coroutine_function_when_supported() -> None: + import inspect + + @periodic(1) + async def work() -> None: ... + + if hasattr(inspect, "markcoroutinefunction"): + assert inspect.iscoroutinefunction(work) + + +async def test_async_max_runs_and_initial_delay() -> None: + @periodic(TICK, max_runs=2, initial_delay=0.02) + async def work() -> None: ... + + work.start() + await asyncio.sleep(0.005) + assert work.runs == 0 + await spin_until(lambda: not work.running) + assert work.runs == 2 + work.start() # restart resets + await spin_until(lambda: not work.running) + assert work.runs == 2 + + +async def test_async_context_manager() -> None: + @periodic(TICK) + async def work() -> None: ... + + async with work.running_in_background() as inner: + assert inner is work + await spin_until(lambda: work.runs >= 1) + assert not work.running + + +async def test_async_stop_cancels_run_in_progress() -> None: + started = asyncio.Event() + cancelled = asyncio.Event() + + @periodic(TICK) + async def work() -> None: + started.set() + try: + await asyncio.sleep(30) + except asyncio.CancelledError: + cancelled.set() + raise + + work.start() + await started.wait() + await work.stop() + assert cancelled.is_set() + assert not work.running + + +async def test_async_stop_from_inside_function() -> None: + @periodic(TICK) + async def work() -> None: + await work.stop() + + work.start() + await spin_until(lambda: not work.running) + assert work.runs == 1 + + +async def test_async_error_modes(caplog: pytest.LogCaptureFixture) -> None: + @periodic(TICK, max_runs=3) + async def logs() -> None: + raise ValueError("bad") + + @periodic(TICK, on_error="stop") + async def stops() -> None: + raise ValueError("bad") + + @periodic(TICK, on_error="raise") + async def raises() -> None: + raise KeyError("bad") + + with caplog.at_level(logging.ERROR, logger="python_wrap"): + logs.start() + stops.start() + raises.start() + await spin_until(lambda: not (logs.running or stops.running or raises.running)) + assert logs.runs == 3 + assert stops.runs == 1 + assert isinstance(stops.last_error, ValueError) + assert await logs.stop() and await stops.stop() + with pytest.raises(KeyError): + await raises.stop() + assert await raises.stop() is True + + +async def test_async_raise_via_context_manager() -> None: + @periodic(TICK, on_error="raise") + async def work() -> None: + raise ValueError("bad") + + with pytest.raises(ValueError): + async with work.running_in_background(): + await spin_until(lambda: not work.running) + + +async def test_async_contextvars_and_events() -> None: + var: contextvars.ContextVar[str] = contextvars.ContextVar("var", default="unset") + seen: list[str] = [] + events: list[str] = [] + _core.listeners.append(lambda e: events.append(e.name)) + listener = _core.listeners[-1] + + @periodic(TICK, max_runs=1) + async def work() -> None: + seen.append(var.get()) + + try: + var.set("caller") + work.start() + await spin_until(lambda: not work.running) + finally: + _core.listeners.remove(listener) + assert seen == ["caller"] + assert events == ["periodic.run"] + + +async def test_async_jitter_runs() -> None: + @periodic(TICK, jitter=0.5, max_runs=3) + async def work() -> None: ... + + work.start() + await spin_until(lambda: not work.running) + assert work.runs == 3 + + +def test_fast_mode_does_not_shorten_schedule() -> None: + from python_wrap.testing import fast_mode + + @periodic(0.2, initial_delay=0.2) + def work() -> None: ... + + with fast_mode(): + work.start() + time.sleep(0.05) + assert work.runs == 0 + work.stop() + + +# ------------------------------------------------------------------ review regressions + + +async def test_async_stop_does_not_swallow_the_callers_cancellation() -> None: + started = asyncio.Event() + release = asyncio.Event() + + @periodic(TICK) + async def work() -> None: + started.set() + try: + await asyncio.sleep(30) + finally: + await release.wait() # slow cleanup keeps stop() waiting + + work.start() + await started.wait() + stopper = asyncio.ensure_future(work.stop()) + await asyncio.sleep(0.01) + assert not stopper.done() + stopper.cancel() # aimed at the task awaiting stop(), not at the loop + with pytest.raises(asyncio.CancelledError): + await stopper + assert work.running # the loop is still finishing its cleanup + release.set() + await spin_until(lambda: not work.running) + assert await work.stop() is True + + +class Fatal(BaseException): + """Not an Exception, so the loop does not handle it: it ends the loop task.""" + + +async def test_async_stop_reraises_what_ended_the_loop_task() -> None: + started = asyncio.Event() + + @periodic(TICK) + async def work() -> None: + started.set() + try: + await asyncio.sleep(30) + finally: + raise Fatal + + work.start() + await started.wait() + with pytest.raises(Fatal): + await work.stop() + + +@pytest.mark.parametrize("swallow", ["return", "raise"]) +async def test_async_stop_ends_a_loop_whose_run_swallows_the_cancel(swallow: str) -> None: + started = asyncio.Event() + + @periodic(TICK) + async def work() -> None: + started.set() + try: + await asyncio.sleep(30) + except asyncio.CancelledError: + if swallow == "raise": + raise ValueError("turned the cancel into an error") from None + + work.start() + await started.wait() + assert await asyncio.wait_for(work.stop(), 2) is True + assert work.runs == 1 + + +async def test_async_stop_from_inside_ends_loop_without_waiting_a_tick() -> None: + @periodic(30) + async def work() -> None: + await work.stop() + + work.start() + await spin_until(lambda: not work.running, timeout=0.5) + assert work.runs == 1 + + +def test_sync_stop_from_inside_ends_loop_without_waiting_a_tick() -> None: + @periodic(30) + def work() -> None: + work.stop() + + work.start() + wait_for(lambda: not work.running, timeout=0.5) + assert work.runs == 1 + + +def test_restart_logs_an_uncollected_raise_error(caplog: pytest.LogCaptureFixture) -> None: + @periodic(TICK, on_error="raise") + def work() -> None: + raise ValueError("lost?") + + work.start() + wait_for(lambda: not work.running) + with caplog.at_level(logging.ERROR, logger="python_wrap"): + work.start() # stop() never collected the error + wait_for(lambda: not work.running) + record = next(r for r in caplog.records if "restarted before stop()" in r.getMessage()) + assert record.levelno == logging.ERROR + assert record.exc_info is not None and isinstance(record.exc_info[1], ValueError) + with pytest.raises(ValueError): # the second loop's own error is still surfaced + work.stop() diff --git a/tests/test_pickling.py b/tests/test_pickling.py new file mode 100644 index 0000000..9a83419 --- /dev/null +++ b/tests/test_pickling.py @@ -0,0 +1,175 @@ +"""Decorated functions pickle by reference and run in spawn-context process pools. + +Per-process state holders (breakers, limiters, bulkheads, budgets, transactions, fake +clocks, policies) refuse to pickle with a clear ``TypeError``. +""" + +from __future__ import annotations + +import concurrent.futures +import copy +import functools +import multiprocessing +import os +import pickle +import site +from typing import Any + +import pytest + +import _pickle_helpers as h +import python_wrap as pw +from python_wrap.testing import FakeClock + +DECORATED = sorted( + name for name in vars(h) if name.startswith(("f_", "af_")) and callable(getattr(h, name)) +) + + +@pytest.mark.parametrize("name", DECORATED) +@pytest.mark.parametrize("protocol", [2, pickle.DEFAULT_PROTOCOL, pickle.HIGHEST_PROTOCOL]) +def test_decorated_function_pickles_by_reference(name: str, protocol: int) -> None: + func = getattr(h, name) + data = pickle.dumps(func, protocol=protocol) + assert pickle.loads(data) is func + assert len(data) < 200, "pickled by value instead of by reference" + + +def test_audit_covers_every_kind_of_wrapper() -> None: + # The objects returned by periodic are not functions; they need __reduce__. + assert isinstance(h.f_periodic, pw.Periodic) + assert isinstance(h.af_periodic, pw.AsyncPeriodic) + assert len(DECORATED) >= 40 + + +def test_periodic_running_state_stays_in_its_process() -> None: + assert pickle.loads(pickle.dumps(h.f_periodic)) is h.f_periodic + assert copy.copy(h.f_periodic) is h.f_periodic # copy also goes through __reduce__ + assert not h.f_periodic.running + + +@pytest.mark.parametrize("cls", [h.Service, h.Model]) +def test_classes_with_decorated_methods_pickle_by_reference(cls: type) -> None: + assert pickle.loads(pickle.dumps(cls)) is cls + for attr in ("add",) if cls is h.Service else ("times", "plus", "retried", "counted"): + method = getattr(cls, attr) + assert pickle.loads(pickle.dumps(method)) is method + + +def test_wrap_methods_instance_and_bound_method_pickle() -> None: + svc = h.Service(10) + assert svc.add(1) == 11 + clone = pickle.loads(pickle.dumps(svc)) + assert clone.base == 10 and clone.add(2) == 12 + bound = pickle.loads(pickle.dumps(svc.add)) + assert bound(3) == 13 + + +def test_instance_with_cached_property_and_memoized_methods_pickles() -> None: + m = h.Model(4) + assert m.square == 16 and m.computed == 1 + assert m.times(2) == 8 and m.plus(1) == 5 and m.retried(1) == 3 and m.counted() == 4 + + clone = pickle.loads(pickle.dumps(m)) + assert clone.n == 4 + # cached_property values (and their locks) are not pickled: the copy recomputes. + assert clone.computed == 1 + assert clone.square == 16 and clone.computed == 2 + assert clone.square == 16 and clone.computed == 2 + assert m.square == 16 and m.computed == 1 # the original keeps its cached value + + for attr, arg, expected in [("times", 3, 12), ("plus", 3, 7), ("retried", 3, 1)]: + bound = pickle.loads(pickle.dumps(getattr(m, attr))) + assert bound(arg) == expected + assert pickle.loads(pickle.dumps(m.counted))() == 4 + + +def test_instance_pickles_before_cached_property_is_used() -> None: + clone = pickle.loads(pickle.dumps(h.Model(3))) + assert clone.square == 9 and clone.computed == 1 + + +@pytest.mark.parametrize( + "make", + [ + lambda: pw.CircuitBreaker(), + lambda: h.breaker, + lambda: pw.Bulkhead(2), + lambda: pw.RateLimiter(5, 1), + lambda: pw.limiter_of(h.f_rate_limit), + lambda: pw.RetryBudget(), + lambda: pw.Transaction(), + lambda: FakeClock(), + ], + ids=[ + "breaker", + "module-breaker", + "bulkhead", + "rate-limiter", + "limiter_of", + "budget", + "transaction", + "fake-clock", + ], +) +def test_per_process_state_refuses_to_pickle(make: Any) -> None: + obj = make() + name = type(obj).__name__ + with pytest.raises(TypeError, match=f"cannot pickle '{name}' object: .*one process"): + pickle.dumps(obj) + with pytest.raises(TypeError, match="one process"): + copy.deepcopy(obj) + + +def test_policy_refuses_to_pickle_but_decorated_functions_do() -> None: + with pytest.raises(TypeError, match="cannot pickle a Policy"): + pickle.dumps(h.shared_policy) + assert pickle.loads(pickle.dumps(h.f_policy)) is h.f_policy + + +def test_refuse_pickle_advice_mentions_decorated_functions() -> None: + with pytest.raises(TypeError, match="Create one Bulkhead per process") as info: + pickle.dumps(pw.Bulkhead(1)) + assert "pickle by reference" in str(info.value) + + +SYNC_IN_CHILD = [ + "f_retry", + "f_memoize", + "f_circuit_breaker", + "f_breaker_object", + "f_timeout", + "f_rate_limit", + "f_bulkhead", + "f_batch_calls", + "f_hedge", + "f_periodic", + "f_policy", + "f_count_calls", + "f_stacked", + "f_enforce_types", +] +ASYNC_IN_CHILD = ["af_retry", "af_memoize", "af_periodic", "af_batch_calls"] + + +def test_spawn_process_pool_runs_decorated_functions() -> None: + tests_dir = os.path.dirname(os.path.abspath(h.__file__)) + ctx = multiprocessing.get_context("spawn") + before = h.f_count_calls.stats.calls + with concurrent.futures.ProcessPoolExecutor( + max_workers=1, # one worker: the per-process counter check below is exact + mp_context=ctx, + # Spawn re-imports _pickle_helpers by name; make sure tests/ is importable there. + initializer=functools.partial(site.addsitedir, tests_dir), + ) as pool: + sync = {name: pool.submit(h.call_in_child, getattr(h, name), 1) for name in SYNC_IN_CHILD} + aio = {name: pool.submit(h.run_async, getattr(h, name), 1) for name in ASYNC_IN_CHILD} + direct = list(pool.map(h.f_retry, [1, 2, 3])) # submitting the function itself + seen = pool.submit(h.count_calls_seen).result(timeout=30) + results = {name: f.result(timeout=30) for name, f in {**sync, **aio}.items()} + assert results == dict.fromkeys(SYNC_IN_CHILD + ASYNC_IN_CHILD, 2) + assert direct == [2, 3, 4] + # f_count_calls ran once in the worker (directly and inside nothing else); the + # parent's counter is untouched: state is per process. + assert seen == 1 + assert h.f_count_calls.stats.calls == before diff --git a/tests/test_retry.py b/tests/test_retry.py index 48ad8c4..f069928 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -10,13 +10,17 @@ import threading import time import warnings +from collections.abc import Iterator from typing import Any import pytest from conftest import FakeClock from python_wrap import _core +from python_wrap.config import configure, configured from python_wrap.retry import ( + RetryBudget, + RetryBudgetStats, RetryState, RetryStats, fallback, @@ -27,6 +31,7 @@ retrying, suppress, ) +from python_wrap.testing import disable_retries, fast_mode class Flaky: @@ -822,3 +827,508 @@ def test_events_emitted(fake_clock: FakeClock) -> None: assert seen[1].fields == {"attempts": 2} assert seen[3].fields["attempts"] == 2 assert seen[4].function == "block" and seen[4].fields == {"attempts": 1} + + +# ======================================================= budgets, results, async hooks + + +@pytest.fixture +def events() -> Iterator[list[_core.Event]]: + seen: list[_core.Event] = [] + _core.listeners.append(seen.append) + try: + yield seen + finally: + _core.listeners.remove(seen.append) + + +def always_fails(counter: list[int]) -> Any: + def f() -> None: + counter.append(1) + raise ConnectionError("down") + + return f + + +# --------------------------------------------------------------------------- budget + + +@pytest.mark.parametrize( + "kwargs", + [ + {"ratio": -0.1}, + {"ratio": float("nan")}, + {"ratio": float("inf")}, + {"min_per_second": -1}, + {"window": 0}, + {"window": float("inf")}, + ], +) +def test_budget_validation(kwargs: dict[str, float]) -> None: + with pytest.raises(ValueError): + RetryBudget(**kwargs) + + +def test_budget_ratio_and_reserve(fake_clock: FakeClock) -> None: + budget = RetryBudget(ratio=0.5, min_per_second=0.2, window=10) # reserve = 2 retries + assert budget.stats == RetryBudgetStats(requests=0, retries=0, rejected=0, allowed=2) + for _ in range(4): + budget.deposit() + assert budget.stats.allowed == 4 # 0.5 * 4 + 2 + assert [budget.try_withdraw() for _ in range(5)] == [True] * 4 + [False] + assert budget.stats == RetryBudgetStats(requests=4, retries=4, rejected=1, allowed=0) + assert "ratio=0.5" in repr(budget) + + +def test_budget_window_slides(fake_clock: FakeClock) -> None: + budget = RetryBudget(ratio=1.0, min_per_second=0, window=10) + budget.deposit() + assert budget.try_withdraw() + assert not budget.try_withdraw() + fake_clock.advance(5) + budget.deposit() + assert budget.try_withdraw() + assert budget.stats.requests == 2 + fake_clock.advance(6) # the first deposit and retry are now older than the window + assert budget.stats == RetryBudgetStats(requests=1, retries=1, rejected=1, allowed=0) + fake_clock.advance(10) + assert budget.stats.requests == 0 + budget.reset() + assert budget.stats == RetryBudgetStats(0, 0, 0, 0) + + +def test_budget_exhaustion_stops_retrying_at_once( + fake_clock: FakeClock, events: list[_core.Event] +) -> None: + budget = RetryBudget(ratio=0, min_per_second=0.1, window=10) # one retry per window + calls: list[int] = [] + f = retry(always_fails(calls), max_attempts=5, delay=1, budget=budget) + with pytest.raises(ConnectionError): + f() + assert len(calls) == 2 # first attempt + the one retry the budget allowed + assert fake_clock.sleeps == [1] # no sleep after the refusal + names = [e.name for e in events] + assert names == ["retry.attempt_failed", "retry.budget_exhausted", "retry.gave_up"] + assert events[1].fields["attempts"] == 2 + assert isinstance(events[1].fields["exception"], ConnectionError) + stats = retry_stats(f) + assert stats is not None + assert (stats.retries, stats.gave_up, stats.budget_exhausted) == (1, 1, 1) + assert stats.snapshot()["budget_exhausted"] == 1 + assert "budget_exhausted=1" in repr(stats) + stats.reset() + assert stats.budget_exhausted == 0 + assert budget.stats.rejected == 1 + + +def test_budget_shared_between_functions_and_retrying(fake_clock: FakeClock) -> None: + budget = RetryBudget(ratio=0, min_per_second=0.2, window=10) # two retries + a_calls: list[int] = [] + b_calls: list[int] = [] + a = retry(always_fails(a_calls), max_attempts=3, delay=0, budget=budget) + b = retry(always_fails(b_calls), max_attempts=3, delay=0, budget=budget) + with pytest.raises(ConnectionError): + a() + with pytest.raises(ConnectionError): + b() + assert (len(a_calls), len(b_calls)) == (3, 1) + tries: list[int] = [] + with pytest.raises(ConnectionError): + for attempt in retrying(max_attempts=3, delay=0, budget=budget): + with attempt: + tries.append(attempt.number) + raise ConnectionError + assert tries == [1] + assert budget.stats.requests == 3 + + +def test_budget_does_not_touch_success_or_non_retryable(fake_clock: FakeClock) -> None: + budget = RetryBudget(ratio=0, min_per_second=0, window=1) + + @retry(budget=budget, on=ConnectionError) + def ok() -> int: + return 1 + + @retry(budget=budget, on=ConnectionError) + def wrong() -> None: + raise KeyError("x") + + assert ok() == 1 + with pytest.raises(KeyError): + wrong() + assert budget.stats == RetryBudgetStats(requests=2, retries=0, rejected=0, allowed=0) + + +def test_budget_async(fake_clock: FakeClock, events: list[_core.Event]) -> None: + budget = RetryBudget(ratio=0, min_per_second=0.1, window=10) + calls: list[int] = [] + + @retry(max_attempts=4, delay=0.5, budget=budget) + async def f() -> None: + calls.append(1) + raise ConnectionError + + with pytest.raises(ConnectionError): + asyncio.run(f()) + assert len(calls) == 2 + assert fake_clock.sleeps == [0.5] + assert "retry.budget_exhausted" in [e.name for e in events] + + +def test_budget_thread_safe() -> None: + budget = RetryBudget(ratio=0.5, min_per_second=0, window=1000) + granted: list[bool] = [] + lock = threading.Lock() + + def worker() -> None: + for _ in range(200): + budget.deposit() + ok = budget.try_withdraw() + ok2 = budget.try_withdraw() + with lock: + granted.extend([ok, ok2]) + + threads = [threading.Thread(target=worker) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join() + stats = budget.stats + assert stats.requests == 1600 + assert sum(granted) == stats.retries == 800 # never more than ratio * requests + assert stats.rejected == 3200 - 800 + + +def test_budget_with_fast_mode_and_disable_retries() -> None: + budget = RetryBudget(ratio=0, min_per_second=0.1, window=10) + calls: list[int] = [] + f = retry(always_fails(calls), max_attempts=5, delay=60, budget=budget) + with fast_mode(), pytest.raises(ConnectionError): + f() + assert len(calls) == 2 + budget.reset() + calls.clear() + with disable_retries(), pytest.raises(ConnectionError): + f() + assert len(calls) == 1 + assert budget.stats.retries == 0 # no retry was attempted, so nothing withdrawn + + +def test_budget_type_checked_and_configurable(fake_clock: FakeClock) -> None: + with pytest.raises(TypeError, match="RetryBudget"): + retry(budget="lots") # type: ignore[arg-type] + with pytest.raises(TypeError, match="RetryBudget"): + retrying(budget=3) # type: ignore[arg-type] + with pytest.raises(TypeError, match="RetryBudget"): + configure(retry={"budget": 0.1}) + budget = RetryBudget(ratio=0, min_per_second=0, window=1) + with configured(retry={"budget": budget, "delay": 0}): + calls: list[int] = [] + f = retry(always_fails(calls)) + g = retry(always_fails([]), budget=None) # explicit None opts out + with pytest.raises(ConnectionError): + f() + assert len(calls) == 1 and budget.stats.rejected == 1 + with pytest.raises(ConnectionError): + g() + assert budget.stats.requests == 1 + + +# ------------------------------------------------------------------ retry_if_result + + +def test_retry_if_result_sync(fake_clock: FakeClock, events: list[_core.Event]) -> None: + replies = iter([None, None, 7]) + seen: list[RetryState] = [] + + @retry(max_attempts=5, delay=1, retry_if_result=lambda r: r is None, on_retry=seen.append) + def poll() -> int | None: + return next(replies) + + assert poll() == 7 + assert fake_clock.sleeps == [1, 1] + assert [(s.attempt, s.exception, s.result) for s in seen] == [(1, None, None), (2, None, None)] + assert [e.name for e in events] == [ + "retry.attempt_failed", + "retry.attempt_failed", + "retry.succeeded", + ] + assert events[0].fields["exception"] is None + assert events[2].fields == {"attempts": 3} + + +def test_retry_if_result_exhausted_returns_last_value( + fake_clock: FakeClock, events: list[_core.Event], caplog: pytest.LogCaptureFixture +) -> None: + replies = iter(["busy-1", "busy-2", "busy-3"]) + seen: list[RetryState] = [] + + @retry( + max_attempts=3, + delay=0, + retry_if_result=lambda r: r.startswith("busy"), + on_retry=seen.append, + ) + def status() -> str: + return next(replies) + + with caplog.at_level(logging.WARNING, logger="python_wrap"): + assert status() == "busy-3" + assert [s.result for s in seen] == ["busy-1", "busy-2"] + assert "returned 'busy-1'" in caplog.text + assert [e.name for e in events][-1] == "retry.gave_up" + assert events[-1].fields == {"attempts": 3, "exception": None} + stats = retry_stats(status) + assert stats is not None + assert (stats.calls, stats.attempts_total, stats.retries, stats.gave_up) == (1, 3, 2, 1) + assert stats.last_exception is None + + +def test_retry_if_result_mixed_with_exceptions(fake_clock: FakeClock) -> None: + replies: Iterator[Any] = iter([ValueError("x"), -1, 5]) + + @retry(max_attempts=4, delay=0, on=ValueError, retry_if_result=lambda r: r < 0) + def f() -> int: + value = next(replies) + if isinstance(value, Exception): + raise value + return int(value) + + assert f() == 5 + assert retry_stats(f).retries == 2 # type: ignore[union-attr] + + +def test_retry_if_result_predicate_error_propagates(fake_clock: FakeClock) -> None: + def bad(_: object) -> bool: + raise RuntimeError("predicate bug") + + with pytest.raises(RuntimeError, match="predicate bug"): + retry(lambda: 1, retry_if_result=bad)() + + +def test_retry_if_result_budget_exhausted_returns_value(fake_clock: FakeClock) -> None: + budget = RetryBudget(ratio=0, min_per_second=0, window=1) + calls: list[int] = [] + + @retry(max_attempts=5, delay=0, retry_if_result=lambda r: r is None, budget=budget) + def f() -> None: + calls.append(1) + + assert f() is None + assert len(calls) == 1 + assert retry_stats(f).budget_exhausted == 1 # type: ignore[union-attr] + + +def test_retry_if_result_under_disable_retries_and_fast_mode() -> None: + calls: list[int] = [] + + @retry(max_attempts=4, delay=30, retry_if_result=lambda r: r is None) + def f() -> None: + calls.append(1) + + with fast_mode(): + assert f() is None + assert len(calls) == 4 + calls.clear() + with disable_retries(): + f() + assert len(calls) == 1 + + +def test_retry_if_result_async_sync_and_async_predicates(fake_clock: FakeClock) -> None: + async def is_none(value: object) -> bool: + await asyncio.sleep(0) + return value is None + + for predicate in (lambda v: v is None, is_none): + replies = iter([None, "ok"]) + + @retry(max_attempts=3, delay=0.25, retry_if_result=predicate) + async def f() -> str | None: + return next(replies) # noqa: B023 # called inside this iteration + + assert asyncio.run(f()) == "ok" + assert fake_clock.sleeps == [0.25, 0.25] + + +def test_retry_if_result_async_exhausted(fake_clock: FakeClock) -> None: + seen: list[RetryState] = [] + + async def note(state: RetryState) -> None: + seen.append(state) + + @retry(max_attempts=2, delay=0, retry_if_result=lambda v: v == 0, on_retry=note) + async def f() -> int: + return 0 + + assert asyncio.run(f()) == 0 + assert [s.result for s in seen] == [0] + assert retry_stats(f).gave_up == 1 # type: ignore[union-attr] + + +def test_retrying_with_result_sync(fake_clock: FakeClock, events: list[_core.Event]) -> None: + replies = iter([None, None, "done"]) + value = None + for attempt in retrying(max_attempts=5, delay=1, retry_if_result=lambda r: r is None): + with attempt: + value = next(replies) + attempt.result = value + assert value == "done" and attempt.result == "done" + assert fake_clock.sleeps == [1, 1] + assert [e.name for e in events][-1] == "retry.succeeded" + + events.clear() + numbers = [] + for attempt in retrying(max_attempts=2, delay=0, retry_if_result=lambda r: r is None): + with attempt: + numbers.append(attempt.number) + assert attempt.result is None # unset reads as None + attempt.result = None + assert numbers == [1, 2] # loop ended normally with the last (bad) value + assert [e.name for e in events] == ["retry.attempt_failed", "retry.gave_up"] + + # Not setting attempt.result means the predicate is not consulted. + for attempt in retrying(max_attempts=3, retry_if_result=lambda r: True): + with attempt: + numbers.append(0) + assert numbers[-1] == 0 and len(numbers) == 3 + + +def test_retrying_with_result_async(fake_clock: FakeClock) -> None: + async def is_none(value: object) -> bool: + return value is None + + async def main() -> list[int]: + numbers: list[int] = [] + replies = iter([None, 1]) + async for attempt in retrying(max_attempts=3, delay=0.5, retry_if_result=is_none): + async with attempt: + numbers.append(attempt.number) + attempt.result = next(replies) + replies = iter([None, None]) + async for attempt in retrying(max_attempts=2, delay=0, retry_if_result=is_none): + async with attempt: + numbers.append(attempt.number) + attempt.result = next(replies) + return numbers + + assert asyncio.run(main()) == [1, 2, 1, 2] + assert fake_clock.sleeps == [0.5, 0] + + +# ------------------------------------------------------------------------ async hooks + + +def test_async_retry_if_and_on_retry_are_awaited(fake_clock: FakeClock) -> None: + seen: list[int] = [] + + async def transient(exc: BaseException) -> bool: + await asyncio.sleep(0) + return "transient" in str(exc) + + async def note(state: RetryState) -> None: + await asyncio.sleep(0) + seen.append(state.attempt) + + errors = iter([ConnectionError("transient"), ConnectionError("transient")]) + + @retry(max_attempts=5, delay=0, retry_if=transient, on_retry=note) + async def f() -> str: + err = next(errors, None) + if err is not None: + raise err + return "ok" + + assert asyncio.run(f()) == "ok" + assert seen == [1, 2] + + @retry(max_attempts=5, delay=0, retry_if=transient) + async def fatal() -> None: + raise ConnectionError("fatal") + + with pytest.raises(ConnectionError, match="fatal"): + asyncio.run(fatal()) + assert retry_stats(fatal).attempts_total == 1 # type: ignore[union-attr] + + +def test_async_function_still_accepts_sync_hooks(fake_clock: FakeClock) -> None: + seen: list[int] = [] + errors = iter([ValueError("a")]) + + @retry(delay=0, retry_if=lambda e: True, on_retry=lambda s: seen.append(s.attempt)) + async def f() -> int: + err = next(errors, None) + if err is not None: + raise err + return 1 + + assert asyncio.run(f()) == 1 + assert seen == [1] + + +@pytest.mark.parametrize("hook", ["retry_if", "retry_if_result", "on_retry"]) +def test_sync_function_rejects_async_hooks_at_decoration(hook: str) -> None: + async def cb(_: object) -> bool: + return True + + deco = retry(**{hook: cb}) + with pytest.raises(TypeError, match=f"{hook}= is an async function"): + + @deco + def f() -> None: ... + + +def test_retrying_async_hooks_need_async_with(fake_clock: FakeClock) -> None: + async def yes(_: object) -> bool: + return True + + with pytest.raises(TypeError, match="async with attempt"): + for attempt in retrying(retry_if=yes, delay=0): + with attempt: + raise ValueError + with pytest.raises(TypeError, match="retry_if_result="): + for attempt in retrying(retry_if_result=yes, delay=0): + with attempt: + attempt.result = 1 + + async def main() -> list[int]: + seen: list[int] = [] + + async def note(state: RetryState) -> None: + seen.append(state.attempt) + + async for attempt in retrying(max_attempts=3, delay=0, retry_if=yes, on_retry=note): + async with attempt: + if attempt.number < 3: + raise ValueError + with pytest.raises(KeyError): + async for attempt in retrying(on=ValueError, retry_if=yes): + async with attempt: + raise KeyError + with pytest.raises(ValueError): + async for attempt in retrying(max_attempts=2, delay=0, retry_if=yes): + async with attempt: + raise ValueError + return seen + + assert asyncio.run(main()) == [1, 2] + + +def test_async_hooks_do_not_retry_cancellation(fake_clock: FakeClock) -> None: + async def yes(_: object) -> bool: + return True + + @retry(on=BaseException, retry_if=yes, delay=0) + async def f() -> None: + raise asyncio.CancelledError + + async def main() -> None: + with pytest.raises(asyncio.CancelledError): + await f() + with pytest.raises(asyncio.CancelledError): + async for attempt in retrying(on=BaseException, retry_if=yes): + async with attempt: + raise asyncio.CancelledError + + asyncio.run(main()) + assert retry_stats(f).attempts_total == 1 # type: ignore[union-attr] diff --git a/uv.lock b/uv.lock index 41a5a29..c78aa6c 100644 --- a/uv.lock +++ b/uv.lock @@ -1782,6 +1782,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/70/e140dffff6e808dc6343598df76e7e2407fd0f581de3524c75fba2e0cf24/nh3-0.3.7-cp38-abi3-win_arm64.whl", hash = "sha256:f04b7d333b27f13ca439da3cf1c75c2fba34f104969f6ce4ac8e7079699c2f4a", size = 621867, upload-time = "2026-08-23T14:26:29.547Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.45.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/02/6e0ae9cc61bd3169d401077b507b3ebc344745171e1051ab430be012dcd9/opentelemetry_api-1.45.1.tar.gz", hash = "sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75", size = 72804, upload-time = "2026-10-06T17:32:58.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.45.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/79/7392e21a1c8f0c61d90b223e31c7e48cb9d452e91a6b820ad24cca5f23c4/opentelemetry_sdk-1.45.1.tar.gz", hash = "sha256:63d24a6ca645019a631e6a51999c73e93adcac1196ca640b8ae78a7cc4762bf3", size = 218324, upload-time = "2026-10-06T17:33:13.26Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/3c/87c42b4bd6dd297536f04cd9383d212ac557ecd49f2cbdcd46da1c9ef5c8/opentelemetry_sdk-1.45.1-py3-none-any.whl", hash = "sha256:c604c11dc429810812348989115fa44bd558772a3d7442afc43d024f2c250ca4", size = 140063, upload-time = "2026-10-06T17:32:55.04Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.66b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/e4/dbbfb2a010c4db2224a5114638acede6fe563d33cc20fb1752cebcbe6298/opentelemetry_semantic_conventions-0.66b1.tar.gz", hash = "sha256:497ca63bf383723411e8eaf60c8779e9877633c936bb641080adab59d0eb6ec8", size = 150250, upload-time = "2026-10-06T17:33:14.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/14/67f8aa798857f8cf686f515bf93d9bb877ce952ddc8efae0fa25b45ce0d6/opentelemetry_semantic_conventions-0.66b1-py3-none-any.whl", hash = "sha256:d4cddeb4315490b35213f55e2bdc9ac54bb1e4d318927475bed62b35545e581b", size = 206279, upload-time = "2026-10-06T17:32:56.103Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -1994,6 +2033,8 @@ dev = [ { name = "hypothesis", version = "6.168.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "hypothesis", version = "6.170.12", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "mypy" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -2017,6 +2058,8 @@ release = [ dev = [ { name = "hypothesis", specifier = ">=6.100" }, { name = "mypy", specifier = ">=1.13" }, + { name = "opentelemetry-api", specifier = ">=1.45.1" }, + { name = "opentelemetry-sdk", specifier = ">=1.45.1" }, { name = "pytest", specifier = ">=8.3" }, { name = "pytest-asyncio", specifier = ">=0.24" }, { name = "pytest-cov", specifier = ">=6.0" }, @@ -2025,7 +2068,7 @@ dev = [ docs = [ { name = "mkdocs-material", specifier = ">=9.5" }, { name = "mkdocs-redirects", specifier = ">=1.2.3" }, - { name = "mkdocstrings", extras = ["python"], specifier = ">=0.27" }, + { name = "mkdocstrings", extras = ["python"], specifier = ">=1.0.6" }, ] release = [ { name = "build", specifier = ">=1.2" },