From 2b2d114eab01b972302240f144a70ffa4310930d Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:53:51 -0800 Subject: [PATCH 01/23] feat: native engine by default with Python fallback (4.10.0) Resolve backend through 'auto': the native engine when datafog-core imports, the pure-Python engine otherwise with a single warning. DATAFOG_BACKEND and backend="python" force the fallback; non-regex engines stay on the Python ML path. Serve prose PERSON and ADDRESS from the embedded model in datafog-core, so names and addresses need no spaCy, GLiNER, download or interpreter. Expose the new credential, account and UUID entity types through the alias API and add datafog.native for the native schema. Deprecation warnings now name 4.10.0 as the last supported version with an exact pin, and VERSIONING.md plus docs/retirement.rst make the window and the removal schedule auditable. Nothing is removed in this release. --- CHANGELOG.MD | 26 +++ README.md | 53 +++-- RELEASE_NOTES_4.10.0.md | 61 ++++++ VERSIONING.md | 123 +++++++++++ datafog/__about__.py | 2 +- datafog/__init__.py | 16 +- datafog/_core_capabilities.py | 13 +- datafog/_legacy_retirement.py | 6 +- datafog/compat/v4.py | 15 +- datafog/engine.py | 194 ++++++++++++++++-- datafog/exceptions.py | 4 + datafog/native.py | 60 ++++++ docs/cli.rst | 3 +- docs/important-concepts.rst | 2 +- docs/index.rst | 28 ++- docs/migration-4.10.md | 92 +++++++++ docs/migration-4.9.md | 5 +- docs/optional-surfaces.rst | 3 +- docs/python-sdk.rst | 20 +- docs/retirement.rst | 115 +++++++++++ scripts/check_rust_install.py | 10 +- setup.py | 8 +- .../{rust-0.4.0.json => rust-1.2.0.json} | 17 +- tests/rust_contract.py | 2 +- tests/test_allowlist.py | 14 +- tests/test_api_bridge_49.py | 2 +- tests/test_backend_resolution.py | 111 ++++++++++ tests/test_contract_481.py | 17 +- tests/test_core04_integration.py | 38 ++-- tests/test_de_pii_regex.py | 17 +- tests/test_legacy_retirement.py | 11 +- tests/test_rust_backend.py | 15 +- tests/test_rust_contract.py | 4 +- 33 files changed, 999 insertions(+), 108 deletions(-) create mode 100644 RELEASE_NOTES_4.10.0.md create mode 100644 VERSIONING.md create mode 100644 datafog/native.py create mode 100644 docs/migration-4.10.md create mode 100644 docs/retirement.rst rename tests/contracts/{rust-0.4.0.json => rust-1.2.0.json} (70%) create mode 100644 tests/test_backend_resolution.py diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 576597a3..1049239e 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,5 +1,31 @@ # ChangeLog +## [4.10.0] + +#### Added + +- Native engine is the default: `backend` resolves through `auto`, which uses + `datafog-core` when it imports and the pure-Python engine otherwise. One + warning on fallback, `DATAFOG_BACKEND` and `backend="python"` to opt out. +- Prose `PERSON` and `ADDRESS` from the model embedded in `datafog-core`; no + spaCy, GLiNER, download or interpreter. +- `API_KEY`, `BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, + `ACCOUNT_NUMBER`, `US_ROUTING_NUMBER`, `NPI` and `UUID` entity types, plus the + German set behind the `de` locale. +- `datafog.native` native schema access and `VERSIONING.md`. + +#### Changed + +- `datafog-core>=1.2.0,<2` is a normal dependency; `[rust]` remains as an + explicit spelling. +- `Development Status: 5 - Production/Stable`. +- Retirement warnings name `4.10.0` as the last supported version with an exact + pin, replacing "the final 4.x release". + +#### Fixed + +- `redact(..., entities=[...])` validates `backend` instead of ignoring it. + ## [4.9.0] #### Added diff --git a/README.md b/README.md index 4fd354a3..0f70fb23 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,9 @@ values never echoed into logs or transcripts: - **Claude Code hook** (`datafog-hook`): gates agent tool calls (shell commands, web requests, file writes, MCP tools) and warns the model when - prompts or tool results carry PII. ~70–90ms per invocation including - process startup. Easiest install is the + prompts or tool results carry PII. ~105–110ms per invocation including + process startup and the native engine load (`DATAFOG_BACKEND=python` trims + ~20ms at the cost of detection coverage). Easiest install is the [Claude Code plugin](https://github.com/DataFog/datafog-claude-plugin): ``` @@ -32,8 +33,8 @@ values never echoed into logs or transcripts: - **LiteLLM guardrail** (`DataFogGuardrail`): redacts or blocks PII in requests and responses at the gateway, for any LiteLLM-proxied provider. - In-process (~40µs per message scanned; a request clears the guardrail in - well under a millisecond), no sidecar service. Setup: + In-process (tens of microseconds per message scanned on the native engine), + no sidecar service. Setup: [examples/litellm_guardrail/](examples/litellm_guardrail/). Both default to the high-precision entity set (`EMAIL`, `PHONE`, @@ -51,7 +52,7 @@ NER live in [benchmarks/](benchmarks/). ## Installation ```bash -# Core install (regex engine) +# Core install: native engine, no extra packages needed pip install datafog # Add spaCy support @@ -70,9 +71,33 @@ pip install datafog[distributed] pip install datafog[all] ``` +`pip install datafog` pulls `datafog-core`, which is the native engine. Wheels +exist for Linux (glibc and musl, x86_64/aarch64/s390x/ppc64le), macOS and +Windows (amd64/arm64); anywhere else the source build needs a Rust toolchain. +If Core cannot be imported, detection falls back to the pure-Python engine and +warns once — `DATAFOG_BACKEND=python` selects that path deliberately. + The `nlp-advanced` extra includes SentencePiece and protobuf for GLiNER's multilingual tokenizer; installing the OCR extra is not required for GLiNER. +## Backends + +`backend` defaults to `auto`: the native engine when `datafog-core` is +importable, the pure-Python regex engine otherwise. `DATAFOG_BACKEND` sets the +default and `backend="python"` / `backend="rust"` override it per call. +`auto` never selects the native engine for `engine="spacy"`, `"gliner"` or +`"smart"`, which stay on the Python ML path. + +The native engine adds detectors the Python engine never had — `API_KEY`, +`BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, +`US_ROUTING_NUMBER`, `NPI`, `UUID` and a German set — and serves prose `PERSON` +and `ADDRESS` from an embedded model, so name and address detection needs no +spaCy and no model download. Findings keep `engine="regex"` provenance and +`confidence=1.0` as compatibility values, not native confidence estimates. +Reviewed differences from the Python engine are listed in +[tests/contracts/rust-1.2.0.json](tests/contracts/rust-1.2.0.json). +`datafog.native` exposes the native schema directly. + Python 3.13 support is certified for the core SDK, CLI, `nlp`, `nlp-advanced`, and `ocr` install profiles. Donut OCR still requires a model that is available locally before runtime use. `distributed` and `all` remain @@ -188,10 +213,12 @@ scan/redact helpers, or guardrail helpers. model. - A Java runtime is required by PySpark. -DataFog 4.9.0 deprecates OCR and Spark with visible use-time -warnings; their APIs and extras will be removed in 5.0. They remain functional -in 4.9. Users who need these features can stay on the final 4.x release. See -the [4.9 migration guide](docs/migration-4.9.md) for the transition plan. +DataFog deprecates OCR and Spark with visible use-time +warnings; their APIs and extras will be removed in 5.0. The last supported +version is 4.10.0 — pin it with `pip install "datafog==4.10.0"` — and every +retired surface is tracked in the [retirement schedule](docs/retirement.rst). +See the [4.9 migration guide](docs/migration-4.9.md) for the transition plan, +and [VERSIONING.md](VERSIONING.md) for what a version number promises. ## Backward-Compatible APIs @@ -261,11 +288,9 @@ The [4.9 migration guide](docs/migration-4.9.md) explains opt-in Rust detection, the native `datafog.v5` preview, and the revised 5.0 retirement schedule for `detect`/`process`, OCR, and Spark. The Python detector remains the default. -The experimental Rust adapter in 4.9.0 requires Core `>=0.4.0,<0.5` and capability -contract 1. Upgrade with `python -m pip install --upgrade "datafog[rust]==4.9.0"` -to evaluate it. Base-only users can install `datafog==4.9.0` without Core; -installing the Rust extra does not change the default backend. See the -[4.9.0 release notes](RELEASE_NOTES_4.9.0.md). +4.10.0 requires Core `>=1.2.0,<2` and capability contract 1, and installs it by +default. See the [4.10.0 release notes](RELEASE_NOTES_4.10.0.md) and the +[4.10 migration guide](docs/migration-4.10.md). Entity labels, locales, and activation settings come from the installed Core, allowing compatible releases to add detectors without a Python update. German detection is opt-in through locale or entity selection; UUID is opt-in diff --git a/RELEASE_NOTES_4.10.0.md b/RELEASE_NOTES_4.10.0.md new file mode 100644 index 00000000..f5c592d4 --- /dev/null +++ b/RELEASE_NOTES_4.10.0.md @@ -0,0 +1,61 @@ +# DataFog Python 4.10.0 + +4.10.0 turns on the native engine. `pip install datafog` installs `datafog-core` +and detection uses it. Nothing is removed in this release and no call changes +shape; `backend="python"` restores the previous behaviour exactly. + +Read the [4.10 migration guide](docs/migration-4.10.md) before upgrading a +pipeline whose output is under test. + +## Upgrade + +```bash +python -m pip install --upgrade "datafog==4.10.0" +``` + +Wheels cover Linux (glibc and musl, x86_64/aarch64/s390x/ppc64le), macOS and +Windows (amd64/arm64). Elsewhere the source build needs Rust 1.88+. If Core +cannot be imported, detection falls back to the pure-Python engine and warns +once. + +## What changes + +- `backend` defaults to `auto`: native when `datafog-core` imports, Python + otherwise. `DATAFOG_BACKEND` sets the default; `backend="python"` forces the + fallback. Non-regex engines stay on the Python path. +- `PERSON` and `ADDRESS` for prose now come from an embedded model compiled into + `datafog-core` — no spaCy, no GLiNER, no model download, no interpreter. +- The native engine adds `API_KEY`, `BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, + `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, `US_ROUTING_NUMBER`, `NPI`, `UUID` and the + German identifiers. Credential detectors are on by default; German and `UUID` + stay opt-in. +- `datafog.native` exposes the native schema, `capabilities()`, the embedded + detector and version reporting. Experimental until 5.0. +- `datafog-core` is pinned `>=1.2.0,<2`. Detection output is contract-versioned. + +## Detection differences + +Four reviewed differences from the Python engine are recorded in +[tests/contracts/rust-1.2.0.json](tests/contracts/rust-1.2.0.json) and asserted +by the suite: Core validates card checksums, requires non-alphanumeric SSN +boundaries, requires a North American area code starting 2-9 (which is what +keeps Unix timestamps out of `PHONE`), and rejects unadvertised locales instead +of ignoring them. + +`(555) 123-4567` and similar form data now match: Core keeps the area-code check +and accepts exchange codes starting with 1. + +## Deprecations unchanged + +`detect()`, `process()`, OCR and Spark still work and still warn; all are removed +in 5.0. Their messages now name `4.10.0` as the last supported version and give +the exact pin. See the [retirement schedule](docs/retirement.rst). + +## Notes + +- `VERSIONING.md` is new and defines what a version number promises here, with a + deprecation window, a detection-output rule and a support window. +- Hook invocations cost ~105–110ms including process startup, of which ~20ms is + the native engine load. `DATAFOG_BACKEND=python` trades that back for coverage. +- `datafog[rust]` remains as an explicit spelling of a dependency the base + install already has. diff --git a/VERSIONING.md b/VERSIONING.md new file mode 100644 index 00000000..9f511c41 --- /dev/null +++ b/VERSIONING.md @@ -0,0 +1,123 @@ +# Versioning and compatibility policy + +DataFog Python follows [Semantic Versioning 2.0.0](https://semver.org/spec/v2.0.0.html). +This document defines what that means here, so a version number is a promise +rather than a guess. + +DataFog versions independently of `datafog-core`. See +[Independent versioning](#independent-versioning). + +## What the numbers mean + +- **MAJOR** (`5.0.0`) — a public API is removed or renamed, an import path moves, + or a default changes such that working caller code stops working. +- **MINOR** (`4.10.0`) — new capability. New entities, new backends, new extras, + new commands, new options, and **changes to detection output**. +- **PATCH** (`4.10.1`) — fixes that do not change findings for a given input, + configuration, and locked dependency set. + +### Detection output is a minor change, never a patch + +The finding set is part of the contract. A patch release must not start finding +more email addresses, or fewer, for input that already worked. If a fix changes +what is found, it ships as a MINOR and the changelog says which entities moved +and in which direction. People write golden files against this output. + +### Dependency-default changes are minor + +Raising a floor, dropping support for a Python version, or changing which +backend is used by default is a MINOR. It is never a PATCH. + +## Deprecation window + +A public API is deprecated in a MINOR, warning at the call boundary, and removed +no earlier than the second following MINOR or 90 days, whichever is longer. + +A deprecation warning must name: + +1. the exact last version that supports the surface, as an installable pin + (`pip install "datafog==4.10.0"`), and +2. the MAJOR that removes it. + +Warnings fire at the public call boundary, before any optional import, so a plain +`import datafog` never pays for a deprecated surface. + +At the removal MAJOR, the removed name stays present as a stub raising +`ImportError` with the same pin, rather than disappearing into an +`AttributeError`. + +## Experimental surface + +Experimental code is marked as such in its documentation and docstrings. It may +break in a MINOR without a deprecation cycle. Anything unmarked is covered by +this policy. + +Currently experimental: + +- `datafog.v5` (native Core preview) +- `datafog.core` (from 4.10.0) +- GLiNER2 and other model sessions +- Browser/WASM entry points + +## Scope + +Covered by this policy: + +- top-level `datafog` callables listed in `datafog.__all__` +- `datafog.compat.v4` and its result classes +- the `datafog` CLI and the `datafog-hook` entry point +- `DataFogGuardrail` and the Claude Code hook integration +- documented configuration keys and environment variables +- the finding set for a given engine, entity selection, and locale + +Not covered: + +- private modules and names beginning with `_` +- `main_lean.py`, `main_original.py`, `__init___original.py` and other + transitional duplicates +- logging text, warning text, and internal error strings +- model accuracy targets for optional ML engines + +## Support window + +The current MAJOR receives fixes for its latest two MINOR releases. Older MINORs +receive security fixes only. Every published version stays installable on PyPI +permanently; a retired surface has a final supported version documented in +[`docs/retirement.rst`](docs/retirement.rst), and that version is the answer. + +## Independent versioning + +`datafog` and `datafog-core` are separate distributions on separate tracks. +`datafog-core` carries the engine; `datafog` carries the application surface. +The transport between them is `capabilities()` and its `contract_version`. +Bumping either `contract_version` or `workspace_contract_version` is MAJOR work +for `datafog-core`, and requires a matching floor in `datafog`. + +Both are exposed separately so a bug report is attributable: + +```python +import datafog, datafog.core +datafog.__version__ # application version +datafog.core.__version__ # engine version +``` + +## Yanking + +A broken PATCH may be yanked on PyPI and replaced by the next PATCH. A broken +MINOR or MAJOR is superseded, not yanked, because dependents are often already +pinned to it; the changelog opens with the replacement version. + +## Enforcement + +- `RELEASE_NOTES_.md` is required for stable releases. +- The changelog classifies every entry as Added / Changed / Deprecated / Removed + / Fixed and names the version at which each deprecation ends. +- `scripts/check_rust_install.py` and the parity suite run against the installed + wheel, not the checkout. + +## History + +Before 4.10.0 this policy was not written down, and the versioning record shows +it: the 5.0 retirement plan was revised mid-stream (commit `5f5bdbb3`), and the +4.9.0 final publish preceded its own `a1` and `a2` tags. 4.10.0 is the first +release prepared under this policy. diff --git a/datafog/__about__.py b/datafog/__about__.py index 21d77306..897e6be2 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.9.0a2" +__version__ = "4.10.0" diff --git a/datafog/__init__.py b/datafog/__init__.py index 591e0ef4..12ecc115 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -11,6 +11,7 @@ import warnings +from ._legacy_retirement import LAST_SUPPORTED as _LAST_SUPPORTED from .__about__ import __version__ from .agent import create_guardrail, filter_output, sanitize, scan_prompt @@ -81,6 +82,13 @@ def __getattr__(name: str): elif name in _LAZY_EXPORTS: module_path, attr_name, extra_name = _LAZY_EXPORTS[name] + if attr_name is None: + # Submodule export (``datafog.native``): return the module itself. + import importlib + + value = importlib.import_module(module_path) + globals()[name] = value + return value try: module = __import__(module_path, fromlist=[attr_name]) value = getattr(module, attr_name) @@ -104,6 +112,7 @@ def _missing_dependency(*args, **kwargs): _LAZY_EXPORTS = { + "native": ("datafog.native", None, None), "app": ("datafog.client", "app", None), "DataFog": ("datafog.main", "DataFog", None), "TextPIIAnnotator": ("datafog.main", "TextPIIAnnotator", None), @@ -131,7 +140,8 @@ def _missing_dependency(*args, **kwargs): def _warn_v5_replacement(old_api: str, replacement: str) -> None: warnings.warn( f"datafog.{old_api}() is deprecated and will be removed in 5.0. Use {replacement} instead. " - "The earlier promise to retain this shim through 5.x has been revised.", + f'Last supported version is {_LAST_SUPPORTED}. Pin it with: pip install "datafog=={_LAST_SUPPORTED}". ' + "The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst.", FutureWarning, stacklevel=3, ) @@ -145,7 +155,7 @@ def scan( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> ScanResult: """Scan using the 4.x compatibility API; Rust detection is opt-in.""" return _compat_v4.scan( @@ -170,7 +180,7 @@ def redact( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> RedactResult: """Redact using the 4.x compatibility API and legacy strategies.""" return _compat_v4.redact( diff --git a/datafog/_core_capabilities.py b/datafog/_core_capabilities.py index a5a0df4b..77cd6c9d 100644 --- a/datafog/_core_capabilities.py +++ b/datafog/_core_capabilities.py @@ -85,7 +85,7 @@ def _requested_locales(locales: Any, lookup: dict[str, str]) -> set[str]: return {_resolve_locale(locale, lookup) for locale in locales or []} -def _activation_config(label: str, metadata: dict) -> dict: +def _activation_config(label: str, metadata: dict) -> dict | None: details = metadata.get(label) if not isinstance(details, dict): raise _invalid(f"missing entity metadata for {label!r}") @@ -95,10 +95,9 @@ def _activation_config(label: str, metadata: dict) -> dict: ): raise _invalid(f"invalid scopes for {label!r}") if "text" not in scopes: - raise ValueError( - f"Entity {label!r} is not available for Rust text scanning " - "(structured-only entity)" - ) + # Structured-only labels are served by the caller, usually from the + # embedded model, so a mixed selection is not an error. + return None activation = details.get("activation") if not isinstance(activation, dict): raise _invalid(f"missing activation metadata for {label!r}") @@ -120,13 +119,15 @@ def scan_configs(core: Any, requested: set[str], locales: Any) -> list[dict]: capabilities = getattr(core, "capabilities", None) if not callable(capabilities): raise _invalid( - "contract version 1 is required; install a compatible Core 0.4.x" + "contract version 1 is required; install a compatible datafog-core" ) supported, lookup, metadata = _read_capabilities(capabilities) selected_locales = _requested_locales(locales, lookup) common: dict[str, Any] = {} for label in sorted(requested & supported): config = _activation_config(label, metadata) + if config is None: + continue for key, value in config.items(): if key == "locale": selected_locales.add(_resolve_locale(value, lookup)) diff --git a/datafog/_legacy_retirement.py b/datafog/_legacy_retirement.py index 5bc96c13..e86506db 100644 --- a/datafog/_legacy_retirement.py +++ b/datafog/_legacy_retirement.py @@ -2,6 +2,9 @@ import warnings +#: Final release on which retired optional surfaces still work. +LAST_SUPPORTED = "4.10.0" + class LegacySurfaceWarning(FutureWarning): """An optional DataFog surface scheduled for removal in 5.0.""" @@ -11,7 +14,8 @@ def warn_legacy_surface(surface: str) -> None: """Warn at the caller's public API boundary, before optional imports.""" warnings.warn( f"DataFog {surface} support is deprecated in 4.9 and will be removed in 5.0. " - "Remain on the final 4.x release if you need continued support.", + f'Last supported version is {LAST_SUPPORTED}. Pin it with: pip install "datafog=={LAST_SUPPORTED}". ' + "See docs/retirement.rst.", LegacySurfaceWarning, stacklevel=3, ) diff --git a/datafog/compat/v4.py b/datafog/compat/v4.py index 0be6b4bb..49d32e2c 100644 --- a/datafog/compat/v4.py +++ b/datafog/compat/v4.py @@ -5,6 +5,7 @@ """ from ..engine import Entity, RedactResult, ScanResult +from ..engine import resolve_backend from ..engine import redact as _redact_entities from ..engine import scan as _scan from ..engine import scan_and_redact as _scan_and_redact @@ -29,13 +30,13 @@ def scan( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> ScanResult: """ - Scan with the legacy result schema; Rust detection is opt-in. + Scan with the legacy result schema. - Defaults to the lightweight regex engine so the core install works without - optional dependency fallback warnings. + Backend selection follows ``datafog.engine.resolve_backend``: native when + ``datafog-core`` is importable, otherwise Python with a one-time warning. ``allowlist`` exempts exact entity texts (your own support address, doc placeholders); ``allowlist_patterns`` exempts entities whose full text @@ -64,7 +65,7 @@ def redact( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> RedactResult: """ Redact with legacy strategies and result schema. @@ -75,8 +76,8 @@ def redact( full-text regex match respectively); they apply to the scan path and are rejected when explicit ``entities`` are supplied. """ - if backend not in ("python", "rust"): - raise ValueError("backend must be one of: python, rust") + # Validate the backend even when explicit entities skip the scan path. + resolve_backend(backend, engine) if preset is not None: try: diff --git a/datafog/engine.py b/datafog/engine.py index 0e2a0fb4..fa6ad926 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -3,13 +3,14 @@ from __future__ import annotations import hashlib +import os import re import warnings from dataclasses import dataclass from functools import lru_cache from typing import Optional -from .exceptions import EngineNotAvailable +from .exceptions import CoreUnavailableWarning, EngineNotAvailable from .processing.text_processing.regex_annotator import RegexAnnotator CANONICAL_TYPE_MAP = { @@ -27,6 +28,13 @@ # Presidio-compatible aliases, so configs migrate without renames. "EMAIL_ADDRESS": "EMAIL", "US_SSN": "SSN", + # Credential detectors are native-only; these spellings keep one vocabulary. + "APIKEY": "API_KEY", + "API-KEY": "API_KEY", + "TOKEN": "BEARER_TOKEN", + "BEARER": "BEARER_TOKEN", + "PRIVATEKEY": "PRIVATE_KEY", + "CREDENTIAL_URI": "CREDENTIAL_URI", } ALL_ENTITY_TYPES = { @@ -48,8 +56,28 @@ "ORGANIZATION", "LOCATION", "ADDRESS", + "API_KEY", + "BEARER_TOKEN", + "JWT", + "PRIVATE_KEY", + "CREDENTIAL_URI", + "ACCOUNT_NUMBER", + "STREET_ADDRESS", + "US_ROUTING_NUMBER", + "NPI", + "UUID", } +#: Labels the native rule engine reports as structured-only, served for text by +#: the embedded model instead. +MODEL_ENTITY_TYPES = {"PERSON", "ADDRESS"} + +#: Environment override for the default backend: ``auto``, ``python`` or ``rust``. +BACKEND_ENV = "DATAFOG_BACKEND" + +_core_warned = False +_cnn_cache: tuple[object, object] | None = None + NER_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} ENTITY_TYPE_PRIORITY = { @@ -209,6 +237,81 @@ def _regex_entities( return _suppress_overlapping_entities(entities) +def _core_module_or_none() -> object | None: + """Return the importable ``datafog_core`` module, or ``None``. + + Deliberately uncached: after the first import this is a ``sys.modules`` + lookup, and not caching keeps ``sys.modules`` patching in tests honest. + """ + try: + import datafog_core + except ImportError: + return None + return datafog_core + + +def _require_core() -> object: + core = _core_module_or_none() + if core is None: + raise ImportError( + "The Rust backend requires datafog-core. " + 'Install with: pip install "datafog[rust]"' + ) + return core + + +def core_available() -> bool: + """Whether the native engine can serve this interpreter.""" + return _core_module_or_none() is not None + + +def _set_core(module: object | None) -> None: + """Test seam: reset cached state derived from the core module.""" + global _cnn_cache + _cnn_cache = None + + +def _cnn_detector_or_none() -> object | None: + """Cached embedded detector, keyed on the module object it came from.""" + global _cnn_cache + core = _core_module_or_none() + if core is None: + return None + if _cnn_cache is not None and _cnn_cache[0] is core: + return _cnn_cache[1] + factory = getattr(core, "EmbeddedCnnDetector", None) + if factory is None: + return None + detector = factory() + _cnn_cache = (core, detector) + return detector + + +def _cnn_findings( + detector: object, requested: set[str], text: str +) -> list[tuple[dict, object]]: + """Serve prose names and address blocks from the embedded model.""" + try: + result = detector.scan_model(text) + except Exception as exc: # relative-path/limit errors surface to the caller + raise _model_error(exc) from exc + return [ + ({}, finding) + for finding in result.findings + if finding.entity_type in requested + ] + + +def _model_error(exc: Exception) -> Exception: + """Re-raise a model failure as a ValueError with its native code attached.""" + code = getattr(exc, "code", "model_error") + reason = getattr(exc, "reason", None) + detail = f"{code}/{reason}" if reason else code + error = ValueError(f"Native model detection failed ({detail}): {exc}") + error.code = code # type: ignore[attr-defined] + return error + + def _rust_entities( text: str, entity_types: Optional[list[str]] = None, @@ -220,22 +323,30 @@ def _rust_entities( suppression before entity selection and allowlists, preserving that order without duplicating the native detectors. """ - try: - import datafog_core - except ImportError as exc: - raise ImportError( - "The Rust backend requires datafog-core. " - 'Install with: pip install "datafog[rust]"' - ) from exc + datafog_core = _require_core() from ._core_capabilities import scan_configs requested = {_canonical_type(value) for value in entity_types or []} - configs = scan_configs(datafog_core, requested, locales) + model_requested = requested & MODEL_ENTITY_TYPES + detector = _cnn_detector_or_none() if model_requested else None + if model_requested and detector is None: + raise ValueError( + f"Entity {', '.join(sorted(model_requested))} needs the embedded model, " + "which this datafog-core build does not provide. Upgrade it with: " + 'pip install --upgrade "datafog-core"' + ) + configs = scan_configs(datafog_core, requested - MODEL_ENTITY_TYPES, locales) entities: list[Entity] = [] seen: set[tuple[str, int, int]] = set() - for config in configs: - for finding in datafog_core.scan(text, config=config): + findings = [ + (config, finding) + for config in configs + for finding in datafog_core.scan(text, config=config) + ] + if detector is not None: + findings.extend(_cnn_findings(detector, model_requested, text)) + for config, finding in findings: canonical_type = _canonical_type(finding.entity_type) start = finding.codepoint_range.start end = finding.codepoint_range.end @@ -404,6 +515,42 @@ def _needs_ner(entity_types: Optional[list[str]]) -> bool: return bool(requested & NER_ENTITY_TYPES) +def _warn_core_missing() -> None: + global _core_warned + if _core_warned: + return + _core_warned = True + warnings.warn( + "datafog-core is not installed, so detection falls back to the pure-Python " + f"engine. Install it to use the native engine, or set {BACKEND_ENV}=python " + "to silence this warning.", + CoreUnavailableWarning, + stacklevel=4, + ) + + +def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: + """Resolve ``auto`` to a concrete backend without importing Core eagerly. + + An explicit ``backend`` wins over ``DATAFOG_BACKEND``. ``auto`` uses the + native engine whenever Core is importable and degrades to Python otherwise. + Non-``regex`` engines always take the Python path, because the native + backend serves ``regex`` and model findings only. + """ + requested = backend or os.environ.get(BACKEND_ENV) or "auto" + normalized = requested.strip().lower() + if normalized not in {"auto", "python", "rust"}: + raise ValueError( + f"backend must be one of: auto, python, rust (from {BACKEND_ENV} or the " + "backend argument)" + ) + if normalized == "auto": + if engine != "regex": + return "python" + return "rust" if core_available() else "python" + return normalized + + def _validate_backend(backend: str, engine: str) -> None: if backend not in {"python", "rust"}: raise ValueError("backend must be one of: python, rust") @@ -419,13 +566,19 @@ def scan( allowlist: Optional[list[str]] = None, allowlist_patterns: Optional[list[str]] = None, *, - backend: str = "python", + backend: Optional[str] = None, ) -> ScanResult: """Scan text for PII entities. - ``backend="rust"`` opts into experimental native detection for - ``engine="regex"``. Results retain legacy regex provenance and confidence; - these are compatibility values, not native confidence estimates. + ``backend`` defaults to ``auto``: the native engine when ``datafog-core`` is + importable, otherwise the pure-Python engine with a one-time warning. Set + ``DATAFOG_BACKEND`` or pass ``backend="python"`` to force the fallback. The + native path serves ``engine="regex"``; non-regex engines always use Python. + Probe names and address blocks come from the embedded model on the native + path, so ``PERSON`` and ``ADDRESS`` work without spaCy or a model download. + + Results retain legacy regex provenance and confidence; these are + compatibility values, not native confidence estimates. ``allowlist`` exempts exact entity texts (e.g. your own support email); ``allowlist_patterns`` exempts entities whose full text matches a regex @@ -437,7 +590,11 @@ def scan( if engine not in {"regex", "spacy", "gliner", "smart"}: raise ValueError("engine must be one of: regex, spacy, gliner, smart") + requested_backend = (backend or os.environ.get(BACKEND_ENV) or "auto").strip().lower() + backend = resolve_backend(backend, engine) _validate_backend(backend, engine) + if backend == "python" and requested_backend == "auto" and not core_available(): + _warn_core_missing() # Validate patterns up front so config errors fail fast even when the # text contains no entities. @@ -611,9 +768,12 @@ def scan_and_redact( allowlist: Optional[list[str]] = None, allowlist_patterns: Optional[list[str]] = None, *, - backend: str = "python", + backend: Optional[str] = None, ) -> RedactResult: - """Scan with the selected backend, then apply legacy Python redaction.""" + """Scan with the selected backend, then apply legacy Python redaction. + + Backend selection matches :func:`scan`. + """ scan_result = scan( text=text, engine=engine, diff --git a/datafog/exceptions.py b/datafog/exceptions.py index 98bc8d0d..7ac32340 100644 --- a/datafog/exceptions.py +++ b/datafog/exceptions.py @@ -70,6 +70,10 @@ def __init__(self, message: str): super().__init__(message, status_code=None) +class CoreUnavailableWarning(RuntimeWarning): + """Emitted once when ``backend="auto"`` falls back to the pure-Python engine.""" + + def raise_for_status_code(status_code: int, error_message: str): """ Raise the appropriate exception based on the status code. diff --git a/datafog/native.py b/datafog/native.py new file mode 100644 index 00000000..cde64934 --- /dev/null +++ b/datafog/native.py @@ -0,0 +1,60 @@ +"""Native detection API, re-exported from the installed ``datafog-core``. + +``datafog`` selects the native engine by default, so most callers never need +this module. Import it to reach the native schema directly: byte and code-point +ranges, transformation records, provider-backed tokenization, structured scans, +the embedded model, and ``capabilities()``. + +The names are re-exports, not a second implementation. Importing ``datafog`` +alone does not load them. + +Experimental: this module may change outside the deprecation policy until 5.0. +""" + +from __future__ import annotations + +import importlib +import importlib.metadata +from typing import Any + +_MODULE = "datafog_core" + +__all__ = [ + "EmbeddedCnnDetector", + "Finding", + "ModelScanResult", + "capabilities", + "contract_version", + "scan", + "scan_and_transform", + "scan_structured", + "transform", + "transform_structured", + "version", +] + + +def _core() -> Any: + try: + return importlib.import_module(_MODULE) + except ImportError as exc: # pragma: no cover - base install always has it + raise ImportError( + "datafog.native requires datafog-core. Install with: " + 'pip install "datafog==4.10.0"' + ) from exc + + +def __getattr__(name: str) -> Any: + if name in __all__: + return getattr(_core(), name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def version() -> str: + """Installed Core version, for bug reports and reproducibility locks.""" + return importlib.metadata.version("datafog-core") + + +def contract_version() -> int: + """Capability contract the installed Core speaks. Currently 1.""" + return int(_core().capabilities()["contract_version"]) diff --git a/docs/cli.rst b/docs/cli.rst index b07a080a..b704508b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -24,7 +24,8 @@ dependencies are: Spark/distributed workflows are Python SDK surfaces rather than first-path CLI commands. Spark support is also deprecated in 4.9 for removal in 5.0. Install ``datafog[distributed]`` when using ``SparkService`` during 4.x. Users needing -OCR/Spark after the cutover can remain on the final 4.x release. See +OCR/Spark after the cutover should pin ``datafog==4.10.0``, the last supported +version. See :doc:`retirement` and :doc:`optional-surfaces` and the :download:`4.9 migration guide `. diff --git a/docs/important-concepts.rst b/docs/important-concepts.rst index 239471bc..4f48c39d 100644 --- a/docs/important-concepts.rst +++ b/docs/important-concepts.rst @@ -39,7 +39,7 @@ Services ``TextService`` remains available. ``ImageService`` and ``SparkService`` are optional legacy services, deprecated in 4.9 for removal in 5.0. They remain functional throughout 4.9; users requiring them after the cutover can remain on -the final 4.x release. See :doc:`optional-surfaces`. +:doc:`retirement`. Existing services: diff --git a/docs/index.rst b/docs/index.rst index df61a539..bb9b3d9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,13 +11,30 @@ Start with :doc:`getting-started` if you want the shortest route from install to scanning text. The roadmap and historical planning pages remain available, but the live user docs are the first path for current text APIs. +4.10.0 native default +===================== + +DataFog 4.10.0 turns the native engine on by default. ``pip install datafog`` +now installs ``datafog-core``, and detection uses it whenever it imports; the +pure-Python engine remains the fallback and ``DATAFOG_BACKEND=python`` forces +it. No public API is removed in 4.10.0 and no existing call changes shape — the +``backend`` keyword is still accepted, and now also accepts ``auto``. + +:download:`Complete 4.10 migration guide `. + +Prose ``PERSON`` and ``ADDRESS`` detection now comes from an embedded model, so +names and addresses no longer need spaCy or a model download. The native engine +adds credential, account and German detectors. Reviewed differences from the +Python engine are recorded in ``tests/contracts/rust-1.2.0.json`` and the +migration guide below. + 4.9.0 migration bridge ====================== -DataFog 4.9.0 is the migration bridge to the planned Rust-backed 5.0 API. -Upgrade the base package with ``python -m pip install --upgrade datafog==4.9.0``; -the native dependency remains optional. The ``rust`` extra requires -``datafog-core>=0.4.0,<0.5`` and capability contract version 1. +DataFog 4.9.0 was the migration bridge to the Rust-backed 5.0 API. It added the +explicit ``backend="rust"`` opt-in, the ``datafog.compat.v4`` facade and the +``datafog.v5`` preview. 4.10.0 built on it by making the native path the +default. 4.9 preserves existing imports, result classes, Python detection defaults, and redaction behavior. It adds explicit experimental Rust detection through @@ -28,7 +45,7 @@ Core schema preview. See :doc:`python-sdk` and the DataFog 4.9.0 deprecates ``detect()``/``process()``, OCR, and Spark for removal in 5.0. The earlier promise to retain ``detect()``/``process()`` throughout 5.x is revised. OCR/Spark remain functional in 4.9; users needing them after the -cutover can remain on the final 4.x release. See :doc:`optional-surfaces`. +cutover can pin :doc:`retirement`. Use DataFog =========== @@ -42,6 +59,7 @@ Use DataFog cli optional-surfaces important-concepts + retirement Reference ========= diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md new file mode 100644 index 00000000..cbec85de --- /dev/null +++ b/docs/migration-4.10.md @@ -0,0 +1,92 @@ +# Migrating to DataFog 4.10.0 + +4.10.0 turns on the native engine. Nothing is removed; most callers change +nothing at all. This page covers what a version number cannot: what you will +notice, and what to do if you do not want it. + +```bash +python -m pip install --upgrade "datafog==4.10.0" +``` + +`datafog-core` is now a normal dependency, so this pulls a platform wheel. On +Linux (glibc or musl, x86_64/aarch64/s390x/ppc64le), macOS and Windows +(amd64/arm64) that is a wheel. Elsewhere it builds from source and needs Rust +1.88+. If Core is missing at runtime, detection falls back to the Python engine +and warns once instead of failing. + +## What changes + +`backend` defaults to `auto`. `auto` selects the native engine when Core +imports, and the Python engine otherwise. `backend="python"` and +`backend="rust"` work exactly as before. `DATAFOG_BACKEND` sets the default +without touching code: + +```bash +DATAFOG_BACKEND=python # always the pure-Python engine, no warning +DATAFOG_BACKEND=auto # the default +``` + +`auto` never picks the native engine for `engine="spacy"`, `"gliner"` or +`"smart"`, because the native path serves `engine="regex"` plus embedded-model +findings. Those engines still run on the Python ML stack and still need their +extras. + +## Detection differences + +The native engine finds things the Python regex engine never did: `API_KEY`, +`BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, +`US_ROUTING_NUMBER`, `NPI`, `UUID`, and the German identifier set behind the +`de` locale. Credential detectors ship enabled by default; German and `UUID` +stay opt-in. + +These differences go the other way and are recorded, not hidden, in +[tests/contracts/rust-1.2.0.json](../tests/contracts/rust-1.2.0.json): + +| Input | Python | Native | Why | +| --- | --- | --- | --- | +| `4111 1111 1111 1112` | `CREDIT_CARD` | none | Core validates the Luhn checksum | +| `x123-45-6789y` | `SSN` | none | Core requires non-alphanumeric boundaries | +| `Invoice 1234567890` | `PHONE` | none | Core requires an area code starting 2-9, which is what keeps Unix timestamps out of `PHONE` | +| `unsupported` locale | locale ignored | `ValueError` | Core validates locales against its inventory | + +`PERSON` and `ADDRESS` used to need spaCy or GLiNER through `[nlp]` and +`[nlp-advanced]`. They now come from an embedded model on the native path, with +no download and no interpreter. `PERSON` for a text scan on `backend="python"` +is still Python-engine territory: the pure-Python regex engine has no name +detection, so select `PERSON` without forcing `backend="python"`. + +## If the output must not move + +Pin the backend, not just the version: + +```python +import datafog + +datafog.scan(text, backend="python") # exact 4.9.0 output +``` + +Or pin the whole package: `pip install "datafog==4.9.0"`. + +Detection output is part of the contract. Per +[VERSIONING.md](../VERSIONING.md), a change to the finding set is a MINOR, never +a PATCH, so a patch release will not move your golden files. + +## Reproducibility + +Core versions independently. If output must be byte-identical across machines, +lock it: + +```bash +pip install "datafog==4.10.0" "datafog-core==1.2.0" +``` + +`datafog.native.version()` reports the installed engine version and +`datafog.native.contract_version()` the contract it speaks, for bug reports. +Core follows its own [versioning policy](https://github.com/DataFog/datafog-core/blob/main/VERSIONING.md); +a contract bump is a Core MAJOR and raises the floor here. + +## What is not changing yet + +`detect()`, `process()`, OCR and Spark all still work and still warn. They are +removed in 5.0, not here. The last supported version is 4.10.0 — see the +[retirement schedule](retirement.rst). diff --git a/docs/migration-4.9.md b/docs/migration-4.9.md index 3bf2b900..4e06e3fe 100644 --- a/docs/migration-4.9.md +++ b/docs/migration-4.9.md @@ -179,8 +179,9 @@ They are now scheduled for removal in 5.0. Use `scan` or `redact` in 4.9 and eva the native preview before upgrading to 5.0. `process(anonymize=True)` used older placeholder/hash semantics; migrating to `redact` can intentionally change output. -Users needing OCR/Spark can remain on the final 4.x release. No successor package -or indefinite support commitment is introduced here. Deprecation notices do not +Users needing OCR/Spark can pin `pip install "datafog==4.10.0"`, the last +supported version. No successor package is introduced here. See +[the retirement schedule](retirement.md). Deprecation notices do not trigger downloads or import heavy dependencies during ordinary text imports. ## Verification and performance diff --git a/docs/optional-surfaces.rst b/docs/optional-surfaces.rst index 96d1494d..006bec49 100644 --- a/docs/optional-surfaces.rst +++ b/docs/optional-surfaces.rst @@ -18,7 +18,8 @@ The default path is: OCR and Spark are deprecated optional surfaces in 4.9 and will be removed in 5.0. Their APIs, installation extras, and existing behavior remain available throughout -4.9. Remain on the final 4.x release if you need continued OCR or Spark support. +4.9. The last supported version is 4.10.0 — pin it with ``pip install +"datafog==4.10.0"``. See :doc:`retirement`. No replacement package is introduced by this migration. Use sites emit ``FutureWarning`` notices, visible with Python's default warning diff --git a/docs/python-sdk.rst b/docs/python-sdk.rst index 2a8a2ddf..98781f96 100644 --- a/docs/python-sdk.rst +++ b/docs/python-sdk.rst @@ -27,10 +27,11 @@ available for existing users. ``TextService(engine="regex")`` is the dependency-light service path; ``spacy``, ``gliner``, ``smart``, OCR, and Spark surfaces require their explicit extras. -4.9.0 compatibility and Core preview ------------------------------------- +4.10.0 native default and Core preview +-------------------------------------- -DataFog 4.9.0 provides an experimental Rust backend and native schema preview. +DataFog 4.10.0 uses the native engine by default and keeps the native schema +preview at ``datafog.v5`` and ``datafog.native``. Existing top-level ``scan``/``redact`` functions retain their result shapes and use the Python backend by default. They delegate through the new facade, whose classes are the same objects as the established result types: @@ -44,9 +45,9 @@ continue to work at the top level in 4.9 but warn of removal in 5.0, revising th previous promise to retain them throughout 5.x. Moving from ``process`` to ``redact`` can change old placeholder and hash output; compare results explicitly. -The adapter requires ``datafog-core>=0.4.0,<0.5``. Install with -``python -m pip install --upgrade "datafog[rust]==4.9.0"`` to evaluate Rust. -A base install of ``datafog==4.9.0`` has no native dependency; installing the +The adapter requires ``datafog-core>=1.2.0,<2``, which the base install pulls. Install with +``datafog-core`` is installed by every ``datafog`` install; ``backend="python"`` +or ``DATAFOG_BACKEND=python`` forces the pure-Python engine. Installing the extra does not change the default Python backend: .. code-block:: python @@ -89,7 +90,7 @@ a separate decision. The adapter requires capability contract version 1 and discovers supported labels, locales, and activation settings from the installed Core. Future finding labels -are preserved. Core 0.4.x updates may change detection output; lock the version +are preserved. Core 1.x updates may change detection output; lock the version when reproducibility is required. German aliases ``de``, ``de-DE``, and ``de_DE`` activate German detectors; ``en-US`` and ``fr`` activate only base detectors. Locale validation trims ASCII whitespace and ignores ASCII case; unsupported @@ -101,7 +102,7 @@ through Core metadata. Core's ``PERSON`` is structured-only and explicitly selecting it for Rust text scanning raises an error. The Python default backend keeps its existing behavior. -Core 0.4.0 adds JWT, private-key, contextual routing-number, and contextual NPI +Core 1.2 adds the same JWT, private-key, contextual routing-number and NPI findings. The legacy overlap policy can prefer ``PHONE`` over a same-span ``NPI``; filtering for NPI then returns no entities. Native ``datafog.v5.scan`` retains NPI, and native transformation can select it. Legacy result and transformation @@ -158,7 +159,8 @@ OCR and Spark remain available as optional surfaces throughout 4.9: DataFog 4.9.0 deprecates OCR and Spark for removal in 5.0. Use-time ``FutureWarning`` notices are visible under normal Python warning filters. Users -needing these features can remain on the final 4.x release; this migration does +needing these features can pin ``datafog==4.10.0``, the last supported version; +this migration does not introduce successor packages or promise indefinite maintenance. See :doc:`optional-surfaces` for install notes and limitations. diff --git a/docs/retirement.rst b/docs/retirement.rst new file mode 100644 index 00000000..17c5c0af --- /dev/null +++ b/docs/retirement.rst @@ -0,0 +1,115 @@ +=================== +Retirement schedule +=================== + +This page is the standing record of surfaces DataFog is removing. It is updated +once per release, and a row is added in the MINOR that deprecates the surface — +never retroactively. + +Ordering and removal rules are defined in +`VERSIONING.md `__. + +Removed or scheduled +==================== + +.. list-table:: + :header-rows: 1 + + * - Surface + - Last supported version + - Replacement + - Removed in + - Deprecated in + * - ``OCRMService`` + - ``4.10.0`` + - none + - ``5.0.0`` + - ``4.9.0`` + * - ``DonutProcessor`` + - ``4.10.0`` + - none + - ``5.0.0`` + - ``4.9.0`` + * - ``[ocr]`` extra (Donut, Tesseract, Pillow) + - ``4.10.0`` + - none + - ``5.0.0`` + - ``4.9.0`` + * - ``[distributed]`` extra (PySpark, pandas) + - ``4.10.0`` + - :ref:`spark-recipe` + - ``5.0.0`` + - ``4.9.0`` + * - ``detect()`` + - ``4.10.0`` + - ``scan()`` + - ``5.0.0`` + - ``4.9.0`` + * - ``process()`` + - ``4.10.0`` + - ``redact()`` + - ``5.0.0`` + - ``4.9.0`` + +Installing ``pip install "datafog==4.10.0"`` keeps every row above working. +That version stays installable on PyPI permanently. + +OCR +=== + +There is no successor package and no 5.x reimplementation. OCR quality was never +the reason anyone adopted DataFog, and a mediocre replacement is worse than a +pin. + +.. code-block:: bash + + pip install "datafog==4.10.0[ocr]" + +If you need continued OCR support beyond that, stay on ``4.10.0``. Track the +surfaces you depend on by name, not by staying on a major version range, so a +lock file review shows the pin. + +.. _spark-recipe: + +Spark +===== + +No distributed API ships in 5.0. Partition the dataframe and call ``scan`` per +partition: one Spark job with a ``mapInPandas`` step, or Dask's +``map_partitions``. Both run the local engine and need no plugin. + +.. code-block:: python + + # Spark + def scan_partition(rows): + for pdf in rows: + pdf["findings"] = pdf["text"].map(lambda t: len(datafog.scan(t))) + yield pdf + + df.mapInPandas(scan_partition, schema="text string, findings long") + +.. code-block:: python + + # Dask + ddf["findings"] = ddf["text"].map_partitions( + lambda s: s.map(lambda t: len(datafog.scan(t))) + ) + +Per-partition ``scan`` is a pure local call with no network and no shared state, +so worker fan-out is just normal framework behaviour. What you lose versus +``SparkService`` is the UDF packaging, not correctness. + +detect and process +================== + +``detect()`` and ``process()`` were the pre-4.0 entry points. +``process(anonymize=True)`` used older placeholder and hash semantics, so +migrating to ``redact()`` can intentionally change output. Read the output +before you commit the change. + +DataFog Core +============ + +``datafog-core`` retires surfaces on its own schedule, documented in that +repository. The Python surface here inherits Core's ``contract_version``; a +contract bump is a Core MAJOR and raises the floor in ``datafog`` accordingly. diff --git a/scripts/check_rust_install.py b/scripts/check_rust_install.py index 26120232..bf84c45d 100644 --- a/scripts/check_rust_install.py +++ b/scripts/check_rust_install.py @@ -1,4 +1,4 @@ -"""Smoke-test installed 4.9 bridge wheels with python -I, outside checkout imports.""" +"""Smoke-test an installed wheel with python -I, outside checkout imports.""" import importlib.metadata import sys @@ -13,18 +13,20 @@ def main(): from datafog import v5 from datafog.compat import v4 - assert importlib.metadata.version("datafog-core").startswith("0.4.") + installed = importlib.metadata.version("datafog-core") + assert installed.split(".")[0] == "1", installed assert datafog_core.capabilities()["contract_version"] == 1 assert v4.Entity is datafog.Entity assert v5.Finding is datafog_core.Finding text = "👋 Contact alice@example.com" - legacy = datafog.scan(text, backend="rust") + # No explicit backend: the native engine must be the resolved default. + legacy = datafog.scan(text) assert isinstance(legacy, datafog.ScanResult) assert legacy.entities[0].text == "alice@example.com" assert ( text[legacy.entities[0].start : legacy.entities[0].end] == "alice@example.com" ) - assert datafog.redact(text, backend="rust").redacted_text == "👋 Contact [EMAIL_1]" + assert datafog.redact(text).redacted_text == "👋 Contact [EMAIL_1]" native = v5.scan(text) assert isinstance(native[0], datafog_core.Finding) assert ( diff --git a/setup.py b/setup.py index 72cb2da3..4f012b22 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,9 @@ # Core lightweight dependencies only core_deps = [ + # Native engine, the default detection path. Exact upper bound: detection + # output is contract-versioned and must stay reproducible. + "datafog-core>=1.2.0,<2", "pydantic>=2.0,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", @@ -84,7 +87,8 @@ ] extras_require = { - "rust": ["datafog-core>=0.4.0,<0.5"], + # Retained for explicitness; the base install already pulls Core. + "rust": ["datafog-core>=1.2.0,<2"], "nlp": nlp_deps, "nlp-advanced": nlp_advanced_deps, "ocr": ocr_deps, @@ -128,7 +132,7 @@ ], }, classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", diff --git a/tests/contracts/rust-0.4.0.json b/tests/contracts/rust-1.2.0.json similarity index 70% rename from tests/contracts/rust-0.4.0.json rename to tests/contracts/rust-1.2.0.json index 541cc3a3..6f46c6b8 100644 --- a/tests/contracts/rust-0.4.0.json +++ b/tests/contracts/rust-1.2.0.json @@ -1,5 +1,5 @@ { - "core_version": "0.4.0", + "core_version": "1.2.0", "cases": { "scan-observation-invalid-card": { "classification": "detector-difference", @@ -41,6 +41,21 @@ }, "warnings": [] } + }, + "scan-observation-ambiguous-number": { + "classification": "detector-difference", + "reason": "Core's North American area-code check requires a leading digit of 2-9, so a bare 10-digit run starting with 1 stays undetected. This is the deliberate Unix-timestamp protection; Python 4.8.1 matches it.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Invoice 1234567890", + "engine_used": "regex" + } + }, + "warnings": [] + } } } } diff --git a/tests/rust_contract.py b/tests/rust_contract.py index 3b477231..85d0d6a9 100644 --- a/tests/rust_contract.py +++ b/tests/rust_contract.py @@ -9,7 +9,7 @@ from tests.contract_481 import FIXTURE, observe -DEVIATIONS = Path(__file__).parent / "contracts" / "rust-0.4.0.json" +DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.2.0.json" TARGETS = { "datafog:scan", "datafog:redact", diff --git a/tests/test_allowlist.py b/tests/test_allowlist.py index 1f3bef6e..0bd85aec 100644 --- a/tests/test_allowlist.py +++ b/tests/test_allowlist.py @@ -10,7 +10,8 @@ EMAIL = "jane.doe@" "example.com" OTHER_EMAIL = "sid@" "example.com" -TIMESTAMP_LIKE = "17830" "25668" # ten digits: matches the PHONE pattern +TIMESTAMP_LIKE = "44123" "45678" # ten digits, NANP-shaped: both backends match +LEADING_ONE_TIMESTAMP = "17830" "25668" # native reads the leading 1 as a country code class TestExactAllowlist: @@ -54,6 +55,17 @@ def test_pattern_suppresses_matching_entities(self): ) assert result.entities == [] + def test_leading_one_timestamp_is_a_documented_native_delta(self): + # The native NANP check reads a leading 1 as a country code and needs a + # nine-digit national number after it, so this misses; the Python engine + # matches. Recorded rather than silently accepted. + assert datafog.scan(f"created {LEADING_ONE_TIMESTAMP}", engine="regex").entities == [] + assert len( + datafog.scan( + f"created {LEADING_ONE_TIMESTAMP}", engine="regex", backend="python" + ).entities + ) == 1 + def test_pattern_matches_full_entity_text_only(self): result = datafog.scan( f"mail {EMAIL}", engine="regex", allowlist_patterns=[r"^jane\."] diff --git a/tests/test_api_bridge_49.py b/tests/test_api_bridge_49.py index 483aa7f8..d47ff561 100644 --- a/tests/test_api_bridge_49.py +++ b/tests/test_api_bridge_49.py @@ -32,7 +32,7 @@ def test_existing_positional_calls_match_compatibility_facade(): def test_backend_is_additive_keyword_only(function): parameter = inspect.signature(function).parameters["backend"] assert parameter.kind is inspect.Parameter.KEYWORD_ONLY - assert parameter.default == "python" + assert parameter.default is None @pytest.mark.parametrize("name", ["scan", "redact"]) diff --git a/tests/test_backend_resolution.py b/tests/test_backend_resolution.py new file mode 100644 index 00000000..729e1a54 --- /dev/null +++ b/tests/test_backend_resolution.py @@ -0,0 +1,111 @@ +"""Backend resolution: native by default, Python fallback, env override.""" + +import builtins + +import pytest + +import datafog +from datafog import engine +from datafog.exceptions import CoreUnavailableWarning + + +@pytest.fixture +def without_core(monkeypatch): + """Hide ``datafog_core`` from the import machinery.""" + original_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + if name == "datafog_core" or name.startswith("datafog_core."): + raise ImportError("hidden for test") + return original_import(name, *args, **kwargs) + + monkeypatch.delitem(__import__("sys").modules, "datafog_core", raising=False) + monkeypatch.setattr(builtins, "__import__", guarded_import) + monkeypatch.setattr(engine, "_cnn_cache", None) + + +def test_auto_resolves_to_native_when_core_is_installed(): + pytest.importorskip("datafog_core") + assert engine.resolve_backend(None, "regex") == "rust" + + +def test_auto_resolves_to_python_without_core(without_core): + assert engine.resolve_backend(None, "regex") == "python" + + +def test_auto_keeps_non_regex_engines_on_python(): + pytest.importorskip("datafog_core") + for engine_name in ("smart", "spacy", "gliner"): + assert engine.resolve_backend(None, engine_name) == "python" + + +def test_python_fallback_warns_once_then_stays_quiet(without_core): + with pytest.warns(CoreUnavailableWarning) as caught: + datafog.scan("mail jane@example.com") + datafog.scan("mail jane@example.com") + assert len(caught) == 1 + + +def test_explicit_python_does_not_warn(): + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("error", CoreUnavailableWarning) + assert datafog.scan("mail jane@example.com", backend="python").entities + + +def test_env_override_selects_the_python_engine(monkeypatch): + pytest.importorskip("datafog_core") + monkeypatch.setenv(engine.BACKEND_ENV, "python") + assert engine.resolve_backend(None, "regex") == "python" + + +def test_explicit_backend_beats_the_env(monkeypatch): + pytest.importorskip("datafog_core") + monkeypatch.setenv(engine.BACKEND_ENV, "python") + assert engine.resolve_backend("rust", "regex") == "rust" + + +def test_env_is_case_and_space_insensitive(monkeypatch): + pytest.importorskip("datafog_core") + monkeypatch.setenv(engine.BACKEND_ENV, " AUTO ") + assert engine.resolve_backend(None, "regex") == "rust" + + +def test_unknown_env_value_rejected(monkeypatch): + monkeypatch.setenv(engine.BACKEND_ENV, "gpu") + with pytest.raises(ValueError, match="backend must be one of"): + engine.scan("text", "regex") + + +def test_explicit_rust_without_core_still_raises(without_core): + with pytest.raises(ImportError, match="datafog\\[rust\\]"): + datafog.scan("mail jane@example.com", backend="rust") + + +def test_person_selection_without_model_raises_a_pin(monkeypatch): + from types import SimpleNamespace + + meta = { + "contract_version": 1, + "supported_entities": ["EMAIL", "PERSON"], + "default_entities": ["EMAIL"], + "locales": {"en-US": {"enabled_entities": []}}, + "entities": { + "EMAIL": {"scopes": ["structured", "text"], "activation": {"kind": "default"}}, + "PERSON": {"scopes": ["structured"], "activation": {"kind": "default"}}, + }, + } + stub = SimpleNamespace(capabilities=lambda: meta, scan=lambda text, config=None: []) + monkeypatch.setitem(__import__("sys").modules, "datafog_core", stub) + monkeypatch.setattr(engine, "_cnn_cache", None) + with pytest.raises(ValueError, match="embedded model"): + datafog.scan("Jane Doe", entity_types=["PERSON"], backend="rust") + + +def test_native_findings_are_classified_as_regex_provenance(): + pytest.importorskip("datafog_core") + result = datafog.scan("mail jane@example.com") + assert result.engine_used == "regex" + assert result.entities[0].engine == "regex" + assert result.entities[0].confidence == 1.0 diff --git a/tests/test_contract_481.py b/tests/test_contract_481.py index 5aa5b767..95158b07 100644 --- a/tests/test_contract_481.py +++ b/tests/test_contract_481.py @@ -6,13 +6,20 @@ import pytest from tests.contract_481 import FIXTURE, observe +from tests.rust_contract import DEVIATIONS CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) +# The default backend is native from 4.10.0, so the reviewed native +# differences are the published 4.10 observations for those cases. +REVIEWED = json.loads(DEVIATIONS.read_text(encoding="utf-8")) def expected_49(case): - """Allow only the four added backend parameters and revised shim notices.""" - expected = copy.deepcopy(case["expected"]) + """Allow only the added backend parameter, revised shim notices and the + reviewed native differences.""" + + deviation = REVIEWED["cases"].get(case["id"]) + expected = copy.deepcopy(deviation["expected"]) if deviation else copy.deepcopy(case["expected"]) if case["id"] == "public-signatures": for target in ( "datafog:scan", @@ -25,7 +32,7 @@ def expected_49(case): "name": "backend", "kind": "KEYWORD_ONLY", "required": False, - "default": "python", + "default": None, } ) if case.get("target") in {"datafog:detect", "datafog:process"}: @@ -41,7 +48,9 @@ def expected_49(case): "message": ( f"datafog.{name}() is deprecated and will be removed in 5.0. " f"Use {replacement} instead. " - "The earlier promise to retain this shim through 5.x has been revised." + 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0". ' + "The earlier promise to retain this shim through 5.x has been revised. " + "See docs/retirement.rst." ), } ] diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index 272060af..f767fd0b 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -50,7 +50,7 @@ def test_supported_wheel_version_and_capability_contract(): version = importlib.metadata.version("datafog-core") - assert version.split(".")[:2] == ["0", "4"], version + assert version.split(".")[:1] == ["1"], version capabilities = core.capabilities() assert capabilities["contract_version"] == 1 supported = capabilities["supported_entities"] @@ -174,12 +174,16 @@ def test_contextual_identifiers_do_not_activate_without_context(label, raw): assert not result.entities -def test_npi_retains_native_identity_but_legacy_overlap_prefers_phone(): - """4.x suppresses overlaps before selection; this is a reviewed difference.""" +def test_contextual_npi_survives_the_legacy_overlap_policy(): + """A leading-1 national number is not PHONE, so NPI is no longer suppressed. + + Core 1.2 accepts exchange codes starting with 1 but keeps the area-code + 2-9 check, so `1234567893` is NPI only. Under Core 0.4 the same span was + ambiguous and the legacy overlap policy dropped the NPI. + """ text = "😀 café NPI 1234567893" native = v5.scan(text) assert [(item.entity_type, item.matched_text) for item in native] == [ - ("PHONE", "1234567893"), ("NPI", "1234567893"), ] assert { @@ -187,10 +191,12 @@ def test_npi_retains_native_identity_but_legacy_overlap_prefers_phone(): } == {(11, 21)} legacy = datafog.scan(text, backend="rust") assert [(item.type, item.text) for item in legacy.entities] == [ - ("PHONE", "1234567893") + ("NPI", "1234567893") + ] + assert [item.type for item in datafog.scan(text, entity_types=["NPI"], backend="rust").entities] == [ + "NPI" ] - assert not datafog.scan(text, entity_types=["NPI"], backend="rust").entities - assert datafog.redact(text, backend="rust").redacted_text == "😀 café NPI [PHONE_1]" + assert datafog.redact(text, backend="rust").redacted_text == "😀 café NPI [NPI_1]" # Native selection occurs before transformation overlap resolution, exposing # the NPI-specific route without changing the 4.x compatibility contract. result = v5.scan_and_transform( @@ -200,9 +206,12 @@ def test_npi_retains_native_identity_but_legacy_overlap_prefers_phone(): assert result.text == "😀 café NPI [NPI]" -def test_structured_only_person_rejected_by_text_adapter(): - with pytest.raises(ValueError, match="PERSON"): - datafog.scan("Jane Doe", entity_types=["PERSON"], backend="rust") +def test_structured_only_person_served_by_the_embedded_model(): + """`PERSON` is structured-only in the rule inventory but the embedded CNN + covers prose, so text selection succeeds instead of raising.""" + assert not datafog.scan("Jane Doe", entity_types=["PERSON"], backend="python").entities + result = datafog.scan("Jane Doe\n14 Penny Lane", entity_types=["PERSON", "ADDRESS"]) + assert [item.type for item in result.entities] == ["PERSON", "ADDRESS"] def test_unsupported_locale_rejected_explicitly(): @@ -210,6 +219,9 @@ def test_unsupported_locale_rejected_explicitly(): datafog.scan("a@example.com", locales=["unsupported"], backend="rust") -def test_python_default_backend_remains_unchanged(): - assert not datafog.scan(JWT).entities - assert [item.type for item in datafog.scan(JWT, backend="rust").entities] == ["JWT"] +def test_native_backend_is_the_default(): + assert [item.type for item in datafog.scan(JWT).entities] == ["JWT"] + assert [item.type for item in datafog.scan(JWT, backend="rust").entities] == [ + "JWT" + ] + assert not datafog.scan(JWT, backend="python").entities diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py index 96901cf2..920c35f3 100644 --- a/tests/test_de_pii_regex.py +++ b/tests/test_de_pii_regex.py @@ -134,12 +134,17 @@ def test_redaction_and_service_locale_support() -> None: def test_german_vat_redaction_suppresses_inner_generic_ssn_match( text: str, vat_text: str ) -> None: - # Default (no locale): v4.4.0 parity — the bare nine-digit run still - # matches the generic SSN pattern even when prefixed by a country code. - scan_result = scan(text, engine="regex") + # Default (no locale): the Python engine keeps v4.4.0 parity and still + # matches the inner nine-digit run. The native engine rejects an SSN that is + # embedded in an alphanumeric token, a reviewed detector difference. + scan_result = scan(text, engine="regex", backend="python") assert [(entity.type, entity.text) for entity in scan_result.entities] == [ ("SSN", "123456789") ] + native_expected = [] if "DE1" in text else [("SSN", "123456789")] + assert [ + (entity.type, entity.text) for entity in scan(text, engine="regex").entities + ] == native_expected # German locale: the longer DE_VAT_ID span wins via the engine's # span-overlap suppression, so the inner SSN match is dropped. @@ -148,8 +153,12 @@ def test_german_vat_redaction_suppresses_inner_generic_ssn_match( ("DE_VAT_ID", vat_text) ] - default_redaction = scan_and_redact(text, engine="regex") + default_redaction = scan_and_redact(text, engine="regex", backend="python") assert default_redaction.redacted_text == text.replace("123456789", "[SSN_1]") + native_redaction = scan_and_redact(text, engine="regex").redacted_text + assert native_redaction == ( + text if native_expected == [] else text.replace("123456789", "[SSN_1]") + ) redaction = scan_and_redact(text, engine="regex", locales=["de"]) assert redaction.redacted_text == text.replace(vat_text, "[DE_VAT_ID_1]") diff --git a/tests/test_legacy_retirement.py b/tests/test_legacy_retirement.py index f5dd80e3..f8be98f0 100644 --- a/tests/test_legacy_retirement.py +++ b/tests/test_legacy_retirement.py @@ -29,7 +29,10 @@ def unexpected_import(*args): warnings.simplefilter("error", LegacySurfaceWarning) with pytest.raises( LegacySurfaceWarning, - match=f"{surface} support is deprecated in 4.9 and will be removed in 5.0", + match=( + f"{surface} support is deprecated in 4.9 and will be removed in 5.0. " + 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0".' + ), ): (ImageService if surface == "OCR" else SparkService)() @@ -40,7 +43,7 @@ def unavailable(name): monkeypatch.setattr(importlib, "import_module", unavailable) with ( - pytest.warns(LegacySurfaceWarning, match="final 4.x release"), + pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"), pytest.raises(ImportError, match=r"datafog\[distributed\]"), ): SparkService() @@ -82,7 +85,7 @@ def test_donut_mock_output_is_preserved(monkeypatch): monkeypatch.setattr(donut_processor, "DONUT_TESTING_ENABLED", False) with pytest.warns(LegacySurfaceWarning): processor = DonutProcessor() - with pytest.warns(LegacySurfaceWarning, match="final 4.x release"): + with pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"): assert asyncio.run(processor.extract_text_from_image(object())) == ( '{"text": "Mock OCR text for testing"}' ) @@ -242,7 +245,7 @@ def test_image_cli_notice_visible_under_default_python_filters(image_argument): args.append("image.png") result = CliRunner().invoke(client.app, args) assert "deprecated in 4.9 and will be removed in 5.0" in result.stderr, result.output -assert "final 4.x release" in result.stderr, result.output +assert 'Last supported version is 4.10.0' in result.stderr, result.output if sys.argv[1] == "True": assert result.exit_code == 0, result.output assert "OCR Pipeline Results: ['OCR text']" in result.stdout diff --git a/tests/test_rust_backend.py b/tests/test_rust_backend.py index 74922c8f..34da6a21 100644 --- a/tests/test_rust_backend.py +++ b/tests/test_rust_backend.py @@ -153,9 +153,12 @@ def test_nonregex_engine_rejected(native, name): assert not calls -def test_bad_backend_rejected(): +def test_bad_backend_rejected(monkeypatch): with pytest.raises(ValueError, match="backend must be one of"): - engine.scan("", "regex", backend="auto") + engine.scan("", "regex", backend="unknown") + monkeypatch.setenv(engine.BACKEND_ENV, "unknown") + with pytest.raises(ValueError, match="backend must be one of"): + engine.scan("", "regex") def test_no_native_import_on_python_path_and_actionable_missing_error(monkeypatch): @@ -198,7 +201,9 @@ def test_all_legacy_transformations_use_native_findings(native, strategy): finding("EMAIL", "a@example.com", 14, 27), ] ) - expected = engine.scan_and_redact(text, "regex", strategy=strategy) + expected = engine.scan_and_redact( + text, "regex", strategy=strategy, backend="python" + ) actual = engine.scan_and_redact(text, "regex", strategy=strategy, backend="rust") assert actual == expected assert calls == [text] @@ -208,7 +213,9 @@ def test_all_legacy_transformations_use_native_findings(native, strategy): def test_real_native_unicode_and_transformations(strategy): pytest.importorskip("datafog_core") text = "😀 München a@example.com and a@example.com" - python_result = engine.scan_and_redact(text, "regex", strategy=strategy) + python_result = engine.scan_and_redact( + text, "regex", strategy=strategy, backend="python" + ) native_result = engine.scan_and_redact( text, "regex", strategy=strategy, backend="rust" ) diff --git a/tests/test_rust_contract.py b/tests/test_rust_contract.py index 9bea3641..c3f16bc0 100644 --- a/tests/test_rust_contract.py +++ b/tests/test_rust_contract.py @@ -15,7 +15,9 @@ def test_native_version_and_review_inventory(): - assert importlib.metadata.version("datafog-core").startswith("0.4.") + installed = importlib.metadata.version("datafog-core") + major, minor = REVIEWED["core_version"].split(".")[:2] + assert installed.split(".")[:2] == [major, minor] assert set(REVIEWED["cases"]) <= {case["id"] for case in CASES} for item in REVIEWED["cases"].values(): assert item["classification"] in { From d0b38929695fd28f1553f437e68194303033c347 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:41:42 -0800 Subject: [PATCH 02/23] chore: require datafog-core 1.3.0 1.2.0 shipped without the exchange-code fix, so 4.10.0 depends on the release that has it. Contract file follows the engine version. --- CHANGELOG.MD | 2 +- README.md | 4 ++-- RELEASE_NOTES_4.10.0.md | 4 ++-- docs/migration-4.10.md | 4 ++-- docs/python-sdk.rst | 2 +- setup.py | 4 ++-- tests/contracts/{rust-1.2.0.json => rust-1.3.0.json} | 2 +- tests/rust_contract.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) rename tests/contracts/{rust-1.2.0.json => rust-1.3.0.json} (98%) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 1049239e..d15aa102 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -16,7 +16,7 @@ #### Changed -- `datafog-core>=1.2.0,<2` is a normal dependency; `[rust]` remains as an +- `datafog-core>=1.3.0,<2` is a normal dependency; `[rust]` remains as an explicit spelling. - `Development Status: 5 - Production/Stable`. - Retirement warnings name `4.10.0` as the last supported version with an exact diff --git a/README.md b/README.md index 0f70fb23..172c0936 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ and `ADDRESS` from an embedded model, so name and address detection needs no spaCy and no model download. Findings keep `engine="regex"` provenance and `confidence=1.0` as compatibility values, not native confidence estimates. Reviewed differences from the Python engine are listed in -[tests/contracts/rust-1.2.0.json](tests/contracts/rust-1.2.0.json). +[tests/contracts/rust-1.3.0.json](tests/contracts/rust-1.3.0.json). `datafog.native` exposes the native schema directly. Python 3.13 support is certified for the core SDK, CLI, `nlp`, @@ -288,7 +288,7 @@ The [4.9 migration guide](docs/migration-4.9.md) explains opt-in Rust detection, the native `datafog.v5` preview, and the revised 5.0 retirement schedule for `detect`/`process`, OCR, and Spark. The Python detector remains the default. -4.10.0 requires Core `>=1.2.0,<2` and capability contract 1, and installs it by +4.10.0 requires Core `>=1.3.0,<2` and capability contract 1, and installs it by default. See the [4.10.0 release notes](RELEASE_NOTES_4.10.0.md) and the [4.10 migration guide](docs/migration-4.10.md). Entity labels, locales, and activation settings come from the installed diff --git a/RELEASE_NOTES_4.10.0.md b/RELEASE_NOTES_4.10.0.md index f5c592d4..fa84344a 100644 --- a/RELEASE_NOTES_4.10.0.md +++ b/RELEASE_NOTES_4.10.0.md @@ -31,12 +31,12 @@ once. stay opt-in. - `datafog.native` exposes the native schema, `capabilities()`, the embedded detector and version reporting. Experimental until 5.0. -- `datafog-core` is pinned `>=1.2.0,<2`. Detection output is contract-versioned. +- `datafog-core` is pinned `>=1.3.0,<2`. Detection output is contract-versioned. ## Detection differences Four reviewed differences from the Python engine are recorded in -[tests/contracts/rust-1.2.0.json](tests/contracts/rust-1.2.0.json) and asserted +[tests/contracts/rust-1.3.0.json](tests/contracts/rust-1.3.0.json) and asserted by the suite: Core validates card checksums, requires non-alphanumeric SSN boundaries, requires a North American area code starting 2-9 (which is what keeps Unix timestamps out of `PHONE`), and rejects unadvertised locales instead diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md index cbec85de..2046974c 100644 --- a/docs/migration-4.10.md +++ b/docs/migration-4.10.md @@ -40,7 +40,7 @@ The native engine finds things the Python regex engine never did: `API_KEY`, stay opt-in. These differences go the other way and are recorded, not hidden, in -[tests/contracts/rust-1.2.0.json](../tests/contracts/rust-1.2.0.json): +[tests/contracts/rust-1.3.0.json](../tests/contracts/rust-1.3.0.json): | Input | Python | Native | Why | | --- | --- | --- | --- | @@ -77,7 +77,7 @@ Core versions independently. If output must be byte-identical across machines, lock it: ```bash -pip install "datafog==4.10.0" "datafog-core==1.2.0" +pip install "datafog==4.10.0" "datafog-core==1.3.0" ``` `datafog.native.version()` reports the installed engine version and diff --git a/docs/python-sdk.rst b/docs/python-sdk.rst index 98781f96..581afc91 100644 --- a/docs/python-sdk.rst +++ b/docs/python-sdk.rst @@ -45,7 +45,7 @@ continue to work at the top level in 4.9 but warn of removal in 5.0, revising th previous promise to retain them throughout 5.x. Moving from ``process`` to ``redact`` can change old placeholder and hash output; compare results explicitly. -The adapter requires ``datafog-core>=1.2.0,<2``, which the base install pulls. Install with +The adapter requires ``datafog-core>=1.3.0,<2``, which the base install pulls. Install with ``datafog-core`` is installed by every ``datafog`` install; ``backend="python"`` or ``DATAFOG_BACKEND=python`` forces the pure-Python engine. Installing the extra does not change the default Python backend: diff --git a/setup.py b/setup.py index 4f012b22..df40d56c 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ core_deps = [ # Native engine, the default detection path. Exact upper bound: detection # output is contract-versioned and must stay reproducible. - "datafog-core>=1.2.0,<2", + "datafog-core>=1.3.0,<2", "pydantic>=2.0,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", @@ -88,7 +88,7 @@ extras_require = { # Retained for explicitness; the base install already pulls Core. - "rust": ["datafog-core>=1.2.0,<2"], + "rust": ["datafog-core>=1.3.0,<2"], "nlp": nlp_deps, "nlp-advanced": nlp_advanced_deps, "ocr": ocr_deps, diff --git a/tests/contracts/rust-1.2.0.json b/tests/contracts/rust-1.3.0.json similarity index 98% rename from tests/contracts/rust-1.2.0.json rename to tests/contracts/rust-1.3.0.json index 6f46c6b8..2c63c98c 100644 --- a/tests/contracts/rust-1.2.0.json +++ b/tests/contracts/rust-1.3.0.json @@ -1,5 +1,5 @@ { - "core_version": "1.2.0", + "core_version": "1.3.0", "cases": { "scan-observation-invalid-card": { "classification": "detector-difference", diff --git a/tests/rust_contract.py b/tests/rust_contract.py index 85d0d6a9..d3be6f66 100644 --- a/tests/rust_contract.py +++ b/tests/rust_contract.py @@ -9,7 +9,7 @@ from tests.contract_481 import FIXTURE, observe -DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.2.0.json" +DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.3.0.json" TARGETS = { "datafog:scan", "datafog:redact", From 288e2c237fe0ca51b6f4ee5b304775c90ffaee0d Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:12:09 -0800 Subject: [PATCH 03/23] ci: re-run after datafog-core 1.3.0 publication From f8d737ff42a8e8abdfb799ee0e4a73e471b64b11 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:27:30 -0800 Subject: [PATCH 04/23] test: score the accuracy corpus against both backends The corpus encodes Python-engine expectations, so it pins backend=python by default and scores the native engine under DATAFOG_ACCURACY_BACKEND=rust against tests/contracts/native-corpus.json, which records the twelve known divergences with reasons. CI runs both variants; pre-commit gains a ruff per-file-ignores entry for the dynamic re-exports in datafog/native.py. --- .flake8 | 5 +- .github/workflows/ci.yml | 10 +++ .github/workflows/release.yml | 5 ++ RELEASE_NOTES_4.10.0.md | 10 +++ datafog/__init__.py | 2 +- datafog/compat/v4.py | 2 +- datafog/engine.py | 40 +++++----- docs/migration-4.10.md | 17 ++++ ruff.toml | 7 ++ tests/contracts/native-corpus.json | 100 ++++++++++++++++++++++++ tests/test_allowlist.py | 18 +++-- tests/test_backend_resolution.py | 9 ++- tests/test_contract_481.py | 6 +- tests/test_core04_integration.py | 15 ++-- tests/test_detection_accuracy.py | 121 +++++++++++++++++++++++++---- 15 files changed, 311 insertions(+), 56 deletions(-) create mode 100644 ruff.toml create mode 100644 tests/contracts/native-corpus.json diff --git a/.flake8 b/.flake8 index e2ed01d9..db0f341c 100644 --- a/.flake8 +++ b/.flake8 @@ -3,4 +3,7 @@ ignore = E203, E266, E501, W503, B006, B007, B008, F401, C416, B950, B904 max-line-length = 88 max-complexity = 18 select = B,C,E,F,W,T4,B9 -exclude = venv, .venv, tests/.datafog_env, examples/venv \ No newline at end of file +exclude = venv, .venv, tests/.datafog_env, examples/venv# datafog.native.py re-exports the installed datafog-core surface through a +# dynamic __getattr__, so its __all__ names resolve at runtime. +per-file-ignores = + datafog/native.py: F822 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be420465..ba447626 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,6 +119,16 @@ jobs: --cov-report=xml \ --cov-report=term-missing + - name: Run detection accuracy corpus on the native engine + if: matrix.python-version == '3.11' && matrix.install-profile == 'nlp-advanced' + env: + DATAFOG_ACCURACY_BACKEND: rust + run: | + pytest tests/test_detection_accuracy.py \ + --tb=short \ + --ignore=tests/test_spark_integration.py \ + --ignore=tests/test_text_service_integration.py + - name: Run tests (nlp) if: matrix.install-profile == 'nlp' run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77e7aedd..d93c6bd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -167,6 +167,11 @@ jobs: - name: Run detection accuracy corpus run: python -m pytest tests/test_detection_accuracy.py -v --tb=short + - name: Run detection accuracy corpus on the native engine + env: + DATAFOG_ACCURACY_BACKEND: rust + run: python -m pytest tests/test_detection_accuracy.py --tb=short + - name: Run performance validation run: | OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 python tests/simple_performance_test.py diff --git a/RELEASE_NOTES_4.10.0.md b/RELEASE_NOTES_4.10.0.md index fa84344a..7608b6e0 100644 --- a/RELEASE_NOTES_4.10.0.md +++ b/RELEASE_NOTES_4.10.0.md @@ -45,6 +45,16 @@ of ignoring them. `(555) 123-4567` and similar form data now match: Core keeps the area-code check and accepts exchange codes starting with 1. +## Accuracy corpus + +The frozen corpus runs twice in CI now: once against the Python engine, whose +expectations it encodes, and once against the native engine with its twelve +recorded divergences in +[tests/contracts/native-corpus.json](tests/contracts/native-corpus.json). The +native divergences are strictness differences (Core validates more) plus email +boundary handling that Core currently exposes through `format: "env"` / +`"sql"` settings — flagged for a default decision in Core 1.4. + ## Deprecations unchanged `detect()`, `process()`, OCR and Spark still work and still warn; all are removed diff --git a/datafog/__init__.py b/datafog/__init__.py index 12ecc115..b0c7f2a8 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -11,8 +11,8 @@ import warnings -from ._legacy_retirement import LAST_SUPPORTED as _LAST_SUPPORTED from .__about__ import __version__ +from ._legacy_retirement import LAST_SUPPORTED as _LAST_SUPPORTED from .agent import create_guardrail, filter_output, sanitize, scan_prompt # Core API functions - always available (lightweight) diff --git a/datafog/compat/v4.py b/datafog/compat/v4.py index 49d32e2c..96d619aa 100644 --- a/datafog/compat/v4.py +++ b/datafog/compat/v4.py @@ -5,8 +5,8 @@ """ from ..engine import Entity, RedactResult, ScanResult -from ..engine import resolve_backend from ..engine import redact as _redact_entities +from ..engine import resolve_backend from ..engine import scan as _scan from ..engine import scan_and_redact as _scan_and_redact diff --git a/datafog/engine.py b/datafog/engine.py index fa6ad926..495c2d18 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -296,9 +296,7 @@ def _cnn_findings( except Exception as exc: # relative-path/limit errors surface to the caller raise _model_error(exc) from exc return [ - ({}, finding) - for finding in result.findings - if finding.entity_type in requested + ({}, finding) for finding in result.findings if finding.entity_type in requested ] @@ -347,23 +345,23 @@ def _rust_entities( if detector is not None: findings.extend(_cnn_findings(detector, model_requested, text)) for config, finding in findings: - canonical_type = _canonical_type(finding.entity_type) - start = finding.codepoint_range.start - end = finding.codepoint_range.end - key = (canonical_type, start, end) - if key in seen or not finding.matched_text.strip(): - continue - seen.add(key) - entities.append( - Entity( - type=canonical_type, - text=finding.matched_text, - start=start, - end=end, - confidence=1.0, - engine="regex", - ) + canonical_type = _canonical_type(finding.entity_type) + start = finding.codepoint_range.start + end = finding.codepoint_range.end + key = (canonical_type, start, end) + if key in seen or not finding.matched_text.strip(): + continue + seen.add(key) + entities.append( + Entity( + type=canonical_type, + text=finding.matched_text, + start=start, + end=end, + confidence=1.0, + engine="regex", ) + ) return _suppress_overlapping_entities(entities) @@ -590,7 +588,9 @@ def scan( if engine not in {"regex", "spacy", "gliner", "smart"}: raise ValueError("engine must be one of: regex, spacy, gliner, smart") - requested_backend = (backend or os.environ.get(BACKEND_ENV) or "auto").strip().lower() + requested_backend = ( + (backend or os.environ.get(BACKEND_ENV) or "auto").strip().lower() + ) backend = resolve_backend(backend, engine) _validate_backend(backend, engine) if backend == "python" and requested_backend == "auto" and not core_available(): diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md index 2046974c..9a730c3b 100644 --- a/docs/migration-4.10.md +++ b/docs/migration-4.10.md @@ -55,6 +55,23 @@ no download and no interpreter. `PERSON` for a text scan on `backend="python"` is still Python-engine territory: the pure-Python regex engine has no name detection, so select `PERSON` without forcing `backend="python"`. +## The accuracy corpus + +`tests/corpus` is the frozen accuracy corpus, and its expectations describe the +Python engine, which is what they were written against. `pytest +tests/test_detection_accuracy.py` pins `backend="python"`. Set +`DATAFOG_ACCURACY_BACKEND=rust` to score the native engine; its twelve known +divergences — eight misses, one added finding and three shifted email +boundaries — are enumerated with reasons in +[tests/contracts/native-corpus.json](../tests/contracts/native-corpus.json), and +CI runs both variants. + +The three shifted email boundaries are the one worth filing a follow-up for: +Core finds `alice@example.com` inside ``email=alice@example.com`` only under its +`format: "env"` setting, and inside a quoted value only under +`format: "sql"`. Log-shaped input wants that behaviour by default; Core 1.4 +should decide that rather than leaving it to each caller. + ## If the output must not move Pin the backend, not just the version: diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..536f394a --- /dev/null +++ b/ruff.toml @@ -0,0 +1,7 @@ +# Only the delta from the default ruff rules. Rule selection stays ruff's +# default so this hook behaves like pre-4.10; flake8 config is in .flake8. + +[lint.per-file-ignores] +# datafog/native.py re-exports the installed datafog-core surface through a +# dynamic __getattr__, so its __all__ names resolve at runtime. +"datafog/native.py" = ["F822"] diff --git a/tests/contracts/native-corpus.json b/tests/contracts/native-corpus.json new file mode 100644 index 00000000..26679ced --- /dev/null +++ b/tests/contracts/native-corpus.json @@ -0,0 +1,100 @@ +{ + "core_version": "1.3.0", + "cases": { + "adjacent-pii-no-separator": { + "missing": [ + [ + "SSN", + "123-45-6789" + ] + ], + "reason": "Core requires non-alphanumeric SSN boundaries; the adjacent-email case embeds it" + }, + "phone-two-values": { + "missing": [ + [ + "PHONE", + "555-000-1111" + ] + ], + "reason": "Core requires the exchange code to start 1-9; 000-1111 is a fictional exchange" + }, + "ssn-second-valid": { + "missing": [ + [ + "SSN", + "987-65-4321" + ] + ], + "reason": "Core rejects SSN after 'Backup SSN' context token in this boundary case" + }, + "date-slash-short": { + "missing": [ + [ + "DATE", + "3/5/2020" + ] + ], + "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" + }, + "date-dash-short": { + "missing": [ + [ + "DATE", + "3-5-2020" + ] + ], + "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" + }, + "date-year-only": { + "missing": [ + [ + "DATE", + "year 2024" + ] + ], + "reason": "Core DATE requires a full date; bare years are not matched" + }, + "de-vat-id-locale": { + "missing": [ + [ + "DE_VAT_ID", + "DE 123456789" + ] + ], + "reason": "Core German VAT requires the digits to follow the country prefix without a space" + }, + "de-iban-locale": { + "missing": [ + [ + "DE_IBAN", + "DE44 5001 0517 5407 3249 31" + ] + ], + "reason": "Core German IBAN accepts the grouped form but this spacing variant differs" + } + }, + "extra": { + "product-code-not-phone": { + "added": [ + [ + "PHONE", + "555-123-4567" + ] + ], + "reason": "Core accepts exchange codes starting with 1, so this product code is PHONE now; the area-code 2-9 check still keeps Unix timestamps out" + }, + "unicode-chinese-name": { + "offsets_shifted": true, + "reason": "Core's default email boundary set does not include a CJK prefix; ENV/SQL format handling covers the Latin cases it ships today" + }, + "json-escaped": { + "offsets_shifted": true, + "reason": "Core treats '=' as an assignment boundary under format: env, not by default" + }, + "markdown-code-block": { + "offsets_shifted": true, + "reason": "Core treats the opening quote as part of the token under format: sql, not by default" + } + } +} diff --git a/tests/test_allowlist.py b/tests/test_allowlist.py index 0bd85aec..f737a9f2 100644 --- a/tests/test_allowlist.py +++ b/tests/test_allowlist.py @@ -59,12 +59,18 @@ def test_leading_one_timestamp_is_a_documented_native_delta(self): # The native NANP check reads a leading 1 as a country code and needs a # nine-digit national number after it, so this misses; the Python engine # matches. Recorded rather than silently accepted. - assert datafog.scan(f"created {LEADING_ONE_TIMESTAMP}", engine="regex").entities == [] - assert len( - datafog.scan( - f"created {LEADING_ONE_TIMESTAMP}", engine="regex", backend="python" - ).entities - ) == 1 + assert ( + datafog.scan(f"created {LEADING_ONE_TIMESTAMP}", engine="regex").entities + == [] + ) + assert ( + len( + datafog.scan( + f"created {LEADING_ONE_TIMESTAMP}", engine="regex", backend="python" + ).entities + ) + == 1 + ) def test_pattern_matches_full_entity_text_only(self): result = datafog.scan( diff --git a/tests/test_backend_resolution.py b/tests/test_backend_resolution.py index 729e1a54..c6cd6d60 100644 --- a/tests/test_backend_resolution.py +++ b/tests/test_backend_resolution.py @@ -41,8 +41,8 @@ def test_auto_keeps_non_regex_engines_on_python(): def test_python_fallback_warns_once_then_stays_quiet(without_core): with pytest.warns(CoreUnavailableWarning) as caught: - datafog.scan("mail jane@example.com") - datafog.scan("mail jane@example.com") + for _ in range(2): + datafog.scan("mail jane@example.com") assert len(caught) == 1 @@ -92,7 +92,10 @@ def test_person_selection_without_model_raises_a_pin(monkeypatch): "default_entities": ["EMAIL"], "locales": {"en-US": {"enabled_entities": []}}, "entities": { - "EMAIL": {"scopes": ["structured", "text"], "activation": {"kind": "default"}}, + "EMAIL": { + "scopes": ["structured", "text"], + "activation": {"kind": "default"}, + }, "PERSON": {"scopes": ["structured"], "activation": {"kind": "default"}}, }, } diff --git a/tests/test_contract_481.py b/tests/test_contract_481.py index 95158b07..43863dfd 100644 --- a/tests/test_contract_481.py +++ b/tests/test_contract_481.py @@ -19,7 +19,11 @@ def expected_49(case): reviewed native differences.""" deviation = REVIEWED["cases"].get(case["id"]) - expected = copy.deepcopy(deviation["expected"]) if deviation else copy.deepcopy(case["expected"]) + expected = ( + copy.deepcopy(deviation["expected"]) + if deviation + else copy.deepcopy(case["expected"]) + ) if case["id"] == "public-signatures": for target in ( "datafog:scan", diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index f767fd0b..8cd218a9 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -193,9 +193,10 @@ def test_contextual_npi_survives_the_legacy_overlap_policy(): assert [(item.type, item.text) for item in legacy.entities] == [ ("NPI", "1234567893") ] - assert [item.type for item in datafog.scan(text, entity_types=["NPI"], backend="rust").entities] == [ - "NPI" - ] + assert [ + item.type + for item in datafog.scan(text, entity_types=["NPI"], backend="rust").entities + ] == ["NPI"] assert datafog.redact(text, backend="rust").redacted_text == "😀 café NPI [NPI_1]" # Native selection occurs before transformation overlap resolution, exposing # the NPI-specific route without changing the 4.x compatibility contract. @@ -209,7 +210,9 @@ def test_contextual_npi_survives_the_legacy_overlap_policy(): def test_structured_only_person_served_by_the_embedded_model(): """`PERSON` is structured-only in the rule inventory but the embedded CNN covers prose, so text selection succeeds instead of raising.""" - assert not datafog.scan("Jane Doe", entity_types=["PERSON"], backend="python").entities + assert not datafog.scan( + "Jane Doe", entity_types=["PERSON"], backend="python" + ).entities result = datafog.scan("Jane Doe\n14 Penny Lane", entity_types=["PERSON", "ADDRESS"]) assert [item.type for item in result.entities] == ["PERSON", "ADDRESS"] @@ -221,7 +224,5 @@ def test_unsupported_locale_rejected_explicitly(): def test_native_backend_is_the_default(): assert [item.type for item in datafog.scan(JWT).entities] == ["JWT"] - assert [item.type for item in datafog.scan(JWT, backend="rust").entities] == [ - "JWT" - ] + assert [item.type for item in datafog.scan(JWT, backend="rust").entities] == ["JWT"] assert not datafog.scan(JWT, backend="python").entities diff --git a/tests/test_detection_accuracy.py b/tests/test_detection_accuracy.py index 61dec7ba..6e7d8dfe 100644 --- a/tests/test_detection_accuracy.py +++ b/tests/test_detection_accuracy.py @@ -281,10 +281,13 @@ def _canon_type(entity_type: str) -> str: def _extract_entities( - text: str, engine: str, locales: list[str] | None = None + text: str, + engine: str, + locales: list[str] | None = None, + backend: str | None = None, ) -> list[dict[str, Any]]: try: - result = scan(text=text, engine=engine, locales=locales) + result = scan(text=text, engine=engine, locales=locales, backend=backend) except (ImportError, EngineNotAvailable) as exc: pytest.skip(f"{engine} engine unavailable in this environment: {exc}") @@ -336,6 +339,44 @@ def _required_expected( return expected_list +def _corpus_backend(engine: str | None = None) -> str: + """Backend under test for the frozen corpus, settable from the CLI. + + The default is ``python`` because the corpus expectations were authored + against that engine. Set ``DATAFOG_ACCURACY_BACKEND=rust`` to score the + native engine; its known divergences are then skipped through + ``tests/contracts/native-corpus.json``. + """ + import os + + value = os.environ.get("DATAFOG_ACCURACY_BACKEND", "python").strip().lower() + if engine is not None and value == "rust" and engine != "regex": + # The native path serves regex plus embedded-model findings; other + # engines are Python ML paths by definition. + value = "python" + if value not in {"python", "rust"}: + raise ValueError("DATAFOG_ACCURACY_BACKEND must be python or rust") + return "python" if value == "rust" and engine != "regex" else value + + +@lru_cache(maxsize=1) +def _native_deltas() -> dict[str, Any]: + path = Path("tests/contracts/native-corpus.json") + if not path.exists(): + return {"core_version": None, "cases": {}} + return json.loads(path.read_text(encoding="utf-8")) + + +def _native_missing(case_id: str) -> list[list[str]]: + """Findings the native engine is known to miss, asserted only as recorded.""" + return _native_deltas()["cases"].get(case_id, {}).get("missing", []) + + +def _native_offsets_shifted(case_id: str) -> bool: + """True when the native engine finds the value with different boundaries.""" + return bool(_native_deltas()["extra"].get(case_id, {}).get("offsets_shifted")) + + def _xfail_if_known_limitation( case: dict[str, Any], engine: str, corpus_kind: str ) -> None: @@ -346,11 +387,19 @@ def _xfail_if_known_limitation( def _assert_expected_found( - case: dict[str, Any], engine: str, corpus_kind: str + case: dict[str, Any], engine: str, corpus_kind: str, backend: str | None = None ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: text = case["input"] - actual = _extract_entities(text, engine, locales=case.get("locales")) + actual = _extract_entities( + text, engine, locales=case.get("locales"), backend=backend + ) expected = _required_expected(case["expected_entities"], engine, corpus_kind) + if (backend or _corpus_backend(engine)) == "rust": + expected = [ + entry + for entry in expected + if [entry["type"], entry["text"]] not in _native_missing(case["id"]) + ] for exp in expected: exp_type = _canon_type(exp["type"]) @@ -358,6 +407,15 @@ def _assert_expected_found( matches = [ ent for ent in actual if ent["type"] == exp_type and ent["text"] == exp_text ] + if _native_offsets_shifted(case["id"]): + # Same entity, different span: the native boundary rules treat the + # surrounding characters differently. Overlap is good enough. + matches = [ + ent + for ent in actual + if ent["type"] == exp_type + and not (ent["end"] <= exp["start"] or ent["start"] >= exp["end"]) + ] if not matches: matches = [ ent @@ -373,7 +431,9 @@ def _assert_expected_found( # If offsets are available from the engine output, validate exact position. with_offsets = [m for m in matches if m["start"] >= 0 and m["end"] >= 0] if with_offsets: - if engine == "regex" or exp_type in STRUCTURED_TYPES: + if ( + engine == "regex" or exp_type in STRUCTURED_TYPES + ) and not _native_offsets_shifted(case["id"]): assert any( m["start"] == exp["start"] and m["end"] == exp["end"] for m in with_offsets @@ -406,7 +466,10 @@ def _compute_metrics( for corpus_kind, cases in corpora: for case in cases: actual = _extract_entities( - case["input"], engine, locales=case.get("locales") + case["input"], + engine, + locales=case.get("locales"), + backend=_corpus_backend(engine), ) expected = _required_expected( case["expected_entities"], engine, corpus_kind @@ -475,7 +538,7 @@ def _prf(scores: dict[str, int]) -> dict[str, float]: @pytest.mark.parametrize("engine", FAST_ENGINES) def test_structured_pii_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "structured") - _assert_expected_found(case, engine, "structured") + _assert_expected_found(case, engine, "structured", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -485,7 +548,7 @@ def test_structured_pii_detection_fast(case: dict[str, Any], engine: str) -> Non @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_structured_pii_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "structured") - _assert_expected_found(case, engine, "structured") + _assert_expected_found(case, engine, "structured", backend=_corpus_backend(engine)) @pytest.mark.parametrize( @@ -494,7 +557,18 @@ def test_structured_pii_detection_slow(case: dict[str, Any], engine: str) -> Non @pytest.mark.parametrize("engine", FAST_ENGINES) def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "negative") - actual = _extract_entities(case["input"], engine, locales=case.get("locales")) + actual = _extract_entities( + case["input"], + engine, + locales=case.get("locales"), + backend=_corpus_backend(engine), + ) + if _corpus_backend(engine) == "rust": + added = { + tuple(pair) + for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) + } + actual = [e for e in actual if (e["type"], e["text"]) not in added] assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -505,7 +579,18 @@ def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "negative") - actual = _extract_entities(case["input"], engine, locales=case.get("locales")) + actual = _extract_entities( + case["input"], + engine, + locales=case.get("locales"), + backend=_corpus_backend(engine), + ) + if _corpus_backend(engine) == "rust": + added = { + tuple(pair) + for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) + } + actual = [e for e in actual if (e["type"], e["text"]) not in added] assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -515,7 +600,9 @@ def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", ["smart"]) def test_unstructured_pii_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "unstructured") - _assert_expected_found(case, engine, "unstructured") + _assert_expected_found( + case, engine, "unstructured", backend=_corpus_backend(engine) + ) @pytest.mark.slow @@ -525,14 +612,16 @@ def test_unstructured_pii_detection_fast(case: dict[str, Any], engine: str) -> N @pytest.mark.parametrize("engine", ["gliner", "spacy"]) def test_unstructured_pii_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "unstructured") - _assert_expected_found(case, engine, "unstructured") + _assert_expected_found( + case, engine, "unstructured", backend=_corpus_backend(engine) + ) @pytest.mark.parametrize("case", load_corpus("mixed_pii.json"), ids=lambda c: c["id"]) @pytest.mark.parametrize("engine", FAST_ENGINES) def test_mixed_pii_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "mixed") - _assert_expected_found(case, engine, "mixed") + _assert_expected_found(case, engine, "mixed", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -540,14 +629,14 @@ def test_mixed_pii_detection_fast(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_mixed_pii_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "mixed") - _assert_expected_found(case, engine, "mixed") + _assert_expected_found(case, engine, "mixed", backend=_corpus_backend(engine)) @pytest.mark.parametrize("case", load_corpus("edge_cases.json"), ids=lambda c: c["id"]) @pytest.mark.parametrize("engine", FAST_ENGINES) def test_edge_case_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "edge") - _assert_expected_found(case, engine, "edge") + _assert_expected_found(case, engine, "edge", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -555,7 +644,7 @@ def test_edge_case_detection_fast(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_edge_case_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "edge") - _assert_expected_found(case, engine, "edge") + _assert_expected_found(case, engine, "edge", backend=_corpus_backend(engine)) @pytest.mark.slow From e3d4300368780ba91fee4d6d6c3ae8b5f45f5ae8 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:38:48 -0800 Subject: [PATCH 05/23] style: prettier formatting for the native corpus delta --- tests/contracts/native-corpus.json | 63 +++++------------------------- 1 file changed, 9 insertions(+), 54 deletions(-) diff --git a/tests/contracts/native-corpus.json b/tests/contracts/native-corpus.json index 26679ced..e3a3bbd7 100644 --- a/tests/contracts/native-corpus.json +++ b/tests/contracts/native-corpus.json @@ -2,86 +2,41 @@ "core_version": "1.3.0", "cases": { "adjacent-pii-no-separator": { - "missing": [ - [ - "SSN", - "123-45-6789" - ] - ], + "missing": [["SSN", "123-45-6789"]], "reason": "Core requires non-alphanumeric SSN boundaries; the adjacent-email case embeds it" }, "phone-two-values": { - "missing": [ - [ - "PHONE", - "555-000-1111" - ] - ], + "missing": [["PHONE", "555-000-1111"]], "reason": "Core requires the exchange code to start 1-9; 000-1111 is a fictional exchange" }, "ssn-second-valid": { - "missing": [ - [ - "SSN", - "987-65-4321" - ] - ], + "missing": [["SSN", "987-65-4321"]], "reason": "Core rejects SSN after 'Backup SSN' context token in this boundary case" }, "date-slash-short": { - "missing": [ - [ - "DATE", - "3/5/2020" - ] - ], + "missing": [["DATE", "3/5/2020"]], "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" }, "date-dash-short": { - "missing": [ - [ - "DATE", - "3-5-2020" - ] - ], + "missing": [["DATE", "3-5-2020"]], "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" }, "date-year-only": { - "missing": [ - [ - "DATE", - "year 2024" - ] - ], + "missing": [["DATE", "year 2024"]], "reason": "Core DATE requires a full date; bare years are not matched" }, "de-vat-id-locale": { - "missing": [ - [ - "DE_VAT_ID", - "DE 123456789" - ] - ], + "missing": [["DE_VAT_ID", "DE 123456789"]], "reason": "Core German VAT requires the digits to follow the country prefix without a space" }, "de-iban-locale": { - "missing": [ - [ - "DE_IBAN", - "DE44 5001 0517 5407 3249 31" - ] - ], + "missing": [["DE_IBAN", "DE44 5001 0517 5407 3249 31"]], "reason": "Core German IBAN accepts the grouped form but this spacing variant differs" } }, "extra": { "product-code-not-phone": { - "added": [ - [ - "PHONE", - "555-123-4567" - ] - ], + "added": [["PHONE", "555-123-4567"]], "reason": "Core accepts exchange codes starting with 1, so this product code is PHONE now; the area-code 2-9 check still keeps Unix timestamps out" }, "unicode-chinese-name": { From 4f9b126d53db4cebd1c11418a158f54ce2d0f593 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:11:36 -0800 Subject: [PATCH 06/23] feat: track datafog-core 1.4.0 email-boundary defaults (#186) Core 1.4.0's text-mode email scanning trims label syntax by default, so `email=alice@example.com` reports `alice@example.com` and quoted values drop the wrapping quote. This moves the reviewed Rust contract file to `rust-1.4.0.json` (no new deviations: every recorded case still matches, verified by running the observation harness against 1.4.0), raises the dependency floor, and resolves the `json-escaped` and `markdown-code-block` native-corpus divergences. `unicode-chinese-name` stays with a corrected reason: its corpus fixture contains literal `?` characters, which are valid local-part characters, so the reviewed expectation remains overlap-only. Also stops the local verification gap this exposed: the two async/optional dependency failures and one accuracy improvement are relative to an unmodified main in the same environment (11 failed there, 10 here). --- CHANGELOG.MD | 14 ++++++++++++++ datafog/__about__.py | 2 +- setup.py | 4 ++-- tests/contracts/native-corpus.json | 10 +--------- .../contracts/{rust-1.3.0.json => rust-1.4.0.json} | 2 +- tests/rust_contract.py | 2 +- 6 files changed, 20 insertions(+), 14 deletions(-) rename tests/contracts/{rust-1.3.0.json => rust-1.4.0.json} (98%) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index d15aa102..ddc47dd9 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,5 +1,19 @@ # ChangeLog +## [4.11.0] + +#### Changed + +- Requires `datafog-core>=1.4.0`. Core's text-mode email scanning trims label + syntax by default, so `email=alice@example.com` reports `alice@example.com`, + quoted values drop the wrapping quote, and `SUPPORT_EMAIL=` labels stay out + of the span. Two recorded native-corpus divergences (`json-escaped`, + `markdown-code-block`) resolve and are removed from the reviewed deltas. The + `unicode-chinese-name` delta stays: its corpus fixture contains literal `?` + characters, which are valid local-part characters, so the reviewed + expectation remains overlap-only. The Rust contract review file moves to + `rust-1.4.0.json` with no new deviations: every recorded case still matches. + ## [4.10.0] #### Added diff --git a/datafog/__about__.py b/datafog/__about__.py index 897e6be2..bdb8bda6 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.10.0" +__version__ = "4.11.0" diff --git a/setup.py b/setup.py index df40d56c..7ce0366d 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ core_deps = [ # Native engine, the default detection path. Exact upper bound: detection # output is contract-versioned and must stay reproducible. - "datafog-core>=1.3.0,<2", + "datafog-core>=1.4.0,<2", "pydantic>=2.0,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", @@ -88,7 +88,7 @@ extras_require = { # Retained for explicitness; the base install already pulls Core. - "rust": ["datafog-core>=1.3.0,<2"], + "rust": ["datafog-core>=1.4.0,<2"], "nlp": nlp_deps, "nlp-advanced": nlp_advanced_deps, "ocr": ocr_deps, diff --git a/tests/contracts/native-corpus.json b/tests/contracts/native-corpus.json index e3a3bbd7..b9c3a893 100644 --- a/tests/contracts/native-corpus.json +++ b/tests/contracts/native-corpus.json @@ -41,15 +41,7 @@ }, "unicode-chinese-name": { "offsets_shifted": true, - "reason": "Core's default email boundary set does not include a CJK prefix; ENV/SQL format handling covers the Latin cases it ships today" - }, - "json-escaped": { - "offsets_shifted": true, - "reason": "Core treats '=' as an assignment boundary under format: env, not by default" - }, - "markdown-code-block": { - "offsets_shifted": true, - "reason": "Core treats the opening quote as part of the token under format: sql, not by default" + "reason": "The corpus fixture itself contains eleven ASCII '?' characters (a mangled CJK name); '?' is a valid RFC local-part character, so the email span includes them in every engine. Overlap is the reviewed expectation." } } } diff --git a/tests/contracts/rust-1.3.0.json b/tests/contracts/rust-1.4.0.json similarity index 98% rename from tests/contracts/rust-1.3.0.json rename to tests/contracts/rust-1.4.0.json index 2c63c98c..6d3aeb4b 100644 --- a/tests/contracts/rust-1.3.0.json +++ b/tests/contracts/rust-1.4.0.json @@ -1,5 +1,5 @@ { - "core_version": "1.3.0", + "core_version": "1.4.0", "cases": { "scan-observation-invalid-card": { "classification": "detector-difference", diff --git a/tests/rust_contract.py b/tests/rust_contract.py index d3be6f66..2b618007 100644 --- a/tests/rust_contract.py +++ b/tests/rust_contract.py @@ -9,7 +9,7 @@ from tests.contract_481 import FIXTURE, observe -DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.3.0.json" +DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.4.0.json" TARGETS = { "datafog:scan", "datafog:redact", From 352cafeeda6398ea7e4dc5aeec673e0863455bb7 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:53:31 -0800 Subject: [PATCH 07/23] fix(engine): preserve detection across backends, spans and large inputs --- datafog/engine.py | 378 +++++++++++++----- .../text_processing/gliner_annotator.py | 78 ++-- .../regex_annotator/regex_annotator.py | 29 +- .../text_processing/spacy_pii_annotator.py | 47 ++- datafog/services/text_service.py | 46 ++- tests/test_allowlist.py | 16 +- tests/test_backend_resolution.py | 4 +- tests/test_core04_integration.py | 5 +- tests/test_engine_api.py | 7 +- tests/test_review_engine.py | 306 ++++++++++++++ 10 files changed, 707 insertions(+), 209 deletions(-) create mode 100644 tests/test_review_engine.py diff --git a/datafog/engine.py b/datafog/engine.py index 495c2d18..ec33081b 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import hmac import os import re import warnings @@ -10,6 +11,8 @@ from functools import lru_cache from typing import Optional +import regex as bounded_re + from .exceptions import CoreUnavailableWarning, EngineNotAvailable from .processing.text_processing.regex_annotator import RegexAnnotator @@ -66,6 +69,10 @@ "US_ROUTING_NUMBER", "NPI", "UUID", + "PASSPORT_NUMBER", + "MEDICAL_RECORD_NUMBER", + "LICENSE_NUMBER", + "URL", } #: Labels the native rule engine reports as structured-only, served for text by @@ -78,7 +85,16 @@ _core_warned = False _cnn_cache: tuple[object, object] | None = None -NER_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} +NER_ENTITY_TYPES = { + "PERSON", + "ORGANIZATION", + "LOCATION", + "ADDRESS", + "PASSPORT_NUMBER", + "MEDICAL_RECORD_NUMBER", + "LICENSE_NUMBER", + "URL", +} ENTITY_TYPE_PRIORITY = { "DE_IBAN": 100, @@ -155,9 +171,13 @@ def _find_all_occurrences(text: str, needle: str) -> list[tuple[int, int]]: def _entities_from_dict( text: str, payload: dict[str, list[str]], engine: str, confidence: float ) -> list[Entity]: + """Compatibility for third-party annotators without offset support. + + Whole-word matching avoids interpreting "Ann" inside "Annual" as a name. + First-party annotators use their original character ranges instead. + """ entities: list[Entity] = [] value_offsets: dict[str, int] = {} - for raw_type, values in payload.items(): canonical_type = _canonical_type(raw_type) if canonical_type not in ALL_ENTITY_TYPES: @@ -165,25 +185,47 @@ def _entities_from_dict( for value in values: if not isinstance(value, str) or not value.strip(): continue - search_start = value_offsets.get(value, 0) - idx = text.find(value, search_start) - if idx < 0: - idx = text.find(value) - end = idx + len(value) if idx >= 0 else -1 - value_offsets[value] = end if end >= 0 else search_start + 1 + pattern = re.compile(r"(? list[Entity]: + method = getattr(annotator, "annotate_with_spans", None) + if not callable(method): + return _entities_from_dict(text, annotator.annotate(text), engine, confidence) + _, result = method(text) + entities = [] + for span in result.spans: + label = _canonical_type(span.label) + if label not in ALL_ENTITY_TYPES: + continue + if ( + not 0 <= span.start < span.end <= len(text) + or text[span.start : span.end] != span.text + ): + raise ValueError(f"{engine} returned an invalid entity span") + entities.append( + Entity(label, span.text, span.start, span.end, confidence, engine) + ) + return entities + + def _entity_length(entity: Entity) -> int: return max(entity.end - entity.start, 0) @@ -195,7 +237,31 @@ def _entities_overlap(left: Entity, right: Entity) -> bool: def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]: - selected: list[Entity] = [] + """Preserve longest-span precedence with O(n log n) interval queries.""" + coordinates = { + value: index + 1 + for index, value in enumerate( + sorted( + {value for entity in entities for value in (entity.start, entity.end)} + ) + ) + } + starts = [0] * (len(coordinates) + 1) + ends = [0] * (len(coordinates) + 1) + + def add(tree: list[int], index: int) -> None: + while index < len(tree): + tree[index] += 1 + index += index & -index + + def count(tree: list[int], index: int) -> int: + total = 0 + while index > 0: + total += tree[index] + index -= index & -index + return total + + selected = [] for entity in sorted( entities, key=lambda item: ( @@ -207,9 +273,12 @@ def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]: item.type, ), ): - if any(_entities_overlap(entity, kept) for kept in selected): + start, end = coordinates[entity.start], coordinates[entity.end] + if count(starts, end - 1) > count(ends, start): continue selected.append(entity) + add(starts, start) + add(ends, end) return sorted(selected, key=lambda item: (item.start, item.end, item.type)) @@ -289,15 +358,28 @@ def _cnn_detector_or_none() -> object | None: def _cnn_findings( detector: object, requested: set[str], text: str -) -> list[tuple[dict, object]]: - """Serve prose names and address blocks from the embedded model.""" - try: - result = detector.scan_model(text) - except Exception as exc: # relative-path/limit errors surface to the caller - raise _model_error(exc) from exc - return [ - ({}, finding) for finding in result.findings if finding.entity_type in requested - ] +) -> list[tuple[int, object]]: + """Bound model work while retaining document character coordinates. + + 4096 characters fit the native byte/token caps even for four-byte Unicode. + Overlap gives names and address blocks context on either side of a seam. + """ + chunk_length, overlap = 4096, 256 + findings = [] + for offset in range(0, max(len(text), 1), chunk_length - overlap): + chunk = text[offset : offset + chunk_length] + try: + result = detector.scan_model(chunk) + except Exception as exc: + raise _model_error(exc) from exc + findings.extend( + (offset, finding) + for finding in result.findings + if finding.entity_type in requested + ) + if offset + chunk_length >= len(text): + break + return findings def _model_error(exc: Exception) -> Exception: @@ -337,17 +419,18 @@ def _rust_entities( configs = scan_configs(datafog_core, requested - MODEL_ENTITY_TYPES, locales) entities: list[Entity] = [] seen: set[tuple[str, int, int]] = set() + native_text = re.sub(r"[\ud800-\udfff]", "\ufffd", text) findings = [ - (config, finding) + (0, finding) for config in configs - for finding in datafog_core.scan(text, config=config) + for finding in datafog_core.scan(native_text, config=config) ] if detector is not None: - findings.extend(_cnn_findings(detector, model_requested, text)) - for config, finding in findings: + findings.extend(_cnn_findings(detector, model_requested, native_text)) + for offset, finding in findings: canonical_type = _canonical_type(finding.entity_type) - start = finding.codepoint_range.start - end = finding.codepoint_range.end + start = offset + finding.codepoint_range.start + end = offset + finding.codepoint_range.end key = (canonical_type, start, end) if key in seen or not finding.matched_text.strip(): continue @@ -355,7 +438,7 @@ def _rust_entities( entities.append( Entity( type=canonical_type, - text=finding.matched_text, + text=text[start:end], start=start, end=end, confidence=1.0, @@ -369,16 +452,22 @@ def _spacy_entities(text: str) -> list[Entity]: annotator = _get_spacy_annotator() if isinstance(annotator, _UnavailableAnnotator): raise EngineNotAvailable(annotator.message) - payload = annotator.annotate(text) - return _entities_from_dict(text, payload, engine="spacy", confidence=0.7) + return [ + entity + for entity in _annotator_entities(text, annotator, "spacy", 0.7) + if entity.type in NER_ENTITY_TYPES + ] -def _gliner_entities(text: str) -> list[Entity]: - annotator = _get_gliner_annotator() +def _gliner_entities(text: str, model_name: str | None = None) -> list[Entity]: + annotator = ( + _get_gliner_annotator() + if model_name is None + else _get_gliner_annotator(model_name) + ) if isinstance(annotator, _UnavailableAnnotator): raise EngineNotAvailable(annotator.message) - payload = annotator.annotate(text) - return _entities_from_dict(text, payload, engine="gliner", confidence=0.8) + return _annotator_entities(text, annotator, "gliner", 0.8) @lru_cache(maxsize=1) @@ -398,15 +487,15 @@ def _get_spacy_annotator(): ) -@lru_cache(maxsize=1) -def _get_gliner_annotator(): +@lru_cache(maxsize=4) +def _get_gliner_annotator(model_name: str = "urchade/gliner_multi_pii-v1"): try: from .processing.text_processing.gliner_annotator import GLiNERAnnotator except ImportError as exc: return _UnavailableAnnotator(str(exc)) try: - annotator = GLiNERAnnotator.create() + annotator = GLiNERAnnotator.create(model_name=model_name) except ImportError as exc: return _UnavailableAnnotator(str(exc)) except Exception as exc: @@ -438,44 +527,39 @@ def _filter_entity_types( return [entity for entity in entities if entity.type in allowed] -# Python's re module backtracks; a quantified group containing another -# quantifier (e.g. ``(a+)+``) can take exponential time on adversarial -# input, and entity text can be attacker-influenced (LLM messages, tool -# output). Reject that construct outright rather than matching under it. -_NESTED_QUANTIFIER = re.compile( - r"\((?:[^()\\]|\\.)*(? list["re.Pattern[str]"]: +def _compile_allowlist_patterns(allowlist_patterns: Optional[list[str]]) -> list: compiled = [] + if isinstance(allowlist_patterns, str): + raise ValueError("allowlist_patterns must be a list of patterns") for raw in allowlist_patterns or []: - if len(raw) > MAX_ALLOWLIST_PATTERN_LENGTH: + if not isinstance(raw, str) or len(raw) > MAX_ALLOWLIST_PATTERN_LENGTH: raise ValueError( - "allowlist_patterns entries must be at most " - f"{MAX_ALLOWLIST_PATTERN_LENGTH} characters" - ) - if _NESTED_QUANTIFIER.search(raw): - raise ValueError( - "allowlist_patterns contains a quantified group with a nested " - f"quantifier ({raw!r}), which risks catastrophic backtracking; " - "rewrite the pattern without nesting quantifiers" + "allowlist_patterns entries must be strings of at most 512 characters" ) try: - compiled.append(re.compile(raw)) - except re.error as exc: + compiled.append(bounded_re.compile(raw)) + except bounded_re.error as exc: raise ValueError( f"allowlist_patterns contains an invalid regex: {raw!r} ({exc})" ) from None return compiled +def _allowlist_matches(pattern: object, value: str) -> bool: + if len(value) > MAX_PATTERN_SUBJECT_LENGTH: + return False + try: + return pattern.fullmatch(value, timeout=ALLOWLIST_MATCH_TIMEOUT) is not None + except TimeoutError: + # A slow exemption must never suppress a sensitive finding. + return False + + def _apply_allowlist( entities: list[Entity], allowlist: Optional[list[str]], @@ -498,11 +582,7 @@ def _apply_allowlist( entity for entity in entities if entity.text not in exact - and not any( - pattern.fullmatch(entity.text) - for pattern in patterns - if len(entity.text) <= MAX_PATTERN_SUBJECT_LENGTH - ) + and not any(_allowlist_matches(pattern, entity.text) for pattern in patterns) ] @@ -532,8 +612,7 @@ def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: An explicit ``backend`` wins over ``DATAFOG_BACKEND``. ``auto`` uses the native engine whenever Core is importable and degrades to Python otherwise. - Non-``regex`` engines always take the Python path, because the native - backend serves ``regex`` and model findings only. + Every engine uses this backend for structured PII; NER adds spans. """ requested = backend or os.environ.get(BACKEND_ENV) or "auto" normalized = requested.strip().lower() @@ -543,8 +622,6 @@ def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: "backend argument)" ) if normalized == "auto": - if engine != "regex": - return "python" return "rust" if core_available() else "python" return normalized @@ -552,8 +629,33 @@ def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: def _validate_backend(backend: str, engine: str) -> None: if backend not in {"python", "rust"}: raise ValueError("backend must be one of: python, rust") - if backend == "rust" and engine != "regex": - raise ValueError("backend='rust' supports only engine='regex'") + + +def _validate_entity_types( + entity_types: Optional[list[str]], backend: str, engine: str +) -> Optional[list[str]]: + if entity_types is None: + return None + if not isinstance(entity_types, (list, tuple)) or any( + not isinstance(value, str) or not value.strip() for value in entity_types + ): + raise ValueError("entity_types must be a list of entity labels") + requested = [_canonical_type(value) for value in entity_types] + if backend == "rust": + from ._core_capabilities import _read_capabilities + + supported, _, _ = _read_capabilities(_require_core().capabilities) + supported = supported | MODEL_ENTITY_TYPES + else: + supported = {_canonical_type(label) for label in RegexAnnotator.LABELS} + if engine != "regex": + supported |= NER_ENTITY_TYPES | {"DATE"} + unsupported = set(requested) - supported + if unsupported: + raise ValueError( + f"Unsupported entity_types for {backend}/{engine}: {', '.join(sorted(unsupported))}" + ) + return requested def scan( @@ -565,13 +667,15 @@ def scan( allowlist_patterns: Optional[list[str]] = None, *, backend: Optional[str] = None, + gliner_model: Optional[str] = None, ) -> ScanResult: """Scan text for PII entities. ``backend`` defaults to ``auto``: the native engine when ``datafog-core`` is importable, otherwise the pure-Python engine with a one-time warning. Set ``DATAFOG_BACKEND`` or pass ``backend="python"`` to force the fallback. The - native path serves ``engine="regex"``; non-regex engines always use Python. + selected backend supplies structured PII for every engine mode; optional + NER engines add entity spans. Probe names and address blocks come from the embedded model on the native path, so ``PERSON`` and ``ADDRESS`` work without spaCy or a model download. @@ -593,6 +697,7 @@ def scan( ) backend = resolve_backend(backend, engine) _validate_backend(backend, engine) + entity_types = _validate_entity_types(entity_types, backend, engine) if backend == "python" and requested_backend == "auto" and not core_available(): _warn_core_missing() @@ -634,7 +739,11 @@ def scan( if engine == "gliner" and _needs_ner(entity_types): try: - gliner_entities = _gliner_entities(text) + gliner_entities = ( + _gliner_entities(text) + if gliner_model is None + else _gliner_entities(text, gliner_model) + ) combined.extend(gliner_entities) engines_used.add("gliner") except EngineNotAvailable: @@ -649,13 +758,16 @@ def scan( if engine == "smart" and _needs_ner(entity_types): try: - gliner_entities = _gliner_entities(text) + gliner_entities = ( + _gliner_entities(text) + if gliner_model is None + else _gliner_entities(text, gliner_model) + ) combined.extend(gliner_entities) engines_used.add("gliner") - except EngineNotAvailable: + except EngineNotAvailable as exc: warnings.warn( - "GLiNER not available, smart scan falling back to spaCy. " - "Install with: pip install datafog[nlp-advanced]", + f"GLiNER unavailable; smart scan falling back to spaCy. {exc}", UserWarning, stacklevel=2, ) @@ -673,7 +785,7 @@ def scan( filtered = _filter_entity_types(combined, entity_types) filtered = _apply_allowlist(filtered, allowlist, allowlist_patterns) - deduped = _dedupe_entities(filtered) + deduped = _suppress_overlapping_entities(_dedupe_entities(filtered)) return ScanResult( entities=deduped, text=text, @@ -685,6 +797,9 @@ def redact( text: str, entities: list[Entity], strategy: str = "token", + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Redact PII entities from text. @@ -703,19 +818,36 @@ def redact( if strategy not in {"token", "mask", "hash", "pseudonymize"}: raise ValueError("strategy must be one of: token, mask, hash, pseudonymize") - redacted_text = text + if hash_key is not None and (not isinstance(hash_key, bytes) or len(hash_key) < 32): + raise ValueError("hash_key must contain at least 32 bytes") + if strategy == "hash": + try: + hashlib.new(hash_type).hexdigest() + except (ValueError, TypeError): + raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None + mapping: dict[str, str] = {} counters: dict[str, int] = {} pseudonym_by_value: dict[tuple[str, str], str] = {} - valid_entities = [ - entity - for entity in entities - if 0 <= entity.start < entity.end <= len(text) and entity.text - ] + valid_entities = [] + for entity in entities: + if ( + not 0 <= entity.start < entity.end <= len(text) + or not entity.text + or text[entity.start : entity.end] != entity.text + ): + raise ValueError( + "entities must contain valid ranges whose text matches the source" + ) + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", entity.type): + raise ValueError("entity types must be uppercase labels") + valid_entities.append(entity) # Returns non-overlapping spans sorted by document position. valid_entities = _suppress_overlapping_entities(valid_entities) + occupied_tokens = set(re.findall(r"\[[A-Z][A-Za-z0-9_]*\]", text)) + # Assign replacements in document order so the per-type counters used by # the token/pseudonymize strategies number the first occurrence _1; spans # are applied right-to-left below so earlier offsets stay valid. @@ -727,30 +859,62 @@ def redact( replacement = "*" * max(len(original), 1) # Distinct same-length values share a mask, so the mask string # cannot key the mapping without collisions. - counters[entity.type] = counters.get(entity.type, 0) + 1 - mapping_key = f"[{entity.type}_MASK_{counters[entity.type]}]" + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + mapping_key = f"[{entity.type}_MASK_{counters[entity.type]}]" + if mapping_key not in occupied_tokens: + break + occupied_tokens.add(mapping_key) elif strategy == "hash": - digest = hashlib.sha256(original.encode("utf-8")).hexdigest()[:12] - replacement = f"[{entity.type}_{digest}]" + encoded = original.encode("utf-8", "surrogatepass") + digest = ( + hmac.new(hash_key, encoded, hash_type).hexdigest() + if hash_key is not None + else hashlib.new(hash_type, encoded).hexdigest() + ) + key = (entity.type, original) + if key not in pseudonym_by_value: + candidate = f"[{entity.type}_{digest}]" + suffix = 1 + while candidate in occupied_tokens: + suffix += 1 + candidate = f"[{entity.type}_{digest}_{suffix}]" + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) + replacement = pseudonym_by_value[key] elif strategy == "pseudonymize": key = (entity.type, original) if key not in pseudonym_by_value: - counters[entity.type] = counters.get(entity.type, 0) + 1 - pseudonym_by_value[key] = ( - f"[{entity.type}_PSEUDO_{counters[entity.type]}]" - ) + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + candidate = f"[{entity.type}_PSEUDO_{counters[entity.type]}]" + if candidate not in occupied_tokens: + break + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) replacement = pseudonym_by_value[key] else: # token - counters[entity.type] = counters.get(entity.type, 0) + 1 - replacement = f"[{entity.type}_{counters[entity.type]}]" + key = (entity.type, original) + if key not in pseudonym_by_value: + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + candidate = f"[{entity.type}_{counters[entity.type]}]" + if candidate not in occupied_tokens: + break + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) + replacement = pseudonym_by_value[key] replacements.append((entity, replacement)) mapping[mapping_key if mapping_key is not None else replacement] = original - for entity, replacement in reversed(replacements): - redacted_text = ( - redacted_text[: entity.start] + replacement + redacted_text[entity.end :] - ) + parts = [] + offset = 0 + for entity, replacement in replacements: + parts.extend((text[offset : entity.start], replacement)) + offset = entity.end + parts.append(text[offset:]) + redacted_text = "".join(parts) return RedactResult( redacted_text=redacted_text, @@ -769,6 +933,8 @@ def scan_and_redact( allowlist_patterns: Optional[list[str]] = None, *, backend: Optional[str] = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Scan with the selected backend, then apply legacy Python redaction. @@ -783,4 +949,10 @@ def scan_and_redact( allowlist_patterns=allowlist_patterns, backend=backend, ) - return redact(text=text, entities=scan_result.entities, strategy=strategy) + return redact( + text=text, + entities=scan_result.entities, + strategy=strategy, + hash_type=hash_type, + hash_key=hash_key, + ) diff --git a/datafog/processing/text_processing/gliner_annotator.py b/datafog/processing/text_processing/gliner_annotator.py index 5a195f89..b871cf63 100644 --- a/datafog/processing/text_processing/gliner_annotator.py +++ b/datafog/processing/text_processing/gliner_annotator.py @@ -10,6 +10,8 @@ from pydantic import BaseModel, ConfigDict +from .regex_annotator.regex_annotator import AnnotationResult, Span + # Default entity types for PII detection using GLiNER # These can be customized based on specific use cases DEFAULT_PII_ENTITIES = [ @@ -92,55 +94,37 @@ def create( f"datafog download-model {model_name} --engine gliner" ) from e - def annotate(self, text: str) -> Dict[str, List[str]]: - """ - Annotate text for PII entities using GLiNER. - - Args: - text: Text to analyze for PII entities - - Returns: - Dictionary mapping entity types to lists of detected entities - """ - try: - if not text: - return { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } - - if len(text) > MAXIMAL_STRING_SIZE: - text = text[:MAXIMAL_STRING_SIZE] - logging.warning(f"Text truncated to {MAXIMAL_STRING_SIZE} characters") - - # Predict entities using GLiNER - entities = self.model.predict_entities(text, self.entity_types) - - # Organize results by entity type - classified_entities: Dict[str, List[str]] = { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } - + def annotate_with_spans(self, text: str): + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text) > MAXIMAL_STRING_SIZE: + raise ValueError( + f"GLiNER input exceeds {MAXIMAL_STRING_SIZE} characters; split the document before scanning" + ) + classified_entities = { + label.upper().replace(" ", "_"): [] for label in self.entity_types + } + spans = [] + if text: + try: + entities = self.model.predict_entities(text, self.entity_types) + except Exception as exc: + raise RuntimeError("GLiNER annotation failed") from exc for entity in entities: - entity_label = entity["label"].upper().replace(" ", "_") - entity_text = entity["text"] + label = entity["label"].upper().replace(" ", "_") + classified_entities.setdefault(label, []).append(entity["text"]) + spans.append( + Span( + label=label, + start=entity["start"], + end=entity["end"], + text=entity["text"], + ) + ) + return classified_entities, AnnotationResult(text=text, spans=spans) - if entity_label in classified_entities: - classified_entities[entity_label].append(entity_text) - else: - # Handle cases where GLiNER returns entity types not in our list - classified_entities[entity_label] = [entity_text] - - return classified_entities - - except Exception as e: - logging.error(f"Error processing text with GLiNER: {str(e)}") - # Return empty annotations in case of error - return { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } + def annotate(self, text: str) -> Dict[str, List[str]]: + return self.annotate_with_spans(text)[0] def set_entity_types(self, entity_types: List[str]) -> None: """ diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py index 44dfa2e0..2c132ac7 100644 --- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py +++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py @@ -40,11 +40,14 @@ class RegexAnnotator: ] LABELS = BASE_LABELS + GERMAN_LABELS DEFAULT_LABELS = BASE_LABELS - SUPPORTED_LOCALES = {"de", "de-de", "de_de"} + SUPPORTED_LOCALES = {"de", "de-de", "de_de", "en", "en-us", "fr"} LOCALE_LABELS = { "de": GERMAN_LABELS, "de-de": GERMAN_LABELS, "de_de": GERMAN_LABELS, + "en": [], + "en-us": [], + "fr": [], } def __init__( @@ -63,14 +66,14 @@ def __init__( # Note: This is broader than the spec to catch more potential emails "EMAIL": re.compile( r""" - (? (? (? (? (? tuple[str, . locales = [locales] normalized = [] for locale in locales: + if not isinstance(locale, str) or not locale.strip(): + raise ValueError("locales must contain nonempty locale identifiers") value = locale.strip().lower() - if not value: - continue if value not in cls.SUPPORTED_LOCALES: allowed = ", ".join(sorted(cls.SUPPORTED_LOCALES)) raise ValueError(f"locale must be one of: {allowed}") diff --git a/datafog/processing/text_processing/spacy_pii_annotator.py b/datafog/processing/text_processing/spacy_pii_annotator.py index d2d1d765..1cb2cbd2 100644 --- a/datafog/processing/text_processing/spacy_pii_annotator.py +++ b/datafog/processing/text_processing/spacy_pii_annotator.py @@ -1,8 +1,9 @@ -import logging from typing import Any, Dict, List from pydantic import BaseModel, ConfigDict +from .regex_annotator.regex_annotator import AnnotationResult, Span + PII_ANNOTATION_LABELS = [ "CARDINAL", "DATE", @@ -28,7 +29,7 @@ class SpacyPIIAnnotator(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) + model_config = ConfigDict(arbitrary_types_allowed=True, protected_namespaces=()) nlp: Any model_name: str = DEFAULT_SPACY_MODEL @@ -53,22 +54,32 @@ def create(cls, model_name: str = DEFAULT_SPACY_MODEL) -> "SpacyPIIAnnotator": return cls(nlp=nlp, model_name=model_name) - def annotate(self, text: str) -> Dict[str, List[str]]: - try: - if not text: - return {label: [] for label in PII_ANNOTATION_LABELS} - if len(text) > MAXIMAL_STRING_SIZE: - text = text[:MAXIMAL_STRING_SIZE] - doc = self.nlp(text) - classified_entities: Dict[str, List[str]] = { - label: [] for label in PII_ANNOTATION_LABELS - } + def annotate_with_spans(self, text: str): + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text) > MAXIMAL_STRING_SIZE: + raise ValueError( + f"spaCy input exceeds {MAXIMAL_STRING_SIZE} characters; split the document before scanning" + ) + classified_entities = {label: [] for label in PII_ANNOTATION_LABELS} + spans = [] + if text: + try: + doc = self.nlp(text) + except Exception as exc: + raise RuntimeError("spaCy annotation failed") from exc for ent in doc.ents: if ent.label_ in classified_entities: classified_entities[ent.label_].append(ent.text) - return classified_entities - except Exception as e: - logging.error(f"Error processing text for PII annotations: {str(e)}") - return { - label: [] for label in PII_ANNOTATION_LABELS - } # Return empty annotations in case of error + spans.append( + Span( + label=ent.label_, + start=ent.start_char, + end=ent.end_char, + text=ent.text, + ) + ) + return classified_entities, AnnotationResult(text=text, spans=spans) + + def annotate(self, text: str) -> Dict[str, List[str]]: + return self.annotate_with_spans(text)[0] diff --git a/datafog/services/text_service.py b/datafog/services/text_service.py index 6b80b26d..eee7ca90 100644 --- a/datafog/services/text_service.py +++ b/datafog/services/text_service.py @@ -61,10 +61,17 @@ def __init__( German structured identifiers. Raises: - AssertionError: If an invalid engine type is provided + ValueError: If the engine or text_chunk_length is invalid ImportError: If spacy/gliner engine is requested but corresponding extra is not installed """ - assert engine in {"regex", "spacy", "gliner", "auto", "smart"}, "Invalid engine" + if engine not in {"regex", "spacy", "gliner", "auto", "smart"}: + raise ValueError("engine must be one of: regex, spacy, gliner, auto, smart") + if ( + not isinstance(text_chunk_length, int) + or isinstance(text_chunk_length, bool) + or text_chunk_length <= 0 + ): + raise ValueError("text_chunk_length must be a positive integer") self.engine = engine self.text_chunk_length = text_chunk_length self.gliner_model = gliner_model @@ -379,17 +386,32 @@ def annotate_text_sync( _start = _time.monotonic() - if len(text) <= self.text_chunk_length: - # Single chunk processing - result = self._annotate_single_chunk(text, structured) + from datafog.engine import scan + + # Structured PII is scanned as a complete document. Fixed-size chunks + # lose entities at their boundaries; native model work chunks separately. + options = { + "engine": "smart" if self.engine == "auto" else self.engine, + "locales": self.locales, + } + if self.gliner_model != "urchade/gliner_multi_pii-v1": + options["gliner_model"] = self.gliner_model + entities = scan(text, **options).entities + if structured: + SpanClass = _get_span_class() + result = [ + SpanClass( + label=entity.type, + start=entity.start, + end=entity.end, + text=entity.text, + ) + for entity in entities + ] else: - # Multi-chunk processing - chunks = self._chunk_text(text) - - if structured: - result = self._annotate_multiple_chunks_structured(chunks) - else: - result = self._annotate_multiple_chunks_dict(chunks) + result = {} + for entity in entities: + result.setdefault(entity.type, []).append(entity.text) try: from datafog.telemetry import ( diff --git a/tests/test_allowlist.py b/tests/test_allowlist.py index f737a9f2..fc4b2865 100644 --- a/tests/test_allowlist.py +++ b/tests/test_allowlist.py @@ -93,13 +93,15 @@ def test_patterns_and_values_combine(self): class TestReDoSGuards: - def test_catastrophic_pattern_rejected(self): - with pytest.raises(ValueError, match="catastrophic backtracking"): - datafog.scan("text", engine="regex", allowlist_patterns=[r"(a+)+$"]) - - def test_nested_star_rejected(self): - with pytest.raises(ValueError, match="catastrophic backtracking"): - datafog.scan("text", engine="regex", allowlist_patterns=[r"(.*)*"]) + def test_nested_quantifier_uses_bounded_matching(self): + assert datafog.scan( + "a@b.com", engine="regex", allowlist_patterns=[r"(a+)+$"] + ).entities + + def test_safe_nested_star_is_supported(self): + assert not datafog.scan( + "a@b.com", engine="regex", allowlist_patterns=[r"(.*)*"] + ).entities def test_overlong_pattern_rejected(self): with pytest.raises(ValueError, match="at most"): diff --git a/tests/test_backend_resolution.py b/tests/test_backend_resolution.py index c6cd6d60..55abb06d 100644 --- a/tests/test_backend_resolution.py +++ b/tests/test_backend_resolution.py @@ -33,10 +33,10 @@ def test_auto_resolves_to_python_without_core(without_core): assert engine.resolve_backend(None, "regex") == "python" -def test_auto_keeps_non_regex_engines_on_python(): +def test_auto_uses_native_structured_layer_for_every_engine(): pytest.importorskip("datafog_core") for engine_name in ("smart", "spacy", "gliner"): - assert engine.resolve_backend(None, engine_name) == "python" + assert engine.resolve_backend(None, engine_name) == "rust" def test_python_fallback_warns_once_then_stays_quiet(without_core): diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index 8cd218a9..b0e2ae9f 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -210,9 +210,8 @@ def test_contextual_npi_survives_the_legacy_overlap_policy(): def test_structured_only_person_served_by_the_embedded_model(): """`PERSON` is structured-only in the rule inventory but the embedded CNN covers prose, so text selection succeeds instead of raising.""" - assert not datafog.scan( - "Jane Doe", entity_types=["PERSON"], backend="python" - ).entities + with pytest.raises(ValueError, match="Unsupported entity_types"): + datafog.scan("Jane Doe", entity_types=["PERSON"], backend="python") result = datafog.scan("Jane Doe\n14 Penny Lane", entity_types=["PERSON", "ADDRESS"]) assert [item.type for item in result.entities] == ["PERSON", "ADDRESS"] diff --git a/tests/test_engine_api.py b/tests/test_engine_api.py index 2a43e147..35aebe2a 100644 --- a/tests/test_engine_api.py +++ b/tests/test_engine_api.py @@ -198,7 +198,7 @@ def test_redact_invalid_strategy_raises_value_error() -> None: redact("test", entities=[], strategy="invalid") -def test_redact_ignores_invalid_spans() -> None: +def test_redact_rejects_invalid_spans() -> None: text = "hello" entities = [ Entity( @@ -219,9 +219,8 @@ def test_redact_ignores_invalid_spans() -> None: ), ] - result = redact(text=text, entities=entities, strategy="token") - assert result.redacted_text == text - assert result.mapping == {} + with pytest.raises(ValueError, match="valid ranges"): + redact(text=text, entities=entities, strategy="token") def test_scan_and_redact_combines_operations() -> None: diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py new file mode 100644 index 00000000..b10e7d33 --- /dev/null +++ b/tests/test_review_engine.py @@ -0,0 +1,306 @@ +"""Regression contracts for the 4.11 review's engine and annotation findings.""" + +import hashlib +import hmac +import random +from types import SimpleNamespace + +import pytest + +from datafog import engine +from datafog.processing.text_processing.gliner_annotator import GLiNERAnnotator +from datafog.processing.text_processing.regex_annotator.regex_annotator import ( + RegexAnnotator, +) +from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator +from datafog.services.text_service import TextService + + +def test_native_surrogate_normalization_preserves_source_and_offsets(): + text = "\ud800😀 ssn 123-45-6789 \udfff" + result = engine.scan(text, engine="regex", backend="rust") + assert result.text == text + assert any(entity.type == "SSN" for entity in result.entities) + assert all(text[item.start : item.end] == item.text for item in result.entities) + assert "123-45-6789" not in engine.redact(text, result.entities).redacted_text + + +def test_interval_selection_preserves_longest_and_tiebreak_contract(): + rng = random.Random(113) + for _ in range(30): + entities = [ + engine.Entity( + rng.choice(["PHONE", "SSN"]), + "", + start, + start + rng.randint(1, 30), + rng.random(), + "regex", + ) + for start in [rng.randint(0, 200) for _ in range(100)] + ] + expected = [] + for entity in sorted( + entities, + key=lambda item: ( + -engine._entity_length(item), + -engine.ENTITY_TYPE_PRIORITY.get(item.type, 0), + -item.confidence, + item.start, + item.end, + item.type, + ), + ): + if not any(engine._entities_overlap(entity, kept) for kept in expected): + expected.append(entity) + assert engine._suppress_overlapping_entities(entities) == sorted( + expected, key=lambda item: (item.start, item.end, item.type) + ) + + +@pytest.mark.parametrize("structured", [False, True]) +@pytest.mark.parametrize("value", ["john.doe@example.com", "123-45-6789"]) +def test_text_service_retains_findings_at_chunk_boundaries(value, structured): + text = "x" * 996 + " " + value + " tail" + result = TextService(text_chunk_length=1000).annotate_text_sync(text, structured) + if structured: + assert any( + item.text == value and text[item.start : item.end] == value + for item in result + ) + else: + assert any(value in items for items in result.values()) + + +def test_spacy_uses_original_offsets_and_does_not_turn_relative_dates_into_pii( + monkeypatch, +): + text = "Annual report prepared by Ann yesterday." + spans = [ + SimpleNamespace(label_="PERSON", text="Ann", start_char=26, end_char=29), + SimpleNamespace(label_="DATE", text="yesterday", start_char=30, end_char=39), + ] + annotator = SpacyPIIAnnotator(nlp=lambda _: SimpleNamespace(ents=spans)) + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: annotator) + result = engine.scan_and_redact(text, engine="spacy") + assert result.redacted_text == "Annual report prepared by [PERSON_1] yesterday." + + +def test_legacy_string_annotator_does_not_match_names_inside_words(): + result = engine._entities_from_dict( + "Annual report by Ann.", {"PERSON": ["Ann"]}, "spacy", 0.7 + ) + assert [(item.start, item.end) for item in result] == [(17, 20)] + + +@pytest.mark.parametrize("mode", ["smart", "spacy", "gliner"]) +def test_every_engine_keeps_native_structured_detection(monkeypatch, mode): + monkeypatch.setattr(engine, "_gliner_entities", lambda _: []) + monkeypatch.setattr(engine, "_spacy_entities", lambda _: []) + text = "card 6011111111111117 ip 2001:db8::1" + result = engine.scan_and_redact(text, engine=mode, backend="rust") + assert {item.type for item in result.entities} == {"CREDIT_CARD", "IP_ADDRESS"} + assert "6011111111111117" not in result.redacted_text + assert "2001:db8::1" not in result.redacted_text + + +@pytest.mark.parametrize("labels", ["EMAIL", ["BOGUS"], [1], [""]]) +def test_entity_type_mistakes_fail_explicitly(labels): + with pytest.raises(ValueError, match="entity_types"): + engine.scan("john@example.com", entity_types=labels) + + +@pytest.mark.parametrize("label", ["JWT", "PERSON", "ORGANIZATION", "UUID"]) +def test_unavailable_fallback_labels_raise(label): + with pytest.raises(ValueError, match="Unsupported entity_types"): + engine.scan( + "john@example.com", engine="regex", backend="python", entity_types=[label] + ) + + +@pytest.mark.parametrize("pattern", [r"(a|a)+$", r"((a+))+$"]) +def test_pathological_allowlist_cannot_suppress_a_finding(pattern): + value = "a" * 300 + "@b.com" + entity = engine.Entity("EMAIL", value, 0, len(value), 1.0, "regex") + assert engine._apply_allowlist([entity], None, [pattern]) == [entity] + + +@pytest.mark.parametrize( + "pattern,value", + [(r"(\d{3}){2}", "123123"), (r"(?:\d{3}-){2}\d{4}", "415-555-2671")], +) +def test_safe_quantified_allowlist_patterns_are_supported(pattern, value): + entity = engine.Entity("PHONE", value, 0, len(value), 1.0, "regex") + assert engine._apply_allowlist([entity], None, [pattern]) == [] + + +@pytest.mark.parametrize( + "annotator", + [ + SpacyPIIAnnotator( + nlp=lambda _: (_ for _ in ()).throw(RuntimeError("raw secret")) + ), + GLiNERAnnotator( + model=SimpleNamespace( + predict_entities=lambda *_: (_ for _ in ()).throw( + RuntimeError("raw secret") + ) + ), + entity_types=["person"], + model_name="test", + ), + ], +) +def test_ner_failures_and_oversize_inputs_are_explicit(annotator): + with pytest.raises(RuntimeError, match="annotation failed"): + annotator.annotate("secret") + with pytest.raises(ValueError, match="exceeds"): + annotator.annotate("x" * 1_000_001) + + +@pytest.mark.parametrize( + "label", ["passport number", "medical record number", "license number", "url"] +) +def test_gliner_extra_labels_and_original_offsets_are_preserved(monkeypatch, label): + text = "prefix SECRET suffix" + annotator = GLiNERAnnotator( + model=SimpleNamespace( + predict_entities=lambda *_: [ + {"label": label, "text": "SECRET", "start": 7, "end": 13} + ] + ), + entity_types=[label], + model_name="test", + ) + monkeypatch.setattr(engine, "_get_gliner_annotator", lambda: annotator) + result = engine.scan( + text, engine="gliner", entity_types=[label.upper().replace(" ", "_")] + ) + assert len(result.entities) == 1 + assert result.entities[0].text == "SECRET" + assert result.entities[0].start == 7 + + +def test_model_chunks_are_bounded_and_keep_document_offsets(): + text = "😀" * 9000 + " Jane Doe" + calls = [] + + def scan_model(chunk): + calls.append(chunk) + start = chunk.find("Jane Doe") + findings = ( + [] + if start < 0 + else [ + SimpleNamespace( + entity_type="PERSON", + codepoint_range=SimpleNamespace(start=start, end=start + 8), + matched_text="Jane Doe", + ) + ] + ) + return SimpleNamespace(findings=findings) + + findings = engine._cnn_findings( + SimpleNamespace(scan_model=scan_model), {"PERSON"}, text + ) + assert len(calls) >= 3 and max(len(chunk.encode()) for chunk in calls) <= 16384 + assert any( + offset + item.codepoint_range.start == text.index("Jane Doe") + for offset, item in findings + ) + + +def test_token_collision_and_repeated_value_mapping(): + text = "[EMAIL_1] a@b.com and a@b.com" + result = engine.scan_and_redact(text, engine="regex") + assert result.redacted_text == "[EMAIL_1] [EMAIL_2] and [EMAIL_2]" + assert result.mapping == {"[EMAIL_2]": "a@b.com"} + + +@pytest.mark.parametrize( + "entity", + [ + engine.Entity("EMAIL", "", 0, 1, 1.0, "regex"), + engine.Entity("EMAIL", "a", 0, 100, 1.0, "regex"), + engine.Entity("EMAIL", "wrong", 0, 1, 1.0, "regex"), + ], +) +def test_invalid_explicit_spans_are_rejected(entity): + with pytest.raises(ValueError, match="valid ranges"): + engine.redact("a", [entity]) + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256", "sha3_256"]) +def test_hash_algorithm_and_hmac_are_honored(algorithm): + text = "a@b.com" + entities = engine.scan(text, engine="regex").entities + plain = engine.redact(text, entities, strategy="hash", hash_type=algorithm) + assert hashlib.new(algorithm, text.encode()).hexdigest() in plain.redacted_text + key = bytes(range(32)) + keyed = engine.redact( + text, entities, strategy="hash", hash_type=algorithm, hash_key=key + ) + assert hmac.new(key, text.encode(), algorithm).hexdigest() in keyed.redacted_text + assert plain != keyed + + +def test_invalid_hash_parameters_fail(): + with pytest.raises(ValueError, match="hash_key"): + engine.redact("text", [], strategy="hash", hash_key=b"short") + with pytest.raises(ValueError, match="hash_type"): + engine.redact("text", [], strategy="hash", hash_type="bogus") + + +@pytest.mark.parametrize("locales", [[""], [1]]) +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_invalid_locales_are_consistently_rejected(locales, backend): + with pytest.raises(ValueError, match="locale"): + engine.scan("safe", engine="regex", locales=locales, backend=backend) + + +@pytest.mark.parametrize("locale", ["fr", "en-US"]) +def test_fallback_accepts_locale_tags_without_extra_detectors(locale): + assert engine.scan( + "a@b.com", engine="regex", locales=[locale], backend="python" + ).entities + + +def test_unicode_email_in_fallback(): + result = engine.scan("mail jörg@müller.de", engine="regex", backend="python") + assert result.entities[0].text == "jörg@müller.de" + + +@pytest.mark.parametrize( + "options", + [ + {"engine": "bogus"}, + {"text_chunk_length": 0}, + {"text_chunk_length": -1}, + {"text_chunk_length": True}, + ], +) +def test_text_service_argument_validation(options): + with pytest.raises(ValueError): + TextService(**options) + + +def test_fallback_adversarial_patterns_complete(): + annotator = RegexAnnotator(locales=["de"]) + for value in [ + "!" * 80000, + "#a" * 40000, + "Steuer-ID" + " " * 80000 + "x", + "Passport" + " " * 80000, + ]: + assert not any(annotator.annotate(value).values()) + + +def test_hash_marker_cannot_collide_with_literal_input(): + value = "a@b.com" + digest = hashlib.sha256(value.encode()).hexdigest() + marker = f"[EMAIL_{digest}]" + text = f"{marker} {value} {value}" + result = engine.scan_and_redact(text, engine="regex", strategy="hash") + assert result.mapping == {f"[EMAIL_{digest}_2]": value} + assert result.redacted_text == f"{marker} [EMAIL_{digest}_2] [EMAIL_{digest}_2]" From f8700e3997461da0f353d7831bb06c62e9a81b8b Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:56:06 -0800 Subject: [PATCH 08/23] fix(regex): keep email context boundaries linear and complete --- .../regex_annotator/regex_annotator.py | 23 ++++++++++---- tests/test_review_engine.py | 20 ++++++++++++ tests/test_rust_backend.py | 31 +++++++++++-------- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py index 2c132ac7..03641e82 100644 --- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py +++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py @@ -67,8 +67,7 @@ def __init__( "EMAIL": re.compile( r""" (? str: - return match.groupdict().get("value") or match.group() + @classmethod + def _match_text(cls, match: Match[str]) -> str: + start, end = cls._match_span(match) + return match.string[start:end] @staticmethod def _match_span(match: Match[str]) -> tuple[int, int]: if "value" in match.groupdict() and match.group("value") is not None: return match.start("value"), match.end("value") - return match.start(), match.end() + start, end = match.start(), match.end() + value = match.group() + if "@" in value and any(char.isalnum() for char in value[: value.rfind("@")]): + local = value[: value.rfind("@")] + # Query/assignment separators and prose wrappers are context, + # not part of the mailbox. Consume the whole lexical run before + # trimming so failed candidates never retry every suffix. + boundary = max((local.rfind(char) for char in "&?=|/"), default=-1) + start += boundary + 1 + while start < end and match.string[start] in ".!?'*{}`": + start += 1 + return start, end @classmethod def create(cls, **kwargs) -> "RegexAnnotator": diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py index b10e7d33..020f0935 100644 --- a/tests/test_review_engine.py +++ b/tests/test_review_engine.py @@ -304,3 +304,23 @@ def test_hash_marker_cannot_collide_with_literal_input(): result = engine.scan_and_redact(text, engine="regex", strategy="hash") assert result.mapping == {f"[EMAIL_{digest}_2]": value} assert result.redacted_text == f"{marker} [EMAIL_{digest}_2] [EMAIL_{digest}_2]" + + +@pytest.mark.parametrize( + "text", + [ + "?" * 11 + "xiaoming@example.cn", + '{"note":"email=alice@example.com"}', + "email = 'dev@example.com'", + "phone=415-555-2671&email=bob@example.com", + ], +) +def test_fallback_email_context_is_trimmed_without_losing_mailboxes(text): + result = engine.scan(text, engine="regex", backend="python", entity_types=["EMAIL"]) + assert len(result.entities) == 1 + assert result.entities[0].text in { + "xiaoming@example.cn", + "alice@example.com", + "dev@example.com", + "bob@example.com", + } diff --git a/tests/test_rust_backend.py b/tests/test_rust_backend.py index 34da6a21..3cbdd607 100644 --- a/tests/test_rust_backend.py +++ b/tests/test_rust_backend.py @@ -66,15 +66,11 @@ def test_selection_alias_and_empty_selection(native, selection): assert len(result.entities) == 1 -def test_unknown_selection_preserves_legacy_empty_result(native): +def test_unknown_selection_is_rejected(native): findings, _ = native findings.append(finding("EMAIL", "a@example.com", 0, 13)) - assert ( - engine.scan( - "a@example.com", "regex", entity_types=["UNKNOWN"], backend="rust" - ).entities - == [] - ) + with pytest.raises(ValueError, match="Unsupported entity_types"): + engine.scan("a@example.com", "regex", entity_types=["UNKNOWN"], backend="rust") def test_unknown_native_label_preserved_without_silently_dropping_pii(native): @@ -84,7 +80,15 @@ def test_unknown_native_label_preserved_without_silently_dropping_pii(native): assert [item.type for item in result.entities] == ["FUTURE_LABEL"] -def test_python_overlap_priority_precedes_selection(native): +def test_python_overlap_priority_precedes_selection(native, monkeypatch): + metadata = capability_fixture() + metadata["supported_entities"].extend(["PHONE", "CREDIT_CARD"]) + for label in ("PHONE", "CREDIT_CARD"): + metadata["entities"][label] = { + "scopes": ["text"], + "activation": {"kind": "default"}, + } + monkeypatch.setattr(sys.modules["datafog_core"], "capabilities", lambda: metadata) findings, _ = native text = "1234567890123456" findings.extend( @@ -116,7 +120,7 @@ def test_python_allowlist_semantics(native, kwargs, count): ) -@pytest.mark.parametrize("pattern", ["[", "(a+)+", "x" * 513]) +@pytest.mark.parametrize("pattern", ["[", "x" * 513]) def test_allowlist_validation_before_native(native, pattern): _, calls = native with pytest.raises(ValueError): @@ -146,11 +150,12 @@ def test_unknown_locale_validation_preserved(native): @pytest.mark.parametrize("name", ["smart", "spacy", "gliner"]) -def test_nonregex_engine_rejected(native, name): +def test_nonregex_engines_use_native_structured_stage(native, name, monkeypatch): _, calls = native - with pytest.raises(ValueError, match="only engine='regex'"): - engine.scan("", name, backend="rust") - assert not calls + monkeypatch.setattr(engine, "_spacy_entities", lambda _: []) + monkeypatch.setattr(engine, "_gliner_entities", lambda _: []) + assert not engine.scan("", name, backend="rust").entities + assert calls == [""] def test_bad_backend_rejected(monkeypatch): From 67730c9662ff4b39cbaa0ad4add6a6a373dbedfd Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:00:22 -0800 Subject: [PATCH 09/23] fix(guardrails): scan complete requests and buffered streaming output --- datafog/agent.py | 24 +- datafog/integrations/claude_code.py | 35 +- datafog/integrations/litellm_guardrail.py | 306 +++++++++++---- tests/test_review_guardrails.py | 434 ++++++++++++++++++++++ 4 files changed, 717 insertions(+), 82 deletions(-) create mode 100644 tests/test_review_guardrails.py diff --git a/datafog/agent.py b/datafog/agent.py index e3cd7bdb..d8bfe56e 100644 --- a/datafog/agent.py +++ b/datafog/agent.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect import warnings from contextlib import contextmanager from dataclasses import dataclass @@ -101,14 +102,27 @@ def filter(self, text: str) -> RedactResult: return result def __call__(self, fn: F) -> F: - """Decorator that applies guardrail filtering to string return values.""" + """Filter string results, awaiting coroutine functions before scanning. + + Other result types raise TypeError rather than escaping unscanned. + """ + + def filtered(output: Any) -> str: + if not isinstance(output, str): + raise TypeError("A guarded function must return a string") + return self.filter(output).redacted_text + + if inspect.iscoroutinefunction(fn): + + @wraps(fn) + async def async_wrapped(*args: Any, **kwargs: Any) -> str: + return filtered(await fn(*args, **kwargs)) + + return async_wrapped # type: ignore[return-value] @wraps(fn) def wrapped(*args: Any, **kwargs: Any) -> Any: - output = fn(*args, **kwargs) - if isinstance(output, str): - return self.filter(output).redacted_text - return output + return filtered(fn(*args, **kwargs)) return wrapped # type: ignore[return-value] diff --git a/datafog/integrations/claude_code.py b/datafog/integrations/claude_code.py index 28c4e787..5eb72fb3 100644 --- a/datafog/integrations/claude_code.py +++ b/datafog/integrations/claude_code.py @@ -22,8 +22,9 @@ whose full text matches are exempt (note: a pattern containing a comma cannot be expressed here). -Failure policy: fail open. A hook bug must never brick a Claude Code -session, so any unexpected error exits non-blocking with no output. +Oversized inputs receive the configured ask/deny decision rather than proceeding +with an unscanned tail. Unexpected hook errors fail open with a non-blocking +error, so a hook bug does not prevent using the Claude Code session. """ import json @@ -50,6 +51,10 @@ _EXIT_FAIL_OPEN = 1 +class ScanLimitExceeded(ValueError): + """The payload cannot be fully inspected within the configured budget.""" + + def _entity_types(env: Mapping[str, str]) -> list[str]: raw = env.get("DATAFOG_HOOK_ENTITIES", "") parsed = [t.strip().upper() for t in raw.split(",") if t.strip()] @@ -82,6 +87,7 @@ def _iter_strings(value: Any) -> Iterator[str]: yield current elif isinstance(current, dict): stack.extend(current.values()) + stack.extend(current.keys()) elif isinstance(current, (list, tuple)): stack.extend(current) @@ -98,9 +104,9 @@ def _scan_findings( counts: dict[str, int] = {} total_budget = TOTAL_SCAN_CHARS for text in _iter_strings(value): - if total_budget <= 0: - break - chunk = text[: min(MAX_SCAN_CHARS, total_budget)] + if len(text) > MAX_SCAN_CHARS or len(text) > total_budget: + raise ScanLimitExceeded("Input exceeds the scan budget") + chunk = text total_budget -= len(chunk) result = datafog.scan( chunk, @@ -191,8 +197,20 @@ def run(payload: dict, env: Mapping[str, str]) -> tuple[int, str]: if handler is None: return _EXIT_OK, "" return _EXIT_OK, handler(payload, env) + except ScanLimitExceeded: + event = payload.get("hook_event_name") + reason = "DataFog PII firewall: input exceeds the scan budget and was not fully scanned. Split the input before proceeding." + if event == "PreToolUse": + return _EXIT_OK, _emit( + event, + { + "permissionDecision": _action(env), + "permissionDecisionReason": reason, + }, + ) + return _EXIT_OK, _emit(event, {"additionalContext": reason}) except Exception as exc: # noqa: BLE001 — fail open by design - print(f"datafog-hook error (fail-open): {exc}", file=sys.stderr) + print(f"datafog-hook error (fail-open): {type(exc).__name__}", file=sys.stderr) return _EXIT_FAIL_OPEN, "" @@ -206,7 +224,10 @@ def main() -> None: if not isinstance(payload, dict): payload = {} except Exception as exc: # noqa: BLE001 — fail open by design - print(f"datafog-hook: invalid hook payload (fail-open): {exc}", file=sys.stderr) + print( + f"datafog-hook: invalid hook payload (fail-open): {type(exc).__name__}", + file=sys.stderr, + ) sys.exit(_EXIT_FAIL_OPEN) code, stdout = run(payload, os.environ) diff --git a/datafog/integrations/litellm_guardrail.py b/datafog/integrations/litellm_guardrail.py index 7e33ed55..5611bec9 100644 --- a/datafog/integrations/litellm_guardrail.py +++ b/datafog/integrations/litellm_guardrail.py @@ -30,7 +30,9 @@ ``datafog`` core; the LiteLLM proxy, where this runs, always ships fastapi). """ +import copy import logging +import re from typing import Any, Optional from fastapi import HTTPException @@ -42,15 +44,74 @@ VALID_ACTIONS = {"redact", "block"} VALID_FAIL_POLICIES = {"open", "closed"} +MAX_STREAM_CHARS = 8_000_000 +MAX_STREAM_CHUNKS = 100_000 logger = logging.getLogger(__name__) +class _RedactionSession: + """Token assignments belong to one request or response, never the adapter.""" + + def __init__(self, strings=()): + self.tokens: dict[tuple[str, str], str] = {} + self.counters: dict[str, int] = {} + self.reserved = { + token for text in strings for token in re.findall(r"\[[A-Z_]+_\d+\]", text) + } + + def token(self, entity): + key = (entity.type, entity.text) + if key not in self.tokens: + number = self.counters.get(entity.type, 0) + 1 + while f"[{entity.type}_{number}]" in self.reserved: + number += 1 + token = f"[{entity.type}_{number}]" + self.counters[entity.type] = number + self.tokens[key] = token + self.reserved.add(token) + return self.tokens[key] + + +def _strings(value): + stack = [value] + visited = set() + while stack: + item = stack.pop() + if isinstance(item, (dict, list)): + if id(item) in visited: + continue + visited.add(id(item)) + if isinstance(item, str): + yield item + elif isinstance(item, dict): + stack.extend(item.values()) + stack.extend(item.keys()) + elif isinstance(item, list): + stack.extend(item) + + +def _get(value, name, default=None): + return ( + value.get(name, default) + if isinstance(value, dict) + else getattr(value, name, default) + ) + + +def _set(value, name, item): + if isinstance(value, dict): + value[name] = item + else: + setattr(value, name, item) + + def _redact_text( text: str, entity_types: list[str], allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, + session: _RedactionSession | None = None, ) -> tuple[str, dict[str, int]]: """Redact ``text``; return (redacted_text, counts per entity type).""" import datafog @@ -65,7 +126,14 @@ def _redact_text( counts: dict[str, int] = {} for entity in result.entities: counts[entity.type] = counts.get(entity.type, 0) + 1 - return result.redacted_text, counts + session = session or _RedactionSession([text]) + parts = [] + cursor = 0 + for entity in sorted(result.entities, key=lambda entity: entity.start): + parts.extend((text[cursor : entity.start], session.token(entity))) + cursor = entity.end + parts.append(text[cursor:]) + return "".join(parts), counts def _summary(counts: dict[str, int]) -> str: @@ -97,39 +165,51 @@ def __init__( self.allowlist_patterns = allowlist_patterns super().__init__(**kwargs) - def _process_content(self, content: Any) -> tuple[Any, dict[str, int]]: - """Redact a message content value (str or list of content parts).""" + def _process_content( + self, content: Any, session=None + ) -> tuple[Any, dict[str, int]]: + """Walk textual input, including nested tool arguments and content parts.""" + session = session or _RedactionSession(_strings(content)) counts: dict[str, int] = {} if isinstance(content, str): redacted, counts = _redact_text( - content, self.entity_types, self.allowlist, self.allowlist_patterns + content, + self.entity_types, + self.allowlist, + self.allowlist_patterns, + session, ) return redacted, counts - if isinstance(content, list): - new_parts = [] - skipped_parts = 0 - for part in content: - if isinstance(part, dict) and isinstance(part.get("text"), str): - redacted, part_counts = _redact_text( - part["text"], - self.entity_types, - self.allowlist, - self.allowlist_patterns, - ) - new_parts.append({**part, "text": redacted}) - for etype, n in part_counts.items(): - counts[etype] = counts.get(etype, 0) + n + if isinstance(content, (list, dict)): + items = enumerate(content) if isinstance(content, list) else content.items() + output = [] if isinstance(content, list) else {} + for key, value in items: + # Media payloads are opaque; preserve them while scanning any + # textual sibling. name/id/url in ordinary user data are text. + media_type = content.get("type") if isinstance(content, dict) else None + if not isinstance(media_type, str): + media_type = None + opaque = { + "image_url": {"image_url"}, + "input_image": {"image_url", "file_id"}, + "image": {"source"}, + "input_audio": {"input_audio"}, + "audio": {"source", "data"}, + "video_url": {"video_url"}, + "file": {"file"}, + "input_file": {"file_id", "file_data"}, + } + if key in opaque.get(media_type, set()): + new_value, found = value, {} else: - # Images and other non-text parts are not scanned — - # count them so the blind spot is auditable. - new_parts.append(part) - skipped_parts += 1 - if skipped_parts: - logger.debug( - "DataFog guardrail: %d non-text content parts not scanned", - skipped_parts, - ) - return new_parts, counts + new_value, found = self._process_content(value, session) + if isinstance(output, list): + output.append(new_value) + else: + output[key] = new_value + for label, count in found.items(): + counts[label] = counts.get(label, 0) + count + return output, counts return content, counts def _handle_engine_error(self, exc: Exception) -> None: @@ -158,21 +238,20 @@ async def async_pre_call_hook( data: dict, call_type: str, ) -> dict: - messages = data.get("messages") - if not isinstance(messages, list): - return data - total_counts: dict[str, int] = {} - new_messages = [] + new_data = dict(data) + fields = ("messages", "prompt", "input") + session = _RedactionSession( + text for field in fields for text in _strings(data.get(field)) + ) try: - for message in messages: - if isinstance(message, dict) and "content" in message: - new_content, counts = self._process_content(message["content"]) - new_messages.append({**message, "content": new_content}) - for etype, n in counts.items(): - total_counts[etype] = total_counts.get(etype, 0) + n - else: - new_messages.append(message) + for field in fields: + if field in data: + new_data[field], counts = self._process_content( + data[field], session + ) + for label, count in counts.items(): + total_counts[label] = total_counts.get(label, 0) + count except Exception as exc: # noqa: BLE001 — fail policy decides self._handle_engine_error(exc) return data @@ -197,7 +276,6 @@ async def async_pre_call_hook( }, ) - new_data = {**data, "messages": new_messages} self._record_guardrail_logging(new_data, total_counts) return new_data @@ -215,42 +293,130 @@ def _record_guardrail_logging( except Exception: # noqa: BLE001 — observability must never break traffic logger.debug("could not record guardrail logging information") + @staticmethod + def _response_fields(response): + """Yield only generated text fields, preserving response metadata.""" + for choice in _get(response, "choices", []) or []: + if isinstance(_get(choice, "text"), str): + yield choice, "text" + message = _get(choice, "message") + if message is None: + continue + yield message, "content" + for call in _get(message, "tool_calls", []) or []: + function = _get(call, "function") + if function is not None: + yield function, "arguments" + function = _get(message, "function_call") + if function is not None: + yield function, "arguments" + async def async_post_call_success_hook( self, data: dict, user_api_key_dict: Any, response: Any, ) -> Any: - """Redact PII from model responses. + """Redact generated text in place after every field has been scanned.""" + try: + fields = list(self._response_fields(response)) + session = _RedactionSession( + text + for target, field in fields + for text in _strings(_get(target, field)) + ) + changes = [] + for target, field in fields: + value, _ = self._process_content(_get(target, field), session) + changes.append((target, field, value)) + for target, field, value in changes: + _set(target, field, value) + except Exception as exc: # noqa: BLE001 — fail policy decides + self._handle_engine_error(exc) + return response - Mutates ``response`` in place — deliberate: litellm post_call - guardrails share the response object rather than cloning it, and - an unredacted clone escaping through another callback would defeat - the purpose. + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict, response, request_data + ): + """Buffer to completion before emitting, protecting split entities. + + Native text and tool argument deltas are joined per choice/tool. The + first corresponding delta carries the redacted text and later deltas + are empty; metadata and finish events retain their original order. + Incomplete or oversized streams are rejected without emitting chunks. """ - choices = getattr(response, "choices", None) - if not choices: - return response + chunks = [] + groups = {} + size = 0 try: - skipped_parts = 0 - for choice in choices: - message = getattr(choice, "message", None) - if message is not None and isinstance(message.content, str): - redacted, counts = _redact_text( - message.content, - self.entity_types, - self.allowlist, - self.allowlist_patterns, - ) - if counts: - message.content = redacted - elif message is not None and message.content is not None: - skipped_parts += 1 - if skipped_parts: - logger.debug( - "DataFog guardrail: %d non-text response parts not scanned", - skipped_parts, + async for original in response: + # Iterators may reuse a mutable chunk object. Keep each emitted + # delta's value and metadata as they existed when received. + chunk = copy.deepcopy(original) + chunks.append(chunk) + for position, choice in enumerate(_get(chunk, "choices", []) or []): + index = _get(choice, "index", position) + fields = [((index, "text"), choice, "text")] + delta = _get(choice, "delta") + if delta is not None: + fields.append(((index, "content"), delta, "content")) + for call_position, call in enumerate( + _get(delta, "tool_calls", []) or [] + ): + function = _get(call, "function") + if function is not None: + fields.append( + ( + ( + index, + "tool", + _get(call, "index", call_position), + ), + function, + "arguments", + ) + ) + function = _get(delta, "function_call") + if function is not None: + fields.append( + ((index, "function_call"), function, "arguments") + ) + for key, target, field in fields: + value = _get(target, field) + if isinstance(value, str): + size += len(value) + groups.setdefault(key, []).append((target, field, value)) + if size > MAX_STREAM_CHARS or len(chunks) > MAX_STREAM_CHUNKS: + raise _StreamLimitExceeded() + except _StreamLimitExceeded: + raise RuntimeError( + "DataFog streaming response exceeded the scan budget; rejecting unscanned output" + ) from None + except Exception as exc: + # A failed upstream stream cannot be safely completed or replayed. + # Provider exceptions can contain private prompt/response text. + raise RuntimeError( + f"DataFog streaming response did not complete ({type(exc).__name__}); rejecting incomplete output" + ) from None + try: + texts = [ + "".join(value for _, _, value in group) for group in groups.values() + ] + session = _RedactionSession(texts) + changes = [] + for group, text in zip(groups.values(), texts): + redacted, _ = self._process_content(text, session) + changes.extend( + (target, field, redacted if i == 0 else "") + for i, (target, field, _) in enumerate(group) ) - except Exception as exc: # noqa: BLE001 — fail policy decides + for target, field, value in changes: + _set(target, field, value) + except Exception as exc: self._handle_engine_error(exc) - return response + for chunk in chunks: + yield chunk + + +class _StreamLimitExceeded(Exception): + """Internal marker; stream-budget failures always reject output.""" diff --git a/tests/test_review_guardrails.py b/tests/test_review_guardrails.py new file mode 100644 index 00000000..ea6c4cbe --- /dev/null +++ b/tests/test_review_guardrails.py @@ -0,0 +1,434 @@ +"""Review regressions for request/response coverage and safe stream failures.""" + +# ruff: noqa: E402 +# Optional proxy dependencies must be present before importing the adapter. +import copy +import json + +import pytest + +pytest.importorskip("litellm") +pytest.importorskip("fastapi") +from fastapi import HTTPException # noqa: E402 +from litellm import ModelResponseStream # noqa: E402 +from litellm.types.utils import Delta, StreamingChoices # noqa: E402 + +import datafog # noqa: E402 +from datafog.integrations import claude_code as hook # noqa: E402 +from datafog.integrations import litellm_guardrail as gateway # noqa: E402 + +EMAIL_A = "alice@" "example.com" +EMAIL_B = "bob@" "example.com" +SSN = "123-45-" "6789" + + +def guard(**kwargs): + return gateway.DataFogGuardrail(guardrail_name="review", **kwargs) + + +async def pre(adapter, data): + return await adapter.async_pre_call_hook(None, None, data, "completion") + + +async def source(chunks): + for chunk in chunks: + yield chunk + + +async def streamed(adapter, chunks): + return [ + chunk + async for chunk in adapter.async_post_call_streaming_iterator_hook( + None, source(chunks), {} + ) + ] + + +@pytest.mark.asyncio +async def test_request_shares_tokens_without_existing_token_collisions(): + original = { + "messages": [ + {"role": "user", "content": f"[EMAIL_1] {EMAIL_A}"}, + {"role": "user", "content": f"other {EMAIL_B}; same {EMAIL_A}"}, + ], + "prompt": EMAIL_B, + "input": [{"content": EMAIL_A}], + } + before = copy.deepcopy(original) + result = await pre(guard(), original) + assert result["messages"][0]["content"] == "[EMAIL_1] [EMAIL_2]" + assert result["messages"][1]["content"] == "other [EMAIL_3]; same [EMAIL_2]" + assert ( + result["prompt"] == "[EMAIL_3]" and result["input"][0]["content"] == "[EMAIL_2]" + ) + assert original == before + another = await pre(guard(), {"prompt": EMAIL_A}) + assert another["prompt"] == "[EMAIL_1]" + + +@pytest.mark.asyncio +async def test_request_scans_tool_arguments_and_structured_input_values(): + data = { + "messages": [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "save", + "arguments": json.dumps({"email": EMAIL_A, "ssn": SSN}), + }, + } + ], + } + ], + "input": {"name": EMAIL_B, "id": SSN, "url": "send " + EMAIL_A}, + } + result = await pre(guard(), data) + assert ( + EMAIL_A not in result["messages"][0]["tool_calls"][0]["function"]["arguments"] + ) + assert SSN not in result["messages"][0]["tool_calls"][0]["function"]["arguments"] + assert result["input"] == { + "name": "[EMAIL_2]", + "id": "[SSN_1]", + "url": "send [EMAIL_1]", + } + assert result["messages"][0]["tool_calls"][0]["id"] == "call_1" + + +@pytest.mark.asyncio +async def test_multimodal_media_remains_intact_and_text_is_scanned(): + image = { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,AA4242424242424242", + "detail": "high", + }, + } + audio = { + "type": "input_audio", + "input_audio": {"data": "4242424242424242", "format": "wav"}, + } + result = await pre( + guard(), + { + "messages": [ + { + "role": "user", + "content": [image, audio, {"type": "text", "text": EMAIL_A}], + } + ] + }, + ) + assert result["messages"][0]["content"][:2] == [image, audio] + assert result["messages"][0]["content"][2]["text"] == "[EMAIL_1]" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["messages", "prompt", "input"]) +async def test_surrogate_cannot_disable_blocking(field): + value = f"ssn {SSN} \ud83d" + data = ( + {field: [{"role": "user", "content": value}]} + if field == "messages" + else {field: value} + ) + with pytest.raises(HTTPException) as error: + await pre(guard(action="block"), data) + assert error.value.status_code == 400 and SSN not in str(error.value.detail) + + +@pytest.mark.asyncio +async def test_response_reserves_tokens_in_tools_and_covers_legacy_and_text(): + response = { + "choices": [ + { + "index": 0, + "message": { + "content": EMAIL_A, + "tool_calls": [ + { + "function": { + "name": "save", + "arguments": f"[EMAIL_1] {EMAIL_B}", + } + } + ], + }, + }, + { + "index": 1, + "message": { + "content": None, + "function_call": {"name": "old", "arguments": EMAIL_A}, + }, + }, + {"index": 2, "text": EMAIL_B}, + ] + } + result = await guard().async_post_call_success_hook({}, None, response) + assert result is response + assert result["choices"][0]["message"]["content"] == "[EMAIL_2]" + assert ( + result["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] + == "[EMAIL_1] [EMAIL_3]" + ) + assert result["choices"][1]["message"]["function_call"]["arguments"] == "[EMAIL_2]" + assert result["choices"][2]["text"] == "[EMAIL_3]" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_response_scanning_is_atomic_and_respects_fail_policy( + monkeypatch, policy +): + actual = gateway._redact_text + + def failing(text, *args, **kwargs): + if text == "second": + raise RuntimeError(EMAIL_A) + return actual(text, *args, **kwargs) + + monkeypatch.setattr(gateway, "_redact_text", failing) + response = { + "choices": [ + {"message": {"content": EMAIL_A}}, + {"message": {"content": "second"}}, + ] + } + before = copy.deepcopy(response) + if policy == "closed": + with pytest.raises(RuntimeError) as error: + await guard(fail_policy=policy).async_post_call_success_hook( + {}, None, response + ) + assert EMAIL_A not in str(error.value) + else: + assert ( + await guard(fail_policy=policy).async_post_call_success_hook( + {}, None, response + ) + is response + ) + assert response == before + + +@pytest.mark.asyncio +async def test_stream_scans_all_choice_and_tool_argument_fragments(): + chunks = [ + { + "id": "first", + "choices": [ + { + "index": 0, + "delta": { + "content": "[EMAIL_1] alice@", + "tool_calls": [ + { + "index": 0, + "id": "call_a", + "function": { + "name": "save", + "arguments": '{"ssn":"123-', + }, + }, + { + "index": 1, + "id": "call_b", + "function": {"name": "save", "arguments": "bob@"}, + }, + ], + }, + }, + { + "index": 1, + "delta": { + "content": "bob@", + "function_call": {"name": "old", "arguments": "alice@"}, + }, + }, + {"index": 2, "text": "bob@"}, + ], + }, + { + "id": "last", + "choices": [ + { + "index": 0, + "delta": { + "content": "example.com", + "tool_calls": [ + {"index": 0, "function": {"arguments": '45-6789"}'}}, + {"index": 1, "function": {"arguments": "example.com"}}, + ], + }, + "finish_reason": "tool_calls", + }, + { + "index": 1, + "delta": { + "content": "example.com", + "function_call": {"arguments": "example.com"}, + }, + "finish_reason": "stop", + }, + {"index": 2, "text": "example.com", "finish_reason": "stop"}, + ], + }, + ] + original = copy.deepcopy(chunks) + output = await streamed(guard(), chunks) + first, last = output + assert first["choices"][0]["delta"]["content"] == "[EMAIL_1] [EMAIL_2]" + calls = first["choices"][0]["delta"]["tool_calls"] + assert calls[0]["function"]["arguments"] == '{"ssn":"[SSN_1]"}' + assert calls[1]["function"]["arguments"] == "[EMAIL_3]" + assert first["choices"][1]["delta"]["content"] == "[EMAIL_3]" + assert first["choices"][1]["delta"]["function_call"]["arguments"] == "[EMAIL_2]" + assert first["choices"][2]["text"] == "[EMAIL_3]" + assert last["choices"][0]["delta"]["content"] == "" + assert last["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] == "" + assert last["choices"][0]["finish_reason"] == "tool_calls" + assert chunks == original + assert first["id"] == "first" and last["id"] == "last" + + +@pytest.mark.asyncio +async def test_stream_handles_real_litellm_native_chunks(): + chunks = [ + ModelResponseStream( + id="same", choices=[StreamingChoices(index=0, delta=Delta(content=part))] + ) + for part in ("alice@", "example.com") + ] + output = await streamed(guard(), chunks) + assert output[0].choices[0].delta.content == "[EMAIL_1]" + assert output[1].choices[0].delta.content == "" + assert chunks[0].choices[0].delta.content == "alice@" + + +@pytest.mark.asyncio +async def test_stream_snapshots_reused_mutable_chunks(): + async def reused(): + chunk = {"choices": [{"index": 0, "delta": {"content": "alice@"}}]} + yield chunk + chunk["choices"][0]["delta"]["content"] = "example.com" + yield chunk + + output = [ + chunk + async for chunk in guard().async_post_call_streaming_iterator_hook( + None, reused(), {} + ) + ] + assert output[0]["choices"][0]["delta"]["content"] == "[EMAIL_1]" + assert output[1]["choices"][0]["delta"]["content"] == "" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_stream_source_errors_emit_nothing_and_hide_provider_text(policy): + async def broken(): + yield {"choices": [{"delta": {"content": EMAIL_A}}]} + raise RuntimeError("provider failed on " + EMAIL_A) + + emitted = [] + with pytest.raises(RuntimeError) as error: + async for chunk in guard( + fail_policy=policy + ).async_post_call_streaming_iterator_hook(None, broken(), {}): + emitted.append(chunk) + assert not emitted and EMAIL_A not in str(error.value) + assert error.value.__cause__ is None and error.value.__suppress_context__ + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit", ["MAX_STREAM_CHARS", "MAX_STREAM_CHUNKS"]) +async def test_stream_limits_reject_without_emission(monkeypatch, limit): + monkeypatch.setattr(gateway, limit, 1) + chunks = [{"choices": [{"delta": {"content": "ab"}}]}] * 2 + emitted = [] + with pytest.raises(RuntimeError, match="scan budget"): + async for chunk in guard().async_post_call_streaming_iterator_hook( + None, source(chunks), {} + ): + emitted.append(chunk) + assert not emitted + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_stream_engine_failures_follow_documented_policy(monkeypatch, policy): + def failing(*args, **kwargs): + raise RuntimeError(EMAIL_A) + + monkeypatch.setattr(gateway, "_redact_text", failing) + chunks = [{"choices": [{"delta": {"content": EMAIL_A}}]}] + if policy == "open": + assert await streamed(guard(fail_policy=policy), chunks) == chunks + else: + with pytest.raises(RuntimeError, match="fail_policy is 'closed'") as error: + await streamed(guard(fail_policy=policy), chunks) + assert EMAIL_A not in str(error.value) + + +@pytest.mark.parametrize("action", ["ask", "deny"]) +def test_hook_oversize_never_silently_allows_unscanned_tail(action): + payload = { + "hook_event_name": "PreToolUse", + "tool_name": "Write", + "tool_input": {"content": "x" * (hook.MAX_SCAN_CHARS + 1) + " " + SSN}, + } + code, output = hook.run(payload, {"DATAFOG_HOOK_ACTION": action}) + decision = json.loads(output)["hookSpecificOutput"] + assert code == 0 and decision["permissionDecision"] == action + assert ( + "not fully scanned" in decision["permissionDecisionReason"] + and SSN not in output + ) + + +def test_hook_surrogate_and_total_budget_cannot_disable_pii_gate(monkeypatch): + payload = { + "hook_event_name": "PreToolUse", + "tool_input": {"command": "echo " + SSN, "description": "\udc00"}, + } + code, output = hook.run(payload, {}) + assert code == 0 and "SSN x1" in output + monkeypatch.setattr(hook, "TOTAL_SCAN_CHARS", 20) + code, output = hook.run( + {"hook_event_name": "PreToolUse", "tool_input": ["a" * 15, "b" * 15]}, {} + ) + assert ( + code == 0 + and json.loads(output)["hookSpecificOutput"]["permissionDecision"] == "ask" + ) + + +@pytest.mark.asyncio +async def test_decorator_awaits_and_rejects_non_string_returns(): + adapter = datafog.create_guardrail() + + @adapter + async def generated(): + return EMAIL_A + + assert await generated() == "[EMAIL_1]" + + @adapter + async def structured(): + return [EMAIL_A] + + with pytest.raises(TypeError, match="must return a string"): + await structured() + + @adapter + def sync_structured(): + return {"text": EMAIL_A} + + with pytest.raises(TypeError, match="must return a string"): + sync_structured() From a9c19fc0b927f50bd5b98639d41feaa8f033251d Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:00:22 -0800 Subject: [PATCH 10/23] test: review wrapper contract changes and qualify shared detection --- datafog/engine.py | 59 +- tests/contracts/rust-2.0.0.json | 77 ++ tests/contracts/wrapper-4.12.0.json | 1149 ++++++++++++++++++++++++ tests/rust_contract.py | 23 +- tests/test_contract_481.py | 67 +- tests/test_core04_integration.py | 7 +- tests/test_core20_integration.py | 58 ++ tests/test_de_pii_regex.py | 5 +- tests/test_rust_contract.py | 11 +- tests/test_spacy_span_contract.py | 56 ++ tests/test_text_service.py | 428 ++------- tests/test_text_service_integration.py | 28 +- 12 files changed, 1539 insertions(+), 429 deletions(-) create mode 100644 tests/contracts/rust-2.0.0.json create mode 100644 tests/contracts/wrapper-4.12.0.json create mode 100644 tests/test_core20_integration.py create mode 100644 tests/test_spacy_span_contract.py diff --git a/datafog/engine.py b/datafog/engine.py index ec33081b..8ce61a5f 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -793,6 +793,35 @@ def scan( ) +def _validate_hash_parameters( + strategy: str, hash_type: str, hash_key: bytes | None +) -> None: + if hash_key is not None and (not isinstance(hash_key, bytes) or len(hash_key) < 32): + raise ValueError("hash_key must contain at least 32 bytes") + if strategy == "hash": + try: + hashlib.new(hash_type).hexdigest() + except (ValueError, TypeError): + raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None + + +def _validated_redaction_entities(text: str, entities: list[Entity]) -> list[Entity]: + valid_entities = [] + for entity in entities: + if ( + not 0 <= entity.start < entity.end <= len(text) + or not entity.text + or text[entity.start : entity.end] != entity.text + ): + raise ValueError( + "entities must contain valid ranges whose text matches the source" + ) + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", entity.type): + raise ValueError("entity types must be uppercase labels") + valid_entities.append(entity) + return _suppress_overlapping_entities(valid_entities) + + def redact( text: str, entities: list[Entity], @@ -809,6 +838,8 @@ def redact( type priority and then confidence. Suppressed spans are omitted from ``RedactResult.entities`` and ``mapping``. + Invalid ranges, mismatched source text, and malformed labels raise ValueError. + ``mapping`` for the ``mask`` strategy is keyed by per-type indexed keys (``[EMAIL_MASK_1]``) rather than the mask string, because distinct same-length values produce identical masks and would collide. @@ -818,39 +849,19 @@ def redact( if strategy not in {"token", "mask", "hash", "pseudonymize"}: raise ValueError("strategy must be one of: token, mask, hash, pseudonymize") - if hash_key is not None and (not isinstance(hash_key, bytes) or len(hash_key) < 32): - raise ValueError("hash_key must contain at least 32 bytes") - if strategy == "hash": - try: - hashlib.new(hash_type).hexdigest() - except (ValueError, TypeError): - raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None + _validate_hash_parameters(strategy, hash_type, hash_key) mapping: dict[str, str] = {} counters: dict[str, int] = {} pseudonym_by_value: dict[tuple[str, str], str] = {} - valid_entities = [] - for entity in entities: - if ( - not 0 <= entity.start < entity.end <= len(text) - or not entity.text - or text[entity.start : entity.end] != entity.text - ): - raise ValueError( - "entities must contain valid ranges whose text matches the source" - ) - if not re.fullmatch(r"[A-Z][A-Z0-9_]*", entity.type): - raise ValueError("entity types must be uppercase labels") - valid_entities.append(entity) - # Returns non-overlapping spans sorted by document position. - valid_entities = _suppress_overlapping_entities(valid_entities) + valid_entities = _validated_redaction_entities(text, entities) occupied_tokens = set(re.findall(r"\[[A-Z][A-Za-z0-9_]*\]", text)) # Assign replacements in document order so the per-type counters used by - # the token/pseudonymize strategies number the first occurrence _1; spans - # are applied right-to-left below so earlier offsets stay valid. + # the token/pseudonymize strategies number the first occurrence _1. + # Assemble the result in one pass without repeated document copies. replacements: list[tuple[Entity, str]] = [] for entity in valid_entities: original = text[entity.start : entity.end] diff --git a/tests/contracts/rust-2.0.0.json b/tests/contracts/rust-2.0.0.json new file mode 100644 index 00000000..324f9cfa --- /dev/null +++ b/tests/contracts/rust-2.0.0.json @@ -0,0 +1,77 @@ +{ + "core_version": "2.0.0", + "cases": { + "scan-observation-invalid-card": { + "classification": "detector-difference", + "reason": "Core validates card checksums; Python 4.8.1 accepts this card-shaped value. Experimental precision difference, not a claim of full parity.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "4111 1111 1111 1112", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "scan-observation-boundary": { + "classification": "detector-difference", + "reason": "Core rejects the SSN embedded in an ASCII alphanumeric token; Python 4.8.1 detects the numeric substring.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "x123-45-6789y", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "locale-unknown": { + "classification": "validation-difference", + "reason": "The capability adapter rejects the unadvertised locale with a backend-specific diagnostic instead of the Python regex locale inventory. The exception remains ValueError.", + "expected": { + "error": { + "type": "ValueError", + "message": "Unsupported locale for the Rust backend: 'xx'" + }, + "warnings": [] + } + }, + "scan-observation-ambiguous-number": { + "classification": "detector-difference", + "reason": "Core's North American area-code check requires a leading digit of 2-9, so a bare 10-digit run starting with 1 stays undetected. This is the deliberate Unix-timestamp protection; Python 4.8.1 matches it.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Invoice 1234567890", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "locale-default-DE_TAX_ID": { + "classification": "detector-difference", + "reason": "CORE-17/PY-L2 suppress a German tax identifier from the generic PHONE detector even when German detectors are not activated. Explicit locale selection still detects DE_TAX_ID.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Steuer-ID 12345678901 liegt vor.", + "engine_used": "regex" + } + }, + "warnings": [] + } + } + }, + "review": "Core 2.0 candidate qualifications retain the four reviewed historical differences below. Wrapper 4.12 fixes are layered separately in wrapper-4.12.0.json." +} diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json new file mode 100644 index 00000000..e4c0835b --- /dev/null +++ b/tests/contracts/wrapper-4.12.0.json @@ -0,0 +1,1149 @@ +{ + "wrapper_version": "4.12.0", + "core_versions": [ + "1.4.0", + "2.0.0" + ], + "source_oracle": "4.8.1.json (unchanged)", + "review": "2026-11-15 PDF remediation: explicit reviewed wrapper changes, not a regenerated historical oracle", + "cases": { + "selection-unknown": { + "classification": "validation-difference", + "reason": "PY-M6 rejects unknown entity labels instead of silently disabling detection.", + "expected": { + "error": { + "type": "ValueError", + "message": "Unsupported entity_types for rust/regex: NOT_A_TYPE" + }, + "warnings": [] + } + }, + "redact-token": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses one collision-safe token for repeated values.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_2] [EMAIL_1] [SSN_1]", + "mapping": { + "[EMAIL_1]": "a@example.com", + "[EMAIL_2]": "b@example.com", + "[SSN_1]": "123-45-6789" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "SSN", + "text": "123-45-6789", + "start": 42, + "end": 53, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "engine-redact-token": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses one collision-safe token for repeated values.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_2] [EMAIL_1]", + "mapping": { + "[EMAIL_1]": "a@example.com", + "[EMAIL_2]": "b@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "redact-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c] [EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [SSN_01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0]", + "mapping": { + "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]": "a@example.com", + "[EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c]": "b@example.com", + "[SSN_01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0]": "123-45-6789" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "SSN", + "text": "123-45-6789", + "start": 42, + "end": 53, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "engine-redact-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c] [EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]", + "mapping": { + "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]": "a@example.com", + "[EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c]": "b@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "preset-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]", + "mapping": { + "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]": "alice@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 0, + "end": 17, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "explicit-invalid-bounds": { + "classification": "validation-difference", + "reason": "PY-L7 rejects malformed explicit spans rather than silently skipping them.", + "expected": { + "error": { + "type": "ValueError", + "message": "entities must contain valid ranges whose text matches the source" + }, + "warnings": [] + } + }, + "explicit-mismatched-text": { + "classification": "validation-difference", + "reason": "PY-L7 rejects explicit span text that differs from the source.", + "expected": { + "error": { + "type": "ValueError", + "message": "entities must contain valid ranges whose text matches the source" + }, + "warnings": [] + } + }, + "error-invalid-pattern": { + "classification": "validation-difference", + "reason": "PY-M9 uses a timeout-capable matcher, whose syntax diagnostic counts positions differently.", + "expected": { + "error": { + "type": "ValueError", + "message": "allowlist_patterns contains an invalid regex: '[' (unterminated character set at position 1)" + }, + "warnings": [] + } + }, + "error-unsafe-pattern": { + "classification": "behavioral-fix", + "reason": "PY-M9 allows nested groups under a hard matching timeout instead of a bypassable syntax heuristic.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "error-oversized-pattern": { + "classification": "validation-difference", + "reason": "PY-M9 validates both pattern type and length with an explicit diagnostic.", + "expected": { + "error": { + "type": "ValueError", + "message": "allowlist_patterns entries must be strings of at most 512 characters" + }, + "warnings": [] + } + }, + "facade-sanitize": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": "[EMAIL_1] [EMAIL_1]", + "warnings": [] + } + }, + "facade-filter_output": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_1]", + "mapping": { + "[EMAIL_1]": "alice@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 0, + "end": 17, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 18, + "end": 35, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "facade-detect": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + }, + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 18, + "end": 35 + } + ], + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.detect() is deprecated and will be removed in 5.0. Use datafog.scan() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "facade-anonymize_text": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": "[EMAIL_1] [EMAIL_1]", + "warnings": [] + } + }, + "facade-process": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + }, + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 18, + "end": 35 + } + ] + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-redact": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_REDACTED]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-replace": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_XXXXX]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 replaces legacy truncated MD5 with the complete selected digest; PY-L3 updates the retirement notice.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-invalid": { + "classification": "validation-difference", + "reason": "PY-L5 rejects an invalid anonymization method instead of choosing an undocumented fallback; PY-L3 updates the notice.", + "expected": { + "error": { + "type": "ValueError", + "message": "method must be 'redact', 'replace', or 'hash'" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "service-default": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route TextService through shared native detection and return populated canonical labels.", + "expected": { + "value": { + "EMAIL": [ + "alice@example.com" + ] + }, + "warnings": [] + } + }, + "service-german": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route TextService through shared native detection with overlap resolution and populated canonical labels.", + "expected": { + "value": { + "DE_TAX_ID": [ + "12345678901" + ] + }, + "warnings": [] + } + }, + "service-batch": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route every TextService batch member through shared native detection, returning populated canonical labels.", + "expected": { + "value": [ + { + "EMAIL": [ + "alice@example.com" + ] + }, + {} + ], + "warnings": [] + } + }, + "public-signatures": { + "classification": "signature-addition", + "reason": "PY-M7 adds keyword-only hash_type/hash_key; TextService custom models use the engine gliner_model keyword; backend remains keyword-only.", + "expected": { + "value": { + "datafog:scan": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "preset", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:protect": { + "parameters": [ + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "on_detect", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + } + ] + }, + "datafog:detect": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:process": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "anonymize", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": false + }, + { + "name": "method", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:sanitize": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:scan_prompt": { + "parameters": [ + { + "name": "prompt", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:filter_output": { + "parameters": [ + { + "name": "output", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:create_guardrail": { + "parameters": [ + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "on_detect", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + } + ] + }, + "datafog.engine:Entity": { + "parameters": [ + { + "name": "type", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "start", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "end", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "confidence", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": [ + "type", + "text", + "start", + "end", + "confidence", + "engine" + ] + }, + "datafog.engine:ScanResult": { + "parameters": [ + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine_used", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": [ + "entities", + "text", + "engine_used" + ] + }, + "datafog.engine:RedactResult": { + "parameters": [ + { + "name": "redacted_text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "mapping", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": [ + "redacted_text", + "mapping", + "entities" + ] + }, + "datafog.engine:scan": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "smart" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "gliner_model", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog.engine:redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog.engine:scan_and_redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "smart" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + } + }, + "warnings": [] + } + }, + "anonymize-text-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 emits the complete chosen digest rather than a truncated unsalted token.", + "expected": { + "value": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]", + "warnings": [] + } + } + } +} diff --git a/tests/rust_contract.py b/tests/rust_contract.py index 2b618007..9e673091 100644 --- a/tests/rust_contract.py +++ b/tests/rust_contract.py @@ -9,7 +9,11 @@ from tests.contract_481 import FIXTURE, observe -DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.4.0.json" +CORE_VERSION = importlib.metadata.version("datafog-core") +CORE_LINE = ".".join(CORE_VERSION.split(".")[:2]) +DEVIATIONS = Path(__file__).parent / "contracts" / f"rust-{CORE_LINE}.0.json" +WRAPPER_DEVIATIONS = Path(__file__).parent / "contracts" / "wrapper-4.12.0.json" + TARGETS = { "datafog:scan", "datafog:redact", @@ -20,6 +24,18 @@ } +def reviewed_deviation(case): + """Layer explicit wrapper fixes on the unchanged historical wheel oracle.""" + wrapper = json.loads(WRAPPER_DEVIATIONS.read_text(encoding="utf-8")) + native = json.loads(DEVIATIONS.read_text(encoding="utf-8")) + return native["cases"].get(case["id"]) or wrapper["cases"].get(case["id"]) + + +def expected_observation(case): + deviation = reviewed_deviation(case) + return copy.deepcopy(deviation["expected"] if deviation else case["expected"]) + + def applicable(case): return ( case.get("operation", "call") == "call" @@ -39,15 +55,14 @@ def main(): parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() contract = json.loads(FIXTURE.read_text(encoding="utf-8")) - deviations = json.loads(DEVIATIONS.read_text(encoding="utf-8")) results = [] for case in contract["cases"]: if not applicable(case): results.append({"id": case["id"], "status": "outside-backend-scope"}) continue actual = native_observation(case) - deviation = deviations["cases"].get(case["id"]) - expected = deviation["expected"] if deviation else case["expected"] + deviation = reviewed_deviation(case) + expected = expected_observation(case) status = "regression" if actual != expected else "match" if status == "match" and deviation: status = deviation["classification"] diff --git a/tests/test_contract_481.py b/tests/test_contract_481.py index 43863dfd..b54ecf1e 100644 --- a/tests/test_contract_481.py +++ b/tests/test_contract_481.py @@ -1,64 +1,33 @@ """Frozen observations from the published 4.8.1 wheel; never regenerate in CI.""" -import copy import json import pytest from tests.contract_481 import FIXTURE, observe -from tests.rust_contract import DEVIATIONS +from tests.rust_contract import WRAPPER_DEVIATIONS, expected_observation CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) -# The default backend is native from 4.10.0, so the reviewed native -# differences are the published 4.10 observations for those cases. -REVIEWED = json.loads(DEVIATIONS.read_text(encoding="utf-8")) +REVIEWED_WRAPPER = json.loads(WRAPPER_DEVIATIONS.read_text(encoding="utf-8")) def expected_49(case): - """Allow only the added backend parameter, revised shim notices and the - reviewed native differences.""" - - deviation = REVIEWED["cases"].get(case["id"]) - expected = ( - copy.deepcopy(deviation["expected"]) - if deviation - else copy.deepcopy(case["expected"]) - ) - if case["id"] == "public-signatures": - for target in ( - "datafog:scan", - "datafog:redact", - "datafog.engine:scan", - "datafog.engine:scan_and_redact", - ): - expected["value"][target]["parameters"].append( - { - "name": "backend", - "kind": "KEYWORD_ONLY", - "required": False, - "default": None, - } - ) - if case.get("target") in {"datafog:detect", "datafog:process"}: - name = case["target"].split(":")[1] - replacement = ( - "datafog.scan()" - if name == "detect" - else "datafog.scan() or datafog.redact()" - ) - expected["warnings"] = [ - { - "type": "FutureWarning", - "message": ( - f"datafog.{name}() is deprecated and will be removed in 5.0. " - f"Use {replacement} instead. " - 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0". ' - "The earlier promise to retain this shim through 5.x has been revised. " - "See docs/retirement.rst." - ), - } - ] - return expected + """The immutable oracle plus individually explained native/wrapper fixes.""" + return expected_observation(case) + + +def test_reviewed_wrapper_deviations_are_explicit_and_in_scope(): + cases = {case["id"]: case for case in CONTRACT["cases"]} + assert set(REVIEWED_WRAPPER["cases"]) <= set(cases) + for case_id, deviation in REVIEWED_WRAPPER["cases"].items(): + assert deviation["reason"] + assert deviation["classification"] in { + "behavioral-fix", + "validation-difference", + "diagnostic-difference", + "signature-addition", + } + assert deviation["expected"] != cases[case_id]["expected"] @pytest.mark.parametrize("case", CONTRACT["cases"], ids=lambda case: case["id"]) diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index b0e2ae9f..9f952bac 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -1,4 +1,4 @@ -"""Exercise actual Core 0.4 wheels through the Python compatibility adapter. +"""Exercise reviewed native wheels through the Python compatibility adapter. Examples originate in Core's fixtures/{german,jwt,private-key,npi, us-routing-number,uuid}.jsonl. Payloads are synthetic detector fixtures, @@ -6,6 +6,7 @@ """ import importlib.metadata +import json import re import pytest @@ -13,6 +14,7 @@ import datafog from datafog import engine, v5 from datafog.compat import v4 +from tests.rust_contract import DEVIATIONS core = pytest.importorskip("datafog_core") @@ -50,7 +52,8 @@ def test_supported_wheel_version_and_capability_contract(): version = importlib.metadata.version("datafog-core") - assert version.split(".")[:1] == ["1"], version + reviewed = json.loads(DEVIATIONS.read_text())["core_version"] + assert version.split(".")[:2] == reviewed.split(".")[:2], version capabilities = core.capabilities() assert capabilities["contract_version"] == 1 supported = capabilities["supported_entities"] diff --git a/tests/test_core20_integration.py b/tests/test_core20_integration.py new file mode 100644 index 00000000..290f8e0d --- /dev/null +++ b/tests/test_core20_integration.py @@ -0,0 +1,58 @@ +"""Core 2 candidate seam regressions; exercised before the dependency floor moves.""" + +import importlib.metadata + +import pytest + +import datafog + +pytestmark = pytest.mark.skipif( + importlib.metadata.version("datafog-core").split(".")[0] != "2", + reason="qualifies the separately built Core 2 candidate before publication", +) + + +@pytest.mark.parametrize("value", ["::ffff:192.168.1.1", "64:ff9b::192.0.2.33"]) +def test_ipv6_embedded_ipv4_tail_is_fully_redacted(value): + result = datafog.redact(value) + assert result.redacted_text == "[IP_ADDRESS_1]" + assert result.mapping == {"[IP_ADDRESS_1]": value} + + +def test_contextual_email_does_not_swallow_neighboring_phone(): + text = "phone=415-555-2671&email=bob@example.com" + result = datafog.scan(text) + assert [(item.type, item.text) for item in result.entities] == [ + ("PHONE", "415-555-2671"), + ("EMAIL", "bob@example.com"), + ] + assert datafog.redact(text).redacted_text == "phone=[PHONE_1]&email=[EMAIL_1]" + + +def test_phone_followed_by_zip_does_not_become_credit_card(): + text = "555-123-4567 94105" + assert [(item.type, item.text) for item in datafog.scan(text).entities] == [ + ("PHONE", "555-123-4567"), + ("ZIP_CODE", "94105"), + ] + assert datafog.scan(text, entity_types=["PHONE"]).entities[0].text == "555-123-4567" + + +def test_adding_model_selection_does_not_drop_large_document_ssns(): + text = "ssn 123-45-6789 " * 3000 + result = datafog.scan(text, entity_types=["SSN", "PERSON"]) + assert len([item for item in result.entities if item.type == "SSN"]) == 3000 + assert all(text[item.start : item.end] == item.text for item in result.entities) + + +def test_international_email_and_person_punctuation(): + assert datafog.scan("mail jörg@müller.de").entities[0].text == "jörg@müller.de" + result = datafog.scan("Jane Doe,\n14 Penny Lane", entity_types=["PERSON"]) + assert result.entities and all( + item.text == item.text.rstrip(",;:.!?") for item in result.entities + ) + + +def test_german_context_prevents_generic_false_positive_labels(): + assert not datafog.scan("Steuer-ID 12345678901 liegt vor.").entities + assert not datafog.scan("IBAN DE44 5001 0517 5407 3249 31 ist gueltig.").entities diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py index 920c35f3..d36cc6af 100644 --- a/tests/test_de_pii_regex.py +++ b/tests/test_de_pii_regex.py @@ -166,7 +166,10 @@ def test_german_vat_redaction_suppresses_inner_generic_ssn_match( def test_top_level_helpers_and_supported_entities_respect_locale() -> None: default_entities = get_supported_entities() - assert all(not entity.startswith("DE_") for entity in default_entities) + assert {"EMAIL", "DE_VAT_ID", "DE_IBAN", "JWT", "PERSON"} <= set(default_entities) + assert "JWT" not in get_supported_entities(backend="python") + # Supported inventory includes opt-in labels; scanning still requires locale activation. + assert "DE_TAX_ID" in default_entities german_entities = get_supported_entities(locales=["de"]) assert "DE_VAT_ID" in german_entities diff --git a/tests/test_rust_contract.py b/tests/test_rust_contract.py index c3f16bc0..73909bf4 100644 --- a/tests/test_rust_contract.py +++ b/tests/test_rust_contract.py @@ -6,7 +6,12 @@ import pytest from tests.contract_481 import FIXTURE -from tests.rust_contract import DEVIATIONS, applicable, native_observation +from tests.rust_contract import ( + DEVIATIONS, + applicable, + expected_observation, + native_observation, +) pytest.importorskip("datafog_core") CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) @@ -30,6 +35,4 @@ def test_native_version_and_review_inventory(): @pytest.mark.parametrize("case", CASES, ids=lambda case: case["id"]) def test_native_contract(case): - deviation = REVIEWED["cases"].get(case["id"]) - expected = deviation["expected"] if deviation else case["expected"] - assert native_observation(case) == expected, case["id"] + assert native_observation(case) == expected_observation(case), case["id"] diff --git a/tests/test_spacy_span_contract.py b/tests/test_spacy_span_contract.py new file mode 100644 index 00000000..7029b391 --- /dev/null +++ b/tests/test_spacy_span_contract.py @@ -0,0 +1,56 @@ +"""Optional real-spaCy offset integration without downloading model weights.""" + +import pytest + +from datafog import engine +from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator +from datafog.services.text_service import TextService + +spacy = pytest.importorskip("spacy") + + +@pytest.fixture +def real_annotator(monkeypatch): + nlp = spacy.blank("en") + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns( + [ + {"label": "PERSON", "pattern": "Ann"}, + {"label": "ORG", "pattern": "Acme Corp"}, + {"label": "DATE", "pattern": "yesterday"}, + ] + ) + annotator = SpacyPIIAnnotator(nlp=nlp, model_name="local-test-ruler") + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: annotator) + return annotator + + +def test_spacy_ranges_redact_actual_name_without_hitting_prefix(real_annotator): + text = "Annual report by Ann yesterday: 6011111111111117" + result = engine.scan_and_redact(text, engine="spacy") + assert ( + result.redacted_text == "Annual report by [PERSON_1] yesterday: [CREDIT_CARD_1]" + ) + assert all(text[item.start : item.end] == item.text for item in result.entities) + + +def test_spacy_service_combines_ner_and_native_detection(real_annotator): + text = "Ann at Acme Corp: 6011111111111117 ip 2001:db8::1" + service = TextService(engine="spacy") + result = service.annotate_text_sync(text) + assert result == { + "PERSON": ["Ann"], + "ORGANIZATION": ["Acme Corp"], + "CREDIT_CARD": ["6011111111111117"], + "IP_ADDRESS": ["2001:db8::1"], + } + assert all( + text[item.start : item.end] == item.text + for item in service.annotate_text_sync(text, structured=True) + ) + + +def test_spacy_repeated_names_keep_each_original_span(real_annotator): + text = "Ann wrote an Annual report for Ann." + result = engine.scan(text, engine="spacy") + assert [(item.start, item.end) for item in result.entities] == [(0, 3), (31, 34)] diff --git a/tests/test_text_service.py b/tests/test_text_service.py index 74156082..3d5973ad 100644 --- a/tests/test_text_service.py +++ b/tests/test_text_service.py @@ -1,366 +1,124 @@ -from unittest.mock import Mock, patch - -import pytest - -# Legacy coverage for the historical shadow TextService. New behavior belongs -# in datafog.services.text_service. -from datafog.services.text_service_original import TextService +"""Public TextService behavior using the shipped service and shared engine.""" +import asyncio +import threading +from types import SimpleNamespace -@pytest.fixture -def mock_annotator(): - mock = Mock() - mock.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - return mock - +import pytest -@pytest.fixture -def mock_regex_annotator(): - mock = Mock() - mock.annotate.return_value = { - "EMAIL": ["john@example.com"], - "PHONE": ["555-555-5555"], - } +from datafog import engine +from datafog.services.text_service import TextService - # Add mock for annotate_with_spans method - from datafog.processing.text_processing.regex_annotator import ( - AnnotationResult, - Span, - ) - spans = [ - Span(label="EMAIL", start=0, end=15, text="john@example.com"), - Span(label="PHONE", start=20, end=32, text="555-555-5555"), - ] - mock.annotate_with_spans.return_value = ( - {"EMAIL": ["john@example.com"], "PHONE": ["555-555-5555"]}, - AnnotationResult(text="test", spans=spans), - ) - return mock +@pytest.mark.parametrize( + "options", + [{"engine": "invalid"}, {"text_chunk_length": 0}, {"text_chunk_length": -1}], +) +def test_invalid_configuration_is_explicit(options): + with pytest.raises(ValueError): + TextService(**options) -@pytest.fixture -def text_service(mock_annotator, mock_regex_annotator): - # Configure regex annotator to return empty results so auto mode falls back to spaCy - # This ensures backward compatibility with existing tests while using 'auto' mode - mock_regex_annotator.annotate.return_value = { - key: [] - for key in ["EMAIL", "PHONE", "SSN", "CREDIT_CARD", "IP_ADDRESS", "DOB", "ZIP"] +def test_default_service_detects_native_only_entities(): + result = TextService().annotate_text_sync("card 6011111111111117 ip 2001:db8::1") + assert result == { + "CREDIT_CARD": ["6011111111111117"], + "IP_ADDRESS": ["2001:db8::1"], } - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - return_value=mock_annotator, - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - return_value=mock_regex_annotator, - ): - # Use 'auto' engine to match production default, but regex will find nothing - # so it will always fall back to spaCy, maintaining test compatibility - return TextService(text_chunk_length=10, engine="auto") - - -@pytest.fixture -def text_service_with_engine(mock_annotator, mock_regex_annotator): - def _create_service(engine="auto"): - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - return_value=mock_annotator, - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - return_value=mock_regex_annotator, - ): - return TextService(text_chunk_length=10, engine=engine) - return _create_service +def test_long_document_seams_preserve_complete_entities(): + value = "john@example.com" + text = "x" * 995 + " " + value + " " + "y" * 1010 + " ssn 123-45-6789" + result = TextService(text_chunk_length=1000).annotate_text_sync(text) + assert result == {"EMAIL": [value], "SSN": ["123-45-6789"]} -def test_init(text_service): - assert text_service.text_chunk_length == 10 - - -def test_init_with_default_engine(text_service): - assert text_service.text_chunk_length == 10 - # We're using 'auto' in our fixture to match production default - assert text_service.engine == "auto" - - -def test_init_with_custom_engine(text_service_with_engine): - service = text_service_with_engine(engine="regex") - assert service.engine == "regex" - - service = text_service_with_engine(engine="spacy") - assert service.engine == "spacy" - - service = text_service_with_engine(engine="auto") - assert service.engine == "auto" - - -def test_init_with_invalid_engine(): - with pytest.raises(AssertionError, match="Invalid engine"): - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - ): - TextService(engine="invalid") - - -def test_chunk_text(text_service): - text = "This is a test sentence for chunking." - chunks = text_service._chunk_text(text) - assert len(chunks) == 4 - assert chunks == ["This is a ", "test sente", "nce for ch", "unking."] +def test_structured_output_preserves_unicode_source_ranges(): + text = "😀 café john@example.com and john@example.com" + result = TextService(text_chunk_length=10).annotate_text_sync(text, structured=True) + assert len(result) == 2 + assert all( + item.label == "EMAIL" and text[item.start : item.end] == item.text + for item in result + ) + assert result[0].start < result[1].start -def test_combine_annotations(text_service): - annotations = [ - {"PER": ["John"], "ORG": ["Acme"]}, - {"PER": ["Doe"], "LOC": ["New York"]}, +def test_batch_results_preserve_input_order_and_duplicate_documents(): + texts = ["a@example.com", "no pii", "a@example.com"] + assert TextService().batch_annotate_text_sync(texts) == [ + {"EMAIL": ["a@example.com"]}, + {}, + {"EMAIL": ["a@example.com"]}, ] - combined = text_service._combine_annotations(annotations) - assert combined == {"PER": ["John", "Doe"], "ORG": ["Acme"], "LOC": ["New York"]} - - -def test_annotate_text_sync(text_service): - result = text_service.annotate_text_sync("John Doe works at Acme Inc") - assert result == { - "PER": ["John Doe", "John Doe", "John Doe"], - "ORG": ["Acme Inc", "Acme Inc", "Acme Inc"], - } - - -def test_batch_annotate_text_sync(text_service): - texts = ["John Doe", "Acme Inc"] - result = text_service.batch_annotate_text_sync(texts) - assert result == { - "John Doe": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - "Acme Inc": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - } @pytest.mark.asyncio -async def test_annotate_text_async(text_service): - result = await text_service.annotate_text_async("John Doe works at Acme Inc") - assert result == { - "PER": ["John Doe", "John Doe", "John Doe"], - "ORG": ["Acme Inc", "Acme Inc", "Acme Inc"], - } +async def test_async_results_match_synchronous_service(): + service = TextService() + texts = ["a@example.com", "ssn 123-45-6789", "no pii"] + assert await service.batch_annotate_text_async( + texts + ) == service.batch_annotate_text_sync(texts) + assert await service.annotate_text_async( + texts[0], structured=True + ) == service.annotate_text_sync(texts[0], structured=True) @pytest.mark.asyncio -async def test_batch_annotate_text_async(text_service): - texts = ["John Doe", "Acme Inc"] - result = await text_service.batch_annotate_text_async(texts) - assert result == { - "John Doe": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - "Acme Inc": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - } - - -def test_long_text_chunking(text_service): - long_text = "John Doe works at Acme Inc. Jane Smith is from New York City." - result = text_service.annotate_text_sync(long_text) - expected_count = len(text_service._chunk_text(long_text)) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -@pytest.mark.asyncio -async def test_long_text_chunking_async(text_service): - long_text = "John Doe works at Acme Inc. Jane Smith is from New York City." - result = await text_service.annotate_text_async(long_text) - expected_count = len(text_service._chunk_text(long_text)) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -def test_empty_string(text_service): - result = text_service.annotate_text_sync("") - assert result == {} - - -def test_short_string(text_service): - result = text_service.annotate_text_sync("Short") - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_special_characters(text_service): - result = text_service.annotate_text_sync("John@Doe.com works at Acme-Inc!!!") - expected_count = len(text_service._chunk_text("John@Doe.com works at Acme-Inc!!!")) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -def test_regex_engine(text_service_with_engine, mock_regex_annotator): - service = text_service_with_engine(engine="regex") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("john@example.com") - - # Should only call the regex annotator - assert mock_regex_annotator.annotate.called - assert not service.spacy_annotator.annotate.called - assert result == {"EMAIL": ["john@example.com"], "PHONE": ["555-555-5555"]} - - -def test_spacy_engine(text_service_with_engine, mock_annotator): - service = text_service_with_engine(engine="spacy") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("John Doe works at Acme Inc") - - # Should only call the spaCy annotator - assert mock_annotator.annotate.called - assert not service.regex_annotator.annotate.called - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_auto_engine_with_regex_results( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - # Configure regex annotator to return results - mock_regex_annotator.annotate.return_value = {"EMAIL": ["john@example.com"]} - - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("john@example.com") - - # Should call regex annotator but not spaCy - assert mock_regex_annotator.annotate.called - assert not mock_annotator.annotate.called - - assert result == {"EMAIL": ["john@example.com"]} - - -def test_auto_engine_with_fallback( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - # Configure regex annotator to return empty results - mock_regex_annotator.annotate.return_value = {"EMAIL": [], "PHONE": []} - - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("John Doe works at Acme Inc") - - # Should call both annotators - assert mock_regex_annotator.annotate.called - assert mock_annotator.annotate.called - - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_structured_output_regex_engine(text_service_with_engine, mock_regex_annotator): - """Test structured output mode with regex engine.""" - # Set up the mock to return spans that match the input text - from datafog.processing.text_processing.regex_annotator import ( - AnnotationResult, - Span, - ) - - # Create spans that will be returned by the mock - test_text = "john@example.com" - spans = [ - # Make sure the end position matches the actual length of the text - Span(label="EMAIL", start=0, end=len(test_text), text=test_text), - ] - - # Update the mock to return spans that match the input text - mock_regex_annotator.annotate_with_spans.return_value = ( - {"EMAIL": [test_text]}, - AnnotationResult(text=test_text, spans=spans), +async def test_async_detection_runs_outside_event_loop_thread(monkeypatch): + loop_thread = threading.get_ident() + worker_threads = [] + + def detect(text, **_): + worker_threads.append(threading.get_ident()) + return SimpleNamespace(entities=[]) + + monkeypatch.setattr(engine, "scan", detect) + await asyncio.wait_for(TextService().annotate_text_async("text"), timeout=2) + assert worker_threads and all(thread != loop_thread for thread in worker_threads) + + +@pytest.mark.parametrize("mode", ["auto", "smart"]) +def test_optional_ner_adds_to_structured_findings(monkeypatch, mode): + text = "Ann owns 6011111111111117" + monkeypatch.setattr( + engine, + "_gliner_entities", + lambda _: [engine.Entity("PERSON", "Ann", 0, 3, 0.8, "gliner")], ) + result = TextService(engine=mode).annotate_text_sync(text) + assert result == {"PERSON": ["Ann"], "CREDIT_CARD": ["6011111111111117"]} - service = text_service_with_engine(engine="regex") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync(test_text, structured=True) - - # Should call regex annotator's annotate_with_spans method - assert mock_regex_annotator.annotate_with_spans.called - - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 1 # Only one span should be returned (EMAIL) - - # Verify the span has the correct properties - assert result[0].label == "EMAIL" - assert result[0].text == test_text - assert result[0].start == 0 - assert result[0].end == len(test_text) - - -def test_structured_output_spacy_engine(text_service_with_engine, mock_annotator): - """Test structured output mode with spaCy engine.""" - service = text_service_with_engine(engine="spacy") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - # Set up mock to return entities that can be found in the test text - test_text = "John Doe works at Acme Inc" - mock_annotator.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} +def test_custom_gliner_model_is_forwarded(monkeypatch): + models = [] - result = service.annotate_text_sync(test_text, structured=True) + def detect(_, model_name): + models.append(model_name) + return [] - # Should call spaCy annotator - assert mock_annotator.annotate.called - - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 2 - - # Check that spans were created correctly - per_spans = [span for span in result if span.label == "PER"] - org_spans = [span for span in result if span.label == "ORG"] - - assert len(per_spans) == 1 - assert per_spans[0].text == "John Doe" - assert per_spans[0].start == test_text.find("John Doe") - assert per_spans[0].end == test_text.find("John Doe") + len("John Doe") - - assert len(org_spans) == 1 - assert org_spans[0].text == "Acme Inc" - assert org_spans[0].start == test_text.find("Acme Inc") - assert org_spans[0].end == test_text.find("Acme Inc") + len("Acme Inc") - - -def test_structured_output_auto_engine( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - """Test structured output mode with auto engine.""" - # Configure regex annotator to return empty spans - from datafog.processing.text_processing.regex_annotator import AnnotationResult - - mock_regex_annotator.annotate_with_spans.return_value = ( - {"EMAIL": [], "PHONE": []}, - AnnotationResult(text="test", spans=[]), + monkeypatch.setattr(engine, "_gliner_entities", detect) + assert ( + TextService(engine="smart", gliner_model="local/custom").annotate_text_sync( + "no pii" + ) + == {} ) + assert models == ["local/custom"] - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - - # Set up mock to return entities that can be found in the test text - test_text = "John Doe works at Acme Inc" - mock_annotator.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - result = service.annotate_text_sync(test_text, structured=True) +def test_locale_activates_additional_shared_detectors(): + text = "Aufenthaltstitel AT1234567 gueltig." + assert TextService(locales=["de"]).annotate_text_sync(text) == { + "DE_RESIDENCE_PERMIT_NUMBER": ["AT1234567"] + } - # Should call both annotators - assert mock_regex_annotator.annotate_with_spans.called - assert mock_annotator.annotate.called - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 2 +def test_empty_and_invalid_text(): + service = TextService() + assert service.annotate_text_sync("") == {} + assert service.annotate_text_sync("", structured=True) == [] + with pytest.raises(TypeError, match="text must be a string"): + service.annotate_text_sync(None) diff --git a/tests/test_text_service_integration.py b/tests/test_text_service_integration.py index d3363f5f..538f63c8 100644 --- a/tests/test_text_service_integration.py +++ b/tests/test_text_service_integration.py @@ -5,6 +5,15 @@ from datafog.services.text_service import TextService +@pytest.fixture +def require_spacy_model(): + spacy = pytest.importorskip("spacy") + try: + spacy.load("en_core_web_lg") + except OSError: + pytest.skip("en_core_web_lg must be provisioned for trained-model integration") + + @pytest.fixture def real_text_service(): """Create a real TextService instance for integration testing.""" @@ -32,7 +41,7 @@ def test_engine_regex_detects_simple_entities(): @pytest.mark.integration -def test_engine_auto_fallbacks_to_spacy(): +def test_engine_auto_fallbacks_to_spacy(require_spacy_model): """Test that auto mode works correctly with entity detection.""" # We need to test the auto mode in a more controlled way # Create a text that contains only named entities (no emails, phones, etc.) @@ -45,7 +54,7 @@ def test_engine_auto_fallbacks_to_spacy(): # Verify spaCy finds named entities assert "PERSON" in spacy_result and spacy_result["PERSON"] - assert "ORG" in spacy_result and spacy_result["ORG"] + assert "ORGANIZATION" in spacy_result and spacy_result["ORGANIZATION"] # Now create a special text that contains both regex-detectable and spaCy-detectable entities mixed_text = "John Smith's email is john.smith@example.com" @@ -61,7 +70,7 @@ def test_engine_auto_fallbacks_to_spacy(): @pytest.mark.integration -def test_engine_spacy_only(): +def test_engine_spacy_adds_to_structured_pii(require_spacy_model): """Test that spaCy engine is always used regardless of regex potential hits.""" # Sample text with both regex-detectable and spaCy-detectable entities text = """John Smith's email is john.smith@example.com. @@ -79,11 +88,10 @@ def test_engine_spacy_only(): # Verify spaCy detected named entities assert "PERSON" in spacy_result and spacy_result["PERSON"] - assert "ORG" in spacy_result and spacy_result["ORG"] + assert "ORGANIZATION" in spacy_result and spacy_result["ORGANIZATION"] - # Verify spaCy did NOT detect the email (which confirms it's using spaCy only) - # This is because spaCy doesn't have a built-in EMAIL entity type - assert "EMAIL" not in spacy_result or not spacy_result["EMAIL"] + # NER augments the shared structured detector. + assert "john.smith@example.com" in spacy_result["EMAIL"] @pytest.mark.integration @@ -130,7 +138,7 @@ def test_structured_annotation_output(): @pytest.mark.integration -def test_debug_entity_types(): +def test_spacy_entity_labels_are_canonical(require_spacy_model): """Debug test to print the actual entity types returned by spaCy.""" # Sample text with named entities text = """John Smith works at Microsoft Corporation in Seattle. @@ -146,8 +154,8 @@ def test_debug_entity_types(): if values: # Only print non-empty lists print(f" {entity_type}: {values}") - # No assertion needed, this is just for debugging - assert True + assert "ORGANIZATION" in spacy_result + assert "ORG" not in spacy_result @pytest.mark.skip(reason="Performance benchmarking requires more setup") From c486a529202ee9c6613105fad366991e2fb618f9 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:01:06 -0800 Subject: [PATCH 11/23] fix(engine): validate entity support for each optional NER engine --- datafog/engine.py | 10 +++++++--- datafog/services/text_service.py | 13 +++++++------ tests/test_review_engine.py | 5 +++++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/datafog/engine.py b/datafog/engine.py index 8ce61a5f..884bb1de 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -85,6 +85,8 @@ _core_warned = False _cnn_cache: tuple[object, object] | None = None +SPACY_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} + NER_ENTITY_TYPES = { "PERSON", "ORGANIZATION", @@ -455,7 +457,7 @@ def _spacy_entities(text: str) -> list[Entity]: return [ entity for entity in _annotator_entities(text, annotator, "spacy", 0.7) - if entity.type in NER_ENTITY_TYPES + if entity.type in SPACY_ENTITY_TYPES ] @@ -648,7 +650,9 @@ def _validate_entity_types( supported = supported | MODEL_ENTITY_TYPES else: supported = {_canonical_type(label) for label in RegexAnnotator.LABELS} - if engine != "regex": + if engine == "spacy": + supported |= SPACY_ENTITY_TYPES + elif engine in {"smart", "gliner"}: supported |= NER_ENTITY_TYPES | {"DATE"} unsupported = set(requested) - supported if unsupported: @@ -676,7 +680,7 @@ def scan( ``DATAFOG_BACKEND`` or pass ``backend="python"`` to force the fallback. The selected backend supplies structured PII for every engine mode; optional NER engines add entity spans. - Probe names and address blocks come from the embedded model on the native + Prose names and address blocks come from the embedded model on the native path, so ``PERSON`` and ``ADDRESS`` work without spaCy or a model download. Results retain legacy regex provenance and confidence; these are diff --git a/datafog/services/text_service.py b/datafog/services/text_service.py index eee7ca90..a7826a7e 100644 --- a/datafog/services/text_service.py +++ b/datafog/services/text_service.py @@ -1,8 +1,8 @@ """Lean text processing service for PII annotation. -Provides synchronous and asynchronous methods for annotating text with personally -identifiable information (PII) using regex patterns. Supports chunking long texts -and batch processing. SpaCy integration available as optional extra. +Provides synchronous, asynchronous and batch access to the shared PII engine. +Structured PII spans are scanned across complete documents; optional NLP engines +add spans with their original character coordinates. """ import asyncio @@ -31,8 +31,8 @@ class TextService: Lightweight service for annotating text with PII entities using regex patterns. This service provides methods to detect and annotate personally identifiable information (PII) - in text using fast regex patterns. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. + in text through the shared engine. Complete-document structured scanning preserves + entities across boundaries, and optional NLP engines add canonical entity spans. For advanced NLP-based detection using spaCy, install the 'nlp' extra: pip install datafog[nlp] @@ -49,7 +49,8 @@ def __init__( Initialize the TextService with specified chunk length and annotation engine. Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. + text_chunk_length: Positive compatibility setting. Structured detection scans the + complete document; the shared engine bounds native model chunks separately. engine: The annotation engine to use. Options are: - "regex": (Default) Use RegexAnnotator for fast pattern-based entity detection - "spacy": Use SpacyPIIAnnotator for NLP-based entity detection (requires nlp extra) diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py index 020f0935..9f47318a 100644 --- a/tests/test_review_engine.py +++ b/tests/test_review_engine.py @@ -324,3 +324,8 @@ def test_fallback_email_context_is_trimmed_without_losing_mailboxes(text): "dev@example.com", "bob@example.com", } + + +def test_spacy_selection_rejects_gliner_only_labels(): + with pytest.raises(ValueError, match="PASSPORT_NUMBER"): + engine.scan("passport SECRET", engine="spacy", entity_types=["PASSPORT_NUMBER"]) From 0102e136b34dbb13ee12463b74485f1e74b889f1 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:02:26 -0800 Subject: [PATCH 12/23] fix: complete review entrypoint, packaging and release safeguards --- .bumpversion.cfg | 2 +- .github/workflows/release.yml | 133 ++++---- .gitignore | 1 + .pre-commit-config.yaml | 4 +- CHANGELOG.MD | 33 ++ MANIFEST.in | 5 + README.md | 46 +-- datafog/__about__.py | 2 +- datafog/__init__.py | 72 +++-- datafog/__init___lean.py | 196 ------------ datafog/__init___original.py | 63 ---- datafog/_legacy_retirement.py | 4 +- datafog/cli.py | 17 + datafog/client.py | 24 +- datafog/compat/v4.py | 12 +- datafog/core.py | 26 +- datafog/main.py | 38 ++- datafog/main_lean.py | 196 ------------ datafog/main_original.py | 263 --------------- datafog/models/annotator.py | 37 ++- datafog/models/anonymizer.py | 70 ++-- datafog/models/common.py | 4 +- datafog/native.py | 10 +- .../spark_processing/pyspark_udfs.py | 16 +- datafog/services/text_service_lean.py | 197 ------------ datafog/services/text_service_original.py | 299 ------------------ docs/migration-4.10.md | 4 +- docs/retirement.rst | 18 +- docs/review-remediation.md | 73 +++++ pyproject.toml | 3 + scripts/check_rust_install.py | 17 +- scripts/release_version.py | 68 ++++ setup.py | 20 +- setup_lean.py | 109 ------- setup_original.py | 92 ------ tests/contracts/wrapper-4.12.0.json | 38 +-- tests/test_client.py | 2 +- tests/test_donut_lazy_import.py | 12 +- tests/test_image_service.py | 5 +- tests/test_legacy_retirement.py | 8 +- tests/test_litellm_guardrail.py | 5 +- tests/test_no_network_core.py | 18 +- tests/test_review_entrypoints.py | 229 ++++++++++++++ tests/test_runtime_dependency_safety.py | 6 +- tox.ini | 2 +- 45 files changed, 803 insertions(+), 1696 deletions(-) create mode 100644 MANIFEST.in delete mode 100644 datafog/__init___lean.py delete mode 100644 datafog/__init___original.py create mode 100644 datafog/cli.py delete mode 100644 datafog/main_lean.py delete mode 100644 datafog/main_original.py delete mode 100644 datafog/services/text_service_lean.py delete mode 100644 datafog/services/text_service_original.py create mode 100644 docs/review-remediation.md create mode 100644 pyproject.toml create mode 100644 scripts/release_version.py delete mode 100644 setup_lean.py delete mode 100644 setup_original.py create mode 100644 tests/test_review_entrypoints.py diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e72c9e2a..46abe29a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.9.0 +current_version = 4.12.0 commit = True tag = True tag_name = v{new_version} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d93c6bd8..32952a23 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,9 +2,9 @@ name: Release on: schedule: - # Monday-Wednesday at 2 AM UTC: alpha builds from dev + # Scheduled runs validate candidates without publishing. - cron: "0 2 * * 1-3" - # Thursday at 2 AM UTC: beta builds from dev + # Thursday beta qualification also stops before publication. - cron: "0 2 * * 4" workflow_dispatch: inputs: @@ -19,7 +19,7 @@ on: dry_run: description: "Dry run (skip PyPI publish)" required: false - default: false + default: true type: boolean dry_run_ref: description: "Candidate ref to validate (allowed only with dry_run)" @@ -86,6 +86,15 @@ jobs: fetch-depth: 0 ref: ${{ steps.resolve.outputs.checkout_ref }} + - name: Reject a development branch missing stable changes + if: steps.resolve.outputs.release_type != 'stable' + run: | + git fetch origin main + git merge-base --is-ancestor origin/main HEAD || { + echo "Development source is missing stable changes; merge main into dev first" + exit 1 + } + - name: Freeze source commit id: source run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" @@ -125,7 +134,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v6 with: @@ -272,7 +281,7 @@ jobs: pip install build twine bump2version - name: Configure git - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true run: | git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]" @@ -282,57 +291,20 @@ jobs: env: VERSION_OVERRIDE: ${{ inputs.version_override }} run: | - set -e + set -euo pipefail git fetch --tags - TYPE="${{ needs.determine-release.outputs.release_type }}" - CURRENT=$(sed -n 's/^__version__ = "\([^"]*\)"/\1/p' datafog/__about__.py) - if [ -z "$CURRENT" ]; then - echo "Failed to parse current version from datafog/__about__.py" - exit 1 - fi - echo "Current version: $CURRENT" - - # Strip any pre-release suffix to get base version - BASE=$(echo "$CURRENT" | sed -E 's/(a|b)[0-9]+([.][0-9A-Za-z]+)?$//') - if [ -n "$VERSION_OVERRIDE" ]; then - BASE="$VERSION_OVERRIDE" - if echo "$BASE" | grep -Eq '(a|b)[0-9]+([.][0-9A-Za-z]+)?$'; then - echo "version_override must be a stable base version like 4.4.0, not a prerelease" - exit 1 - fi - fi - if ! echo "$BASE" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then - echo "Release base must have numeric major.minor.patch format" - exit 1 - fi - echo "Base version: $BASE" - - if [ "$TYPE" = "alpha" ]; then - ALPHA_NUM=1 - while git tag -l "v${BASE}a${ALPHA_NUM}" | grep -q .; do - ALPHA_NUM=$((ALPHA_NUM + 1)) - done - VERSION="${BASE}a${ALPHA_NUM}" - - elif [ "$TYPE" = "beta" ]; then - BETA_NUM=1 - while git tag -l "v${BASE}b${BETA_NUM}" | grep -q .; do - BETA_NUM=$((BETA_NUM + 1)) - done - VERSION="${BASE}b${BETA_NUM}" - - else - VERSION="$BASE" - fi - + VERSION=$(python scripts/release_version.py --type "$TYPE" --override "$VERSION_OVERRIDE") echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "Publishing version: $VERSION" - - sed -i "s/__version__ = \".*\"/__version__ = \"$VERSION\"/" datafog/__about__.py - if grep -q 'version="' setup.py 2>/dev/null; then - sed -i "s/version=\".*\"/version=\"$VERSION\"/" setup.py - fi + RELEASE_VERSION="$VERSION" python - <<'PYCODE' + import os, re + from pathlib import Path + version = os.environ["RELEASE_VERSION"] + about = Path("datafog/__about__.py") + about.write_text(re.sub(r'^__version__ = ".*"', f'__version__ = "{version}"', about.read_text(), flags=re.M)) + config = Path(".bumpversion.cfg") + config.write_text(re.sub(r'^current_version = .*', f'current_version = {version}', config.read_text(), flags=re.M)) + PYCODE - name: Generate changelog env: @@ -371,7 +343,7 @@ jobs: if-no-files-found: error - name: Verify release branch still matches tested source - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: RELEASE_BRANCH: ${{ needs.determine-release.outputs.target_branch }} TESTED_SHA: ${{ needs.determine-release.outputs.source_sha }} @@ -382,30 +354,35 @@ jobs: exit 1 fi + - name: Commit and tag the validated release before publication + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true + env: + VERSION: ${{ steps.version.outputs.version }} + BRANCH: ${{ needs.determine-release.outputs.target_branch }} + run: | + set -euo pipefail + git add datafog/__about__.py .bumpversion.cfg + if ! git diff --cached --quiet; then + git commit -m "chore: bump version to $VERSION [skip ci]" + fi + git tag -a "v$VERSION" -m "Release $VERSION" + git push --atomic origin "HEAD:$BRANCH" "v$VERSION" + - name: Publish to PyPI - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | python -m twine upload dist/* --verbose - - name: Commit version bump & create release - if: inputs.dry_run != true + - name: Create GitHub release + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: GITHUB_TOKEN: ${{ secrets.GH_PAT }} + VERSION: ${{ steps.version.outputs.version }} + TYPE: ${{ needs.determine-release.outputs.release_type }} run: | - VERSION="${{ steps.version.outputs.version }}" - TYPE="${{ needs.determine-release.outputs.release_type }}" - BRANCH="${{ needs.determine-release.outputs.target_branch }}" - - git add datafog/__about__.py setup.py - git commit -m "chore: bump version to $VERSION [skip ci]" || echo "No version changes to commit" - git push origin "HEAD:$BRANCH" - - git tag -a "v$VERSION" -m "Release $VERSION" - git push origin "v$VERSION" - PRERELEASE_FLAG="" TITLE="" if [ "$TYPE" = "alpha" ]; then @@ -426,7 +403,7 @@ jobs: dist/* - name: Dry run summary - if: inputs.dry_run == true + if: github.event_name != 'workflow_dispatch' || inputs.dry_run == true run: | echo "DRY RUN COMPLETE" echo "Would have published: ${{ steps.version.outputs.version }}" @@ -434,9 +411,25 @@ jobs: ls -la dist/ # ── 4. Cleanup old pre-releases ─────────────────────────────────────── + sync-development: + needs: [determine-release, publish] + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.determine-release.outputs.release_type == 'stable' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: dev + fetch-depth: 0 + token: ${{ secrets.GH_PAT }} + - name: Bring stable release metadata back to development + run: | + git fetch origin main + git merge --ff-only origin/main + git push origin HEAD:dev + cleanup: needs: [determine-release, publish] - if: needs.determine-release.outputs.release_type != 'stable' && inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.determine-release.outputs.release_type != 'stable' runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/.gitignore b/.gitignore index 46214511..a594afa1 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ docs/* !docs/migration-4.8.1-contract.md !docs/migration-4.9.md !docs/core-0.4-integration.md +!docs/review-remediation.md !docs/agents/ !docs/agents/** !docs/audit/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4063257e..f6b76791 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort args: [--profile=black] exclude: .venv - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 26.3.1 hooks: - id: black language_version: python3 diff --git a/CHANGELOG.MD b/CHANGELOG.MD index ddc47dd9..6bf46b28 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,5 +1,38 @@ # ChangeLog +## [Unreleased] + +#### Security + +- Preserve native structured PII detection in smart, spaCy, GLiNER, legacy and + CLI paths. Normalize lone surrogates without moving source spans; retain + structured findings on bounded model inputs. Require review when hook payloads + exceed scan budgets, rather than silently allowing unscanned tails. +- Bound user allowlist matching with timeouts and remove quadratic overlap and + Python fallback regex behavior. Redact exact NER offsets and whole TextService + inputs so repeated names and chunk boundaries cannot expose PII. +- Cover LiteLLM tool arguments, prompt/input requests and complete streamed + responses, with request-scoped tokens. Hide configured API keys in CLI output. + +#### Changed + +- Hash APIs honor the selected algorithm and emit full digests. Optional + `hash_key` enables HMAC with a key of at least 32 bytes. Plain hashes provide + deterministic pseudonyms and do not resist guessing of low-entropy PII. +- Reuse tokens for repeated values, avoid existing token collisions, reject + invalid explicit ranges and unsupported entity selections, and report backend + entity capabilities. Async decorators await results before filtering. +- Keep Core within the reviewed minor series, raise the Pydantic floor to 2.1, + include spaCy in `nlp-advanced`, and allow Torch 2.x versions compatible with + `datafog-core[gliner2]`. Add an isolated build declaration and runnable sdist + tests; remove unused duplicate implementation modules. +- Repair Pydantic optional defaults, legacy anonymizer enum filters and overlaps, + Spark label aliases, implicit pipeline scanning and invalid CLI operations. +- Scheduled release jobs qualify candidates without publication. Manual runs + default to dry runs, reject stale version bases and development history, gate + Python 3.13, record release refs before upload, and synchronize stable metadata + back to development after publication. + ## [4.11.0] #### Changed diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..75a7cea8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include LICENSE README.md pyproject.toml tox.ini requirements-test.txt +recursive-include tests *.py *.json *.txt +recursive-include scripts *.py +prune tests/.datafog_env +global-exclude __pycache__ *.py[cod] diff --git a/README.md b/README.md index 172c0936..375cbc53 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DataFog Python +# DataFog Python DataFog is a Python library for detecting and redacting personally identifiable information (PII). @@ -85,8 +85,9 @@ multilingual tokenizer; installing the OCR extra is not required for GLiNER. `backend` defaults to `auto`: the native engine when `datafog-core` is importable, the pure-Python regex engine otherwise. `DATAFOG_BACKEND` sets the default and `backend="python"` / `backend="rust"` override it per call. -`auto` never selects the native engine for `engine="spacy"`, `"gliner"` or -`"smart"`, which stay on the Python ML path. +Every engine uses this backend for structured PII. `engine="spacy"`, +`"gliner"` and `"smart"` add model spans to the same structured findings; +selecting an ML engine does not disable native card or credential detection. The native engine adds detectors the Python engine never had — `API_KEY`, `BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, @@ -95,7 +96,7 @@ and `ADDRESS` from an embedded model, so name and address detection needs no spaCy and no model download. Findings keep `engine="regex"` provenance and `confidence=1.0` as compatibility values, not native confidence estimates. Reviewed differences from the Python engine are listed in -[tests/contracts/rust-1.3.0.json](tests/contracts/rust-1.3.0.json). +[tests/contracts/rust-1.4.0.json](tests/contracts/rust-1.4.0.json). `datafog.native` exposes the native schema directly. Python 3.13 support is certified for the core SDK, CLI, `nlp`, @@ -151,10 +152,8 @@ import datafog text = "Steuer-ID 12345678901 liegt vor." -print(datafog.scan(text, engine="regex").entities) -# [] - -print(datafog.scan(text, engine="regex", locales=["de"]).entities) +print(datafog.scan(text, engine="regex", locales=["de"], + entity_types=["DE_TAX_ID"]).entities) # [Entity(type='DE_TAX_ID', text='12345678901', ...)] ``` @@ -215,7 +214,8 @@ scan/redact helpers, or guardrail helpers. DataFog deprecates OCR and Spark with visible use-time warnings; their APIs and extras will be removed in 5.0. The last supported -version is 4.10.0 — pin it with `pip install "datafog==4.10.0"` — and every +version in this release train is 4.12.0 — after publication, pin it with +`pip install "datafog==4.12.0"` — and every retired surface is tracked in the [retirement schedule](docs/retirement.rst). See the [4.9 migration guide](docs/migration-4.9.md) for the transition plan, and [VERSIONING.md](VERSIONING.md) for what a version number promises. @@ -245,6 +245,8 @@ print(result["PHONE"]) ## CLI +Install the command dependencies with `pip install "datafog[cli]"`. + ```bash # Scan text datafog scan-text "john@example.com" @@ -284,20 +286,18 @@ Telemetry does not include input text or detected PII values. ## Development -The [4.9 migration guide](docs/migration-4.9.md) explains opt-in Rust detection, -the native `datafog.v5` preview, and the revised 5.0 retirement schedule for -`detect`/`process`, OCR, and Spark. The Python detector remains the default. - -4.10.0 requires Core `>=1.3.0,<2` and capability contract 1, and installs it by -default. See the [4.10.0 release notes](RELEASE_NOTES_4.10.0.md) and the -[4.10 migration guide](docs/migration-4.10.md). -Entity labels, locales, and activation settings come from the installed -Core, allowing compatible releases to add detectors without a Python update. -German detection is opt-in through locale or entity selection; UUID is opt-in -through `entity_types=["UUID"]`. Core's structured-only `PERSON` is unavailable -for explicit Rust text selection. The legacy overlap policy can suppress NPI in -favor of PHONE; the migration guide explains native alternatives. Lock the Core -version if detection output must remain reproducible. +The native engine is the default, with a warned Python fallback. The 4.12 +review fixes preserve structured detection across every engine and legacy API. +The dependency stays within the reviewed Core 1.4 minor series (`~=1.4.0`); +Core 2.0 is a separate release candidate that must be published and qualified +before this SDK raises its floor. See [release preparation](docs/review-remediation.md). + +Entity labels, locales and activation settings come from the installed Core. +`get_supported_entities()` reports that backend's supported inventory, including +opt-in labels; it does not mean every label is enabled by default. German +detection is opt-in through locale or entity selection. `UUID`, `PERSON` and +`ADDRESS` can be selected explicitly. Unsupported labels and invalid arguments +raise errors instead of silently disabling detection. The [4.8.1 compatibility contract](docs/migration-4.8.1-contract.md) records published Python behavior for the Rust migration, with frozen fixtures and diff --git a/datafog/__about__.py b/datafog/__about__.py index bdb8bda6..ade1e2b4 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.11.0" +__version__ = "4.12.0" diff --git a/datafog/__init__.py b/datafog/__init__.py index b0c7f2a8..f9a1695e 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -157,7 +157,7 @@ def scan( *, backend: str | None = None, ) -> ScanResult: - """Scan using the 4.x compatibility API; Rust detection is opt-in.""" + """Scan using the 4.x compatibility API; native detection is the default.""" return _compat_v4.scan( text, engine, @@ -181,6 +181,8 @@ def redact( allowlist_patterns: list[str] | None = None, *, backend: str | None = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Redact using the 4.x compatibility API and legacy strategies.""" return _compat_v4.redact( @@ -194,6 +196,8 @@ def redact( allowlist, allowlist_patterns, backend=backend, + hash_type=hash_type, + hash_key=hash_key, ) @@ -242,18 +246,15 @@ def _detect_impl(text: str) -> list: _start = _time.monotonic() - _lazy_import_regex_annotator() - annotator = RegexAnnotator() - # Use the structured output to get proper positions - _, result = annotator.annotate_with_spans(text) + result = scan(text) # Convert to simple format, filtering out empty matches entities = [] - for span in result.spans: + for span in result.entities: if span.text.strip(): # Only include non-empty matches entities.append( { - "type": span.label, + "type": span.type, "value": span.text, "start": span.start, "end": span.end, @@ -284,7 +285,14 @@ def _detect_impl(text: str) -> list: return entities -def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: +def process( + text: str, + anonymize: bool = False, + method: str = "redact", + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, +) -> dict: """ Process text to detect and optionally anonymize PII. @@ -311,31 +319,37 @@ def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: _start = _time.monotonic() + if method not in {"redact", "replace", "hash"}: + raise ValueError("method must be 'redact', 'replace', or 'hash'") findings = _detect_impl(text) result = {"original": text, "findings": findings} - if anonymize: - anonymized = text - # Simple anonymization - replace from end to start to preserve positions - for finding in sorted(findings, key=lambda x: x["start"], reverse=True): - start, end = finding["start"], finding["end"] - entity_type = finding["type"] - - if method == "redact": - replacement = f"[{entity_type}_REDACTED]" - elif method == "replace": - replacement = f"[{entity_type}_XXXXX]" - elif method == "hash": - import hashlib - - replacement = f"[{entity_type}_{hashlib.md5(finding['value'].encode()).hexdigest()[:8]}]" - else: - replacement = f"[{entity_type}]" - - anonymized = anonymized[:start] + replacement + anonymized[end:] - - result["anonymized"] = anonymized + if anonymize and method == "hash": + entities = [ + Entity( + item["type"], item["value"], item["start"], item["end"], 1.0, "regex" + ) + for item in findings + ] + result["anonymized"] = redact( + text, + entities=entities, + strategy="hash", + hash_type=hash_type, + hash_key=hash_key, + ).redacted_text + elif anonymize: + parts = [] + cursor = 0 + suffix = "REDACTED" if method == "redact" else "XXXXX" + for finding in sorted(findings, key=lambda item: item["start"]): + parts.extend( + (text[cursor : finding["start"]], f"[{finding['type']}_{suffix}]") + ) + cursor = finding["end"] + parts.append(text[cursor:]) + result["anonymized"] = "".join(parts) try: from .telemetry import _get_duration_bucket, track_function_call diff --git a/datafog/__init___lean.py b/datafog/__init___lean.py deleted file mode 100644 index 3f1165b8..00000000 --- a/datafog/__init___lean.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Historical shadow package export module. - -This file is not the live DataFog 4.5 package export surface. Use -``datafog/__init__.py`` for current package exports, lazy optional imports, -top-level helpers, and compatibility shims. This snapshot remains importable -only as historical reference until legacy cleanup can remove it safely. - -DataFog: Lightning-fast PII detection and anonymization library. - -Core package provides regex-based PII detection, 100x+ faster than NER-based -detection on identical payloads (reproduce: python benchmarks/run.py). -Optional extras available for advanced features: -- pip install datafog[nlp] - for spaCy integration -- pip install datafog[ocr] - for image/OCR processing -- pip install datafog[all] - for all features -""" - -from .__about__ import __version__ - -# Core imports - always available -from .models.annotator import AnnotationResult, AnnotatorRequest -from .models.anonymizer import ( - AnonymizationResult, - Anonymizer, - AnonymizerRequest, - AnonymizerType, -) -from .models.common import EntityTypes -from .processing.text_processing.regex_annotator import RegexAnnotator - -# Optional imports with graceful fallback -try: - from .client import app -except ImportError: - app = None - -try: - from .main import DataFog, TextPIIAnnotator -except ImportError: - DataFog = None - TextPIIAnnotator = None - -try: - from .services.text_service import TextService -except ImportError: - TextService = None - - -# Optional heavy features - only import if dependencies available -def _optional_import(name, module_path, extra_name): - """Helper to import optional modules with helpful error messages.""" - try: - module = __import__(module_path, fromlist=[name]) - return getattr(module, name) - except ImportError: - - def _missing_dependency(*args, **kwargs): - raise ImportError( - f"{name} requires additional dependencies. " - f"Install with: pip install datafog[{extra_name}]" - ) - - return _missing_dependency - - -# OCR/Image processing - requires 'ocr' extra -DonutProcessor = _optional_import( - "DonutProcessor", "datafog.processing.image_processing.donut_processor", "ocr" -) -PytesseractProcessor = _optional_import( - "PytesseractProcessor", - "datafog.processing.image_processing.pytesseract_processor", - "ocr", -) -ImageService = _optional_import("ImageService", "datafog.services.image_service", "ocr") - -# NLP processing - requires 'nlp' extra -SpacyPIIAnnotator = _optional_import( - "SpacyPIIAnnotator", "datafog.processing.text_processing.spacy_pii_annotator", "nlp" -) - -# Distributed processing - requires 'distributed' extra -SparkService = _optional_import( - "SparkService", "datafog.services.spark_service", "distributed" -) - - -# Simple API for core functionality -def detect(text: str) -> list: - """ - Detect PII in text using regex patterns. - - Args: - text: Input text to scan for PII - - Returns: - List of detected PII entities - - Example: - >>> from datafog import detect - >>> detect("Contact john@example.com") - [{'type': 'EMAIL', 'value': 'john@example.com', 'start': 8, 'end': 24}] - """ - annotator = RegexAnnotator() - result = annotator.annotate(text) - - # Convert to simple format - entities = [] - for entity_type, matches in result.items(): - for match in matches: - entities.append( - { - "type": entity_type, - "value": match, - "start": text.find(match), - "end": text.find(match) + len(match), - } - ) - - return entities - - -def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: - """ - Process text to detect and optionally anonymize PII. - - Args: - text: Input text to process - anonymize: Whether to anonymize detected PII - method: Anonymization method ('redact', 'replace', 'hash') - - Returns: - Dictionary with original text, anonymized text (if requested), and findings - - Example: - >>> from datafog import process - >>> process("Contact john@example.com", anonymize=True) - { - 'original': 'Contact john@example.com', - 'anonymized': 'Contact [EMAIL_REDACTED]', - 'findings': [{'type': 'EMAIL', 'value': 'john@example.com', ...}] - } - """ - findings = detect(text) - - result = {"original": text, "findings": findings} - - if anonymize: - anonymized = text - # Simple anonymization - replace from end to start to preserve positions - for finding in sorted(findings, key=lambda x: x["start"], reverse=True): - start, end = finding["start"], finding["end"] - entity_type = finding["type"] - - if method == "redact": - replacement = f"[{entity_type}_REDACTED]" - elif method == "replace": - replacement = f"[{entity_type}_XXXXX]" - elif method == "hash": - import hashlib - - replacement = f"[{entity_type}_{hashlib.md5(finding['value'].encode()).hexdigest()[:8]}]" - else: - replacement = f"[{entity_type}]" - - anonymized = anonymized[:start] + replacement + anonymized[end:] - - result["anonymized"] = anonymized - - return result - - -# Core exports -__all__ = [ - "__version__", - "detect", - "process", - "AnnotationResult", - "AnnotatorRequest", - "AnonymizationResult", - "Anonymizer", - "AnonymizerRequest", - "AnonymizerType", - "EntityTypes", - "RegexAnnotator", - # Optional exports (may be None if dependencies missing) - "DataFog", - "TextPIIAnnotator", - "TextService", - "app", - "DonutProcessor", - "PytesseractProcessor", - "ImageService", - "SpacyPIIAnnotator", - "SparkService", -] diff --git a/datafog/__init___original.py b/datafog/__init___original.py deleted file mode 100644 index 380511b1..00000000 --- a/datafog/__init___original.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Historical shadow package export module. - -This file is not the live DataFog 4.5 package export surface. Use -``datafog/__init__.py`` for current package exports, lazy optional imports, -top-level helpers, and compatibility shims. This eager-export snapshot remains -importable only as historical reference until legacy cleanup can remove it -safely. -""" - -from .__about__ import __version__ -from .client import app -from .config import OperationType, get_config -from .main import DataFog, TextPIIAnnotator -from .models.annotator import ( - AnalysisExplanation, - AnnotationResult, - AnnotationResultWithAnaysisExplanation, - AnnotatorRequest, -) -from .models.anonymizer import ( - AnonymizationResult, - Anonymizer, - AnonymizerRequest, - AnonymizerType, -) -from .models.common import AnnotatorMetadata, EntityTypes, Pattern, PatternRecognizer -from .models.spacy_nlp import SpacyAnnotator -from .processing.image_processing.donut_processor import DonutProcessor -from .processing.image_processing.image_downloader import ImageDownloader -from .processing.image_processing.pytesseract_processor import PytesseractProcessor -from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator -from .services.image_service import ImageService -from .services.spark_service import SparkService -from .services.text_service import TextService - -__all__ = [ - "DonutProcessor", - "DataFog", - "ImageService", - "OperationType", - "SparkService", - "TextPIIAnnotator", - "TextService", - "SpacyPIIAnnotator", - "ImageDownloader", - "PytesseractProcessor", - "__version__", - "app", - "AnalysisExplanation", - "AnnotationResult", - "AnnotationResultWithAnaysisExplanation", - "AnnotatorRequest", - "AnnotatorMetadata", - "EntityTypes", - "Pattern", - "PatternRecognizer", - "get_config", - "SpacyAnnotator", - "AnonymizerType", - "AnonymizerRequest", - "AnonymizationResult", - "Anonymizer", -] diff --git a/datafog/_legacy_retirement.py b/datafog/_legacy_retirement.py index e86506db..41753955 100644 --- a/datafog/_legacy_retirement.py +++ b/datafog/_legacy_retirement.py @@ -2,8 +2,10 @@ import warnings +from .__about__ import __version__ + #: Final release on which retired optional surfaces still work. -LAST_SUPPORTED = "4.10.0" +LAST_SUPPORTED = __version__ class LegacySurfaceWarning(FutureWarning): diff --git a/datafog/cli.py b/datafog/cli.py new file mode 100644 index 00000000..d5a86302 --- /dev/null +++ b/datafog/cli.py @@ -0,0 +1,17 @@ +"""Console bootstrap with an actionable optional dependency error.""" + + +def main(): + try: + import typer # noqa: F401 + except ImportError: + import sys + + print( + 'The datafog command requires the CLI extra: pip install "datafog[cli]"', + file=sys.stderr, + ) + raise SystemExit(1) from None + from .client import app + + app() diff --git a/datafog/client.py b/datafog/client.py index fd9a303d..cde33431 100644 --- a/datafog/client.py +++ b/datafog/client.py @@ -46,6 +46,15 @@ def list_entities(): app = typer.Typer() +def _operations(value: str) -> list[OperationType]: + try: + return [OperationType(op.strip()) for op in value.split(",")] + except ValueError: + raise typer.BadParameter( + "operations must use scan, extract, redact, replace, or hash" + ) from None + + @app.command() def scan_image( image_urls: List[str] = typer.Argument( @@ -70,9 +79,8 @@ def scan_image( typer.echo("No image URLs or file paths provided. Please provide at least one.") raise typer.Exit(code=1) - logging.basicConfig(level=logging.INFO) # Convert comma-separated string operations to a list of OperationType objects - operation_list = [OperationType(op.strip()) for op in operations.split(",")] + operation_list = _operations(operations) ocr_client = DataFog(operations=operation_list) try: results = asyncio.run(ocr_client.run_ocr_pipeline(image_urls=image_urls)) @@ -130,9 +138,8 @@ def scan_text( typer.echo("No texts provided.") raise typer.Exit(code=1) - logging.basicConfig(level=logging.INFO) # Convert comma-separated string operations to a list of OperationType objects - operation_list = [OperationType(op.strip()) for op in operations.split(",")] + operation_list = _operations(operations) text_client = DataFog(operations=operation_list, locales=locale) try: results = text_client.run_text_pipeline_sync(str_list=str_list) @@ -179,7 +186,11 @@ def show_config(): Prints the current DataFog configuration. """ - typer.echo(get_config()) + config = get_config() + values = dict(config) if isinstance(config, dict) else config.model_dump() + if values.get("api_key"): + values["api_key"] = "[REDACTED]" + typer.echo(values) @app.command() @@ -432,12 +443,11 @@ def hash_text( typer.echo("No text provided to hash.") raise typer.Exit(code=1) - # HashType is retained for backward-compatible CLI signature. - _ = hash_type result = scan_and_redact( text=text, engine="smart", strategy="hash", + hash_type=hash_type.value, locales=locale, ) typer.echo(result.redacted_text) diff --git a/datafog/compat/v4.py b/datafog/compat/v4.py index 96d619aa..ce6f00f4 100644 --- a/datafog/compat/v4.py +++ b/datafog/compat/v4.py @@ -66,6 +66,8 @@ def redact( allowlist_patterns: list[str] | None = None, *, backend: str | None = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """ Redact with legacy strategies and result schema. @@ -92,7 +94,13 @@ def redact( "allowlist/allowlist_patterns cannot be combined with explicit " "entities; filter the entities before calling redact" ) - return _redact_entities(text=text, entities=entities, strategy=strategy) + return _redact_entities( + text=text, + entities=entities, + strategy=strategy, + hash_type=hash_type, + hash_key=hash_key, + ) return _scan_and_redact( text=text, @@ -103,4 +111,6 @@ def redact( allowlist=allowlist, allowlist_patterns=allowlist_patterns, backend=backend, + hash_type=hash_type, + hash_key=hash_key, ) diff --git a/datafog/core.py b/datafog/core.py index e56c2aa5..2153ee94 100644 --- a/datafog/core.py +++ b/datafog/core.py @@ -89,6 +89,9 @@ def anonymize_text( text: str, method: Union[str, AnonymizerType] = "redact", locales: List[str] | None = None, + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> str: """ Simple text anonymization using lightweight regex engine. @@ -103,7 +106,7 @@ def anonymize_text( Example: >>> result = anonymize_text("Contact john@example.com", method="redact") >>> print(result) - "Contact [EMAIL_REDACTED]" + "Contact [EMAIL_1]" """ import time as _time @@ -129,6 +132,8 @@ def anonymize_text( engine=REGEX_ENGINE, strategy=strategy_map[method], locales=locales, + hash_type=hash_type, + hash_key=hash_key, ) try: @@ -211,18 +216,29 @@ def scan_text( return result -def get_supported_entities(locales: List[str] | None = None) -> List[str]: +def get_supported_entities( + locales: List[str] | None = None, *, backend: str | None = None +) -> List[str]: """ - Get list of PII entity types supported by the regex engine. + Get the selected backend's supported entity types, including opt-in labels. Returns: List of supported entity type names Example: >>> entities = get_supported_entities() - >>> print(entities) - ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DATE', 'ZIP_CODE'] + >>> "EMAIL" in entities + True """ + from datafog.engine import _require_core, resolve_backend + + if resolve_backend(backend, "regex") == "rust": + from datafog._core_capabilities import _read_capabilities, scan_configs + + core = _require_core() + supported, _, _ = _read_capabilities(core.capabilities) + scan_configs(core, set(), locales) + return sorted(supported) annotator = RegexAnnotator(locales=locales) legacy_map = {"DOB": "DATE", "ZIP": "ZIP_CODE"} result = [] diff --git a/datafog/main.py b/datafog/main.py index 0dc6f5d5..30fd1eed 100644 --- a/datafog/main.py +++ b/datafog/main.py @@ -15,12 +15,12 @@ from datafog._legacy_retirement import warn_legacy_surface from .config import OperationType +from .core import get_supported_entities from .engine import scan, scan_and_redact from .models.anonymizer import Anonymizer, AnonymizerType, HashType from .processing.text_processing.regex_annotator import RegexAnnotator logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) class DataFog: @@ -42,6 +42,7 @@ def __init__( hash_type: HashType = HashType.SHA256, anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, locales: List[str] | None = None, + hash_key: bytes | None = None, ): self.locales = locales self.regex_annotator = RegexAnnotator(locales=locales) @@ -59,6 +60,7 @@ def __init__( hash_type=hash_type, anonymizer_type=anonymizer_type ) self.hash_type = hash_type + self.hash_key = hash_key self.anonymizer_type = anonymizer_type self.logger = logging.getLogger(__name__) self.logger.info("Initializing lightweight DataFog class with regex engine") @@ -106,7 +108,15 @@ def run_text_pipeline_sync(self, str_list: List[str]) -> List: _start = _time.monotonic() try: self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: + if any( + op in self.operations + for op in ( + OperationType.SCAN, + OperationType.REDACT, + OperationType.REPLACE, + OperationType.HASH, + ) + ): annotated_text = [self.detect(text) for text in str_list] self.logger.info( @@ -190,8 +200,11 @@ def detect(self, text: str) -> dict: # Only pre-populate keys for labels active under the configured # locales so the default output shape matches v4.4.0 (no DE_* keys # unless German locale support is enabled). - result = {label: [] for label in RegexAnnotator.active_labels_for(self.locales)} legacy_map = {"DATE": "DOB", "ZIP_CODE": "ZIP"} + result = { + legacy_map.get(label, label): [] + for label in get_supported_entities(self.locales) + } for entity in scan_result.entities: label = legacy_map.get(entity.type, entity.type) result.setdefault(label, []).append(entity.text) @@ -239,6 +252,8 @@ def process( _start = _time.monotonic() + if method not in {"redact", "replace", "hash"}: + raise ValueError("method must be 'redact', 'replace', or 'hash'") annotations_dict = self.detect(text) result = {"original": text, "findings": annotations_dict} @@ -249,12 +264,18 @@ def process( "replace": "pseudonymize", "hash": "hash", } - strategy = strategy_map.get(method, "token") + strategy = strategy_map[method] redact_result = scan_and_redact( text=text, engine="regex", strategy=strategy, locales=self.locales, + hash_type=( + self.hash_type.value + if isinstance(self.hash_type, HashType) + else self.hash_type + ), + hash_key=self.hash_key, ) result["anonymized"] = redact_result.redacted_text @@ -298,6 +319,7 @@ class TextPIIAnnotator: """ def __init__(self, locales: List[str] | None = None): + self.locales = locales self.regex_annotator = RegexAnnotator(locales=locales) def run(self, text, output_path=None): @@ -312,7 +334,13 @@ def run(self, text, output_path=None): Dictionary mapping entity types to lists of found entities """ try: - annotated_text = self.regex_annotator.annotate(text) + result = scan(text=text, engine="regex", locales=self.locales) + legacy = {"DATE": "DOB", "ZIP_CODE": "ZIP"} + annotated_text = {} + for entity in result.entities: + annotated_text.setdefault( + legacy.get(entity.type, entity.type), [] + ).append(entity.text) # Optionally, output the results to a JSON file if output_path: diff --git a/datafog/main_lean.py b/datafog/main_lean.py deleted file mode 100644 index f67501e6..00000000 --- a/datafog/main_lean.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Historical shadow main module. - -This file is not the live DataFog 4.5 ``DataFog`` implementation. Use -``datafog/main.py`` for the current backward-compatible class and text/OCR -compatibility methods. This lean snapshot remains importable only as historical -reference until legacy cleanup can remove it safely. - -Lean main module for DataFog core functionality. - -This module contains the lightweight core classes for DataFog: -- DataFog: Main class for regex-based PII detection -- TextPIIAnnotator: Class for annotating PII in text using regex patterns - -These classes provide the core PII detection functionality without heavy dependencies. -""" - -import json -import logging -from typing import List - -from .config import OperationType -from .models.anonymizer import Anonymizer, AnonymizerType, HashType -from .processing.text_processing.regex_annotator import RegexAnnotator - -logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) - - -class DataFog: - """ - Lightweight main class for regex-based PII detection and anonymization. - - Handles text processing operations using fast regex patterns for PII detection. - For advanced features like OCR, spaCy, or Spark, install additional extras. - - Attributes: - regex_annotator: Core regex-based PII annotator. - operations: List of operations to perform. - anonymizer: Anonymizer for PII redaction, replacement, or hashing. - """ - - def __init__( - self, - operations: List[OperationType] = [OperationType.SCAN], - hash_type: HashType = HashType.SHA256, - anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, - ): - self.regex_annotator = RegexAnnotator() - self.operations: List[OperationType] = operations - self.anonymizer = Anonymizer( - hash_type=hash_type, anonymizer_type=anonymizer_type - ) - self.logger = logging.getLogger(__name__) - self.logger.info("Initializing lightweight DataFog class with regex engine") - self.logger.info(f"Operations: {operations}") - self.logger.info(f"Hash Type: {hash_type}") - self.logger.info(f"Anonymizer Type: {anonymizer_type}") - - def run_text_pipeline_sync(self, str_list: List[str]) -> List[str]: - """ - Run the text pipeline synchronously on a list of input text. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List[str]: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: - annotated_text = [] - - for text in str_list: - # Use regex annotator for core PII detection - annotations = self.regex_annotator.annotate(text) - annotated_text.append(annotations) - - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if any( - op in self.operations - for op in [ - OperationType.REDACT, - OperationType.REPLACE, - OperationType.HASH, - ] - ): - return [ - self.anonymizer.anonymize(text, annotations).anonymized_text - for text, annotations in zip( - str_list, annotated_text, strict=True - ) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return str_list - except Exception as e: - self.logger.error(f"Error in run_text_pipeline_sync: {str(e)}") - raise - - def detect(self, text: str) -> dict: - """ - Simple PII detection using regex patterns. - - Args: - text: Input text to scan for PII - - Returns: - Dictionary mapping entity types to lists of found entities - """ - return self.regex_annotator.annotate(text) - - def process( - self, text: str, anonymize: bool = False, method: str = "redact" - ) -> dict: - """ - Process text to detect and optionally anonymize PII. - - Args: - text: Input text to process - anonymize: Whether to anonymize detected PII - method: Anonymization method ('redact', 'replace', 'hash') - - Returns: - Dictionary with original text, anonymized text (if requested), and findings - """ - annotations = self.detect(text) - - result = {"original": text, "findings": annotations} - - if anonymize: - if method == "redact": - anonymizer_type = AnonymizerType.REDACT - elif method == "replace": - anonymizer_type = AnonymizerType.REPLACE - elif method == "hash": - anonymizer_type = AnonymizerType.HASH - else: - anonymizer_type = AnonymizerType.REDACT - - anonymized_result = self.anonymizer.anonymize( - text, annotations, anonymizer_type - ) - result["anonymized"] = anonymized_result.anonymized_text - - return result - - -class TextPIIAnnotator: - """ - Lightweight class for annotating PII in text using regex patterns. - - Provides functionality to detect and annotate Personally Identifiable Information (PII) - in text using fast regex patterns instead of heavy NLP models. - - Attributes: - regex_annotator: RegexAnnotator instance for text annotation. - """ - - def __init__(self): - self.regex_annotator = RegexAnnotator() - - def run(self, text, output_path=None): - """ - Run PII annotation on text using regex patterns. - - Args: - text: Input text to annotate - output_path: Optional path to save results as JSON - - Returns: - Dictionary mapping entity types to lists of found entities - """ - try: - annotated_text = self.regex_annotator.annotate(text) - - # Optionally, output the results to a JSON file - if output_path: - with open(output_path, "w") as f: - json.dump(annotated_text, f) - - return annotated_text - - except Exception as e: - logging.error(f"Error in TextPIIAnnotator.run: {str(e)}") - raise diff --git a/datafog/main_original.py b/datafog/main_original.py deleted file mode 100644 index 9594c824..00000000 --- a/datafog/main_original.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Historical shadow main module. - -This file is not the live DataFog 4.5 ``DataFog`` implementation. Use -``datafog/main.py`` for the current backward-compatible class and use the -optional OCR/Spark service modules directly when those surfaces are needed. -This original full-featured snapshot remains importable for legacy tests until -cleanup can remove it safely. - -Main module for DataFog. - -This module contains the core classes for DataFog: -- DataFog: Main class for running OCR and text processing pipelines. -- TextPIIAnnotator: Class for annotating PII in text. - -These classes provide high-level interfaces for image and text processing, -including OCR, PII detection, annotation, and anonymization. -""" - -import json -import logging -from typing import List - -from .config import OperationType -from .models.anonymizer import Anonymizer, AnonymizerType, HashType -from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator -from .services.image_service import ImageService -from .services.spark_service import SparkService -from .services.text_service import TextService - -logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) - - -class DataFog: - """ - Main class for running OCR and text processing pipelines. - - Handles image and text processing operations, including OCR, PII detection, and anonymization. - - Attributes: - image_service: Service for image processing and OCR. - text_service: Service for text processing and annotation. - spark_service: Optional Spark service for distributed processing. - operations: List of operations to perform. - anonymizer: Anonymizer for PII redaction, replacement, or hashing. - """ - - def __init__( - self, - image_service=None, - text_service=None, - spark_service=None, - operations: List[OperationType] = [OperationType.SCAN], - hash_type: HashType = HashType.SHA256, - anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, - ): - self.image_service = image_service or ImageService() - self.text_service = text_service or TextService() - self.spark_service: SparkService = spark_service - self.operations: List[OperationType] = operations - self.anonymizer = Anonymizer( - hash_type=hash_type, anonymizer_type=anonymizer_type - ) - self.logger = logging.getLogger(__name__) - self.logger.info( - "Initializing DataFog class with the following services and operations:" - ) - self.logger.info(f"Image Service: {type(self.image_service)}") - self.logger.info(f"Text Service: {type(self.text_service)}") - self.logger.info( - f"Spark Service: {type(self.spark_service) if self.spark_service else 'None'}" - ) - self.logger.info(f"Operations: {operations}") - self.logger.info(f"Hash Type: {hash_type}") - self.logger.info(f"Anonymizer Type: {anonymizer_type}") - - async def run_ocr_pipeline(self, image_urls: List[str]): - """ - Run the OCR pipeline asynchronously on a list of images provided via URL. - - This method performs optical character recognition (OCR) on the images specified by the URLs. - If PII annotation is enabled, it also annotates the extracted text for personally identifiable information. - If redaction, replacement, or hashing is enabled, it applies the corresponding anonymization. - - Args: - image_urls (List[str]): A list of URLs pointing to the images to be processed. - - Returns: - List: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the OCR or text processing. - """ - try: - extracted_text = await self.image_service.ocr_extract(image_urls) - self.logger.info(f"OCR extraction completed for {len(image_urls)} images.") - - return await self._process_text(extracted_text) - except Exception as e: - logging.error(f"Error in run_ocr_pipeline: {str(e)}") - return [f"Error: {str(e)}"] - - async def run_text_pipeline(self, str_list: List[str]): - """ - Run the text pipeline asynchronously on a list of input text. - - This method processes a list of text strings, potentially annotating them for personally - identifiable information (PII) and applying anonymization if enabled. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - return await self._process_text(str_list) - except Exception as e: - self.logger.error(f"Error in run_text_pipeline: {str(e)}") - raise - - async def _process_text(self, text_list: List[str]): - """ - Internal method to process text based on enabled operations. - """ - if OperationType.SCAN in self.operations: - annotated_text = await self.text_service.batch_annotate_text_async( - text_list - ) - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if OperationType.REDACT in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.REDACT - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - elif OperationType.REPLACE in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.REPLACE - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - elif OperationType.HASH in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.HASH - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return text_list - - def run_text_pipeline_sync(self, str_list: List[str]) -> List[str]: - """ - Run the text pipeline synchronously on a list of input text. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List[str]: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: - annotated_text = self.text_service.batch_annotate_text_sync(str_list) - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if any( - op in self.operations - for op in [ - OperationType.REDACT, - OperationType.REPLACE, - OperationType.HASH, - ] - ): - return [ - self.anonymizer.anonymize(text, annotations).anonymized_text - for text, annotations in zip( - str_list, annotated_text, strict=True - ) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return str_list - except Exception as e: - self.logger.error(f"Error in run_text_pipeline_sync: {str(e)}") - raise - - def _add_attributes(self, attributes: dict): - """ - Add multiple attributes to the DataFog instance. - - This private method allows for the dynamic addition of multiple attributes to the - DataFog instance. It iterates through the provided dictionary of attributes and - adds each key-value pair as an attribute. - - Args: - attributes (dict): A dictionary where keys are attribute names and values are - the corresponding attribute values to be added. - - Note: - This method is intended for internal use and may be used for extending the - functionality of the DataFog class dynamically. Care should be taken when - using this method to avoid overwriting existing attributes. - """ - for key, value in attributes.items(): - setattr(self, key, value) - - -class TextPIIAnnotator: - """ - Class for annotating PII in text. - - Provides functionality to detect and annotate Personally Identifiable Information (PII) in text. - - Attributes: - text_annotator: SpacyPIIAnnotator instance for text annotation. - spark_processor: Optional SparkService for distributed processing. - """ - - def __init__(self): - self.text_annotator = SpacyPIIAnnotator.create() - self.spark_processor: SparkService = None - - def run(self, text, output_path=None): - try: - annotated_text = self.text_annotator.annotate(text) - - # Optionally, output the results to a JSON file - if output_path: - with open(output_path, "w") as f: - json.dump(annotated_text, f) - - return annotated_text - - finally: - # Ensure Spark resources are released - if self.spark_processor: - self.spark_processor.stop() diff --git a/datafog/models/annotator.py b/datafog/models/annotator.py index 2f915ea4..489689b5 100644 --- a/datafog/models/annotator.py +++ b/datafog/models/annotator.py @@ -22,12 +22,12 @@ class AnnotatorRequest(BaseModel): text: str language: str - correlation_id: Optional[str] - score_threshold: Optional[float] - entities: Optional[List[EntityTypes]] - return_decision_process: Optional[bool] - ad_hoc_recognizers: Optional[List[PatternRecognizer]] - context: Optional[List[str]] + correlation_id: Optional[str] = None + score_threshold: Optional[float] = None + entities: Optional[List[EntityTypes]] = None + return_decision_process: Optional[bool] = None + ad_hoc_recognizers: Optional[List[PatternRecognizer]] = None + context: Optional[List[str]] = None class AnnotationResult(BaseModel): @@ -39,16 +39,19 @@ class AnnotationResult(BaseModel): start: int end: int - score: Optional[float] + score: Optional[float] = None entity_type: str - recognition_metadata: Optional[AnnotatorMetadata] + recognition_metadata: Optional[AnnotatorMetadata] = None - @field_validator("entity_type") + @field_validator("entity_type", mode="before") @classmethod def validate_entity_type(cls, v): - if v not in EntityTypes.__members__: - return "UNKNOWN" - return v + from datafog.engine import ALL_ENTITY_TYPES, CANONICAL_TYPE_MAP + + if isinstance(v, EntityTypes): + v = v.name + v = CANONICAL_TYPE_MAP.get(v, v) + return v if v in ALL_ENTITY_TYPES else "UNKNOWN" class AnalysisExplanation(BaseModel): @@ -60,14 +63,14 @@ class AnalysisExplanation(BaseModel): """ recognizer: str - pattern_name: Optional[str] - pattern: Optional[str] + pattern_name: Optional[str] = None + pattern: Optional[str] = None original_score: float score: float - textual_explanation: Optional[str] + textual_explanation: Optional[str] = None score_context_improvement: float supportive_context_word: str - validation_result: Optional[float] + validation_result: Optional[float] = None class AnnotationResultWithAnaysisExplanation(AnnotationResult): @@ -75,4 +78,4 @@ class AnnotationResultWithAnaysisExplanation(AnnotationResult): Extends AnnotationResult with detailed analysis explanation. """ - analysis_explanation: Optional[AnalysisExplanation] + analysis_explanation: Optional[AnalysisExplanation] = None diff --git a/datafog/models/anonymizer.py b/datafog/models/anonymizer.py index 6b81f5be..30d079bb 100644 --- a/datafog/models/anonymizer.py +++ b/datafog/models/anonymizer.py @@ -50,6 +50,33 @@ class Anonymizer(BaseModel): def __init__(self, **data): super().__init__(**data) + def _selected_annotations(self, text, annotations): + from datafog.engine import CANONICAL_TYPE_MAP + + selected = ( + None + if self.entities is None + else { + CANONICAL_TYPE_MAP.get(entity.name, entity.name) + for entity in self.entities + } + ) + merged = [] + for annotation in sorted(annotations, key=lambda a: (a.start, -a.end)): + if not 0 <= annotation.start < annotation.end <= len(text): + raise ValueError( + "Annotation range must be nonempty and inside the text" + ) + if selected is not None and annotation.entity_type not in selected: + continue + if merged and annotation.start < merged[-1].end: + merged[-1] = merged[-1].model_copy( + update={"end": max(merged[-1].end, annotation.end)} + ) + else: + merged.append(annotation) + return reversed(merged) + def anonymize( self, text: str, annotations: List[AnnotationResult] ) -> AnonymizationResult: @@ -68,19 +95,18 @@ def replace_pii( ) -> AnonymizationResult: """Replace PII in text with anonymized values.""" replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if not self.entities or annotation.entity_type in self.entities: - replacement = self._generate_replacement( - text[annotation.start : annotation.end], annotation.entity_type - ) - replacements.append( - { - "original": text[annotation.start : annotation.end], - "replacement": replacement, - "entity_type": annotation.entity_type, - } - ) - text = text[: annotation.start] + replacement + text[annotation.end :] + for annotation in self._selected_annotations(text, annotations): + replacement = self._generate_replacement( + text[annotation.start : annotation.end], annotation.entity_type + ) + replacements.append( + { + "original": text[annotation.start : annotation.end], + "replacement": replacement, + "entity_type": annotation.entity_type, + } + ) + text = text[: annotation.start] + replacement + text[annotation.end :] return AnonymizationResult( anonymized_text=text, anonymized_entities=replacements @@ -88,13 +114,13 @@ def replace_pii( def _generate_replacement(self, original: str, entity_type: EntityTypes) -> str: """Generate a replacement for the given entity.""" - if entity_type == EntityTypes.PERSON: + if entity_type == "PERSON": return f"[PERSON_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.ORGANIZATION: + elif entity_type == "ORGANIZATION": return f"[ORGANIZATION_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.LOCATION: + elif entity_type == "LOCATION": return f"[LOCATION_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.DATE: + elif entity_type == "DATE": return "[REDACTED_DATE]" else: return f"[{entity_type}_{secrets.token_hex(4).upper()}]" @@ -104,10 +130,7 @@ def hash_pii( ) -> AnonymizationResult: """Hash PII in text.""" replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if self.entities and annotation.entity_type not in self.entities: - continue - + for annotation in self._selected_annotations(text, annotations): start, end = annotation.start, annotation.end original = text[start:end] replacement = self._hash_text(original) @@ -139,10 +162,7 @@ def redact_pii( self, text: str, annotations: List[AnnotationResult] ) -> AnonymizationResult: replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if self.entities and annotation.entity_type not in self.entities: - continue - + for annotation in self._selected_annotations(text, annotations): start, end = annotation.start, annotation.end original = text[start:end] replacement = "[REDACTED]" diff --git a/datafog/models/common.py b/datafog/models/common.py index f01d6651..1f72d4aa 100644 --- a/datafog/models/common.py +++ b/datafog/models/common.py @@ -40,8 +40,8 @@ class PatternRecognizer(BaseModel): name: str supported_language: str patterns: List[Pattern] - deny_list: Optional[List[str]] - context: Optional[List[str]] + deny_list: Optional[List[str]] = None + context: Optional[List[str]] = None supported_entity: str diff --git a/datafog/native.py b/datafog/native.py index cde64934..44b6fb29 100644 --- a/datafog/native.py +++ b/datafog/native.py @@ -40,7 +40,7 @@ def _core() -> Any: except ImportError as exc: # pragma: no cover - base install always has it raise ImportError( "datafog.native requires datafog-core. Install with: " - 'pip install "datafog==4.10.0"' + 'pip install "datafog[rust]"' ) from exc @@ -52,7 +52,13 @@ def __getattr__(name: str) -> Any: def version() -> str: """Installed Core version, for bug reports and reproducibility locks.""" - return importlib.metadata.version("datafog-core") + _core() + try: + return importlib.metadata.version("datafog-core") + except importlib.metadata.PackageNotFoundError as exc: + raise ImportError( + "datafog-core distribution metadata is missing; reinstall datafog[rust]" + ) from exc def contract_version() -> int: diff --git a/datafog/processing/spark_processing/pyspark_udfs.py b/datafog/processing/spark_processing/pyspark_udfs.py index aaa53ece..bb9b8c18 100644 --- a/datafog/processing/spark_processing/pyspark_udfs.py +++ b/datafog/processing/spark_processing/pyspark_udfs.py @@ -28,8 +28,9 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]: if text: if len(text) > MAXIMAL_STRING_SIZE: - # Cut the strings for required sizes - text = text[:MAXIMAL_STRING_SIZE] + raise ValueError( + "Spark PII input exceeds the maximum supported string length" + ) nlp = broadcasted_nlp.value doc = nlp(text) @@ -38,8 +39,15 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]: _label: [] for _label in PII_ANNOTATION_LABELS } for ent in doc.ents: - # Add entities from extracted values - classified_entities[ent.label_].append(ent.text) + label = { + "PERSON": "PER", + "GPE": "LOC", + "NORP": "NRP", + "DATE": "DATE_TIME", + "TIME": "DATE_TIME", + }.get(ent.label_, ent.label_) + if label in classified_entities: + classified_entities[label].append(ent.text) return [_ent for _ent in classified_entities.values()] else: diff --git a/datafog/services/text_service_lean.py b/datafog/services/text_service_lean.py deleted file mode 100644 index 1262db3f..00000000 --- a/datafog/services/text_service_lean.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Historical shadow text service module. - -This file is not the live DataFog 4.5 text service. Use -``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER, -auto, and smart engine boundary. This lean snapshot remains importable only as -historical reference until legacy cleanup can remove it safely. - -Lean text processing service for PII annotation. - -Provides synchronous and asynchronous methods for annotating text with personally -identifiable information (PII) using regex patterns. Supports chunking long texts -and batch processing. SpaCy integration available as optional extra. -""" - -import asyncio -from typing import Dict, List, Union - -from datafog.processing.text_processing.regex_annotator.regex_annotator import ( - RegexAnnotator, - Span, -) - - -class TextService: - """ - Lightweight service for annotating text with PII entities using regex patterns. - - This service provides methods to detect and annotate personally identifiable information (PII) - in text using fast regex patterns. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. - - For advanced NLP-based detection using spaCy, install the 'nlp' extra: - pip install datafog[nlp] - """ - - def __init__(self, text_chunk_length: int = 1000, engine: str = "regex"): - """ - Initialize the TextService with specified chunk length and annotation engine. - - Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. - engine: The annotation engine to use. Options are: - - "regex": (Default) Use RegexAnnotator for fast pattern-based entity detection - - "spacy": Use SpacyPIIAnnotator for NLP-based entity detection (requires nlp extra) - - "auto": Try RegexAnnotator first and fall back to SpacyPIIAnnotator if no entities found - - Raises: - AssertionError: If an invalid engine type is provided - ImportError: If spacy engine is requested but nlp extra is not installed - """ - assert engine in {"regex", "spacy", "auto"}, "Invalid engine" - self.engine = engine - self.regex_annotator = RegexAnnotator() - self.text_chunk_length = text_chunk_length - - # Only initialize spacy if needed and available - self.spacy_annotator = None - if engine in {"spacy", "auto"}: - try: - from datafog.processing.text_processing.spacy_pii_annotator import ( - SpacyPIIAnnotator, - ) - - self.spacy_annotator = SpacyPIIAnnotator.create() - except ImportError: - if engine == "spacy": - raise ImportError( - "SpaCy engine requires additional dependencies. " - "Install with: pip install datafog[nlp]" - ) - # For auto mode, just continue with regex only - self.spacy_annotator = None - - def _chunk_text(self, text: str) -> List[str]: - """Split the text into chunks of specified length.""" - return [ - text[i : i + self.text_chunk_length] - for i in range(0, len(text), self.text_chunk_length) - ] - - def _combine_annotations( - self, chunk_annotations: List[Dict[str, List[str]]] - ) -> Dict[str, List[str]]: - """Combine annotations from multiple chunks.""" - combined = {} - for annotations in chunk_annotations: - for entity_type, entities in annotations.items(): - if entity_type not in combined: - combined[entity_type] = [] - combined[entity_type].extend(entities) - return combined - - def annotate_text_sync( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text synchronously for PII entities. - - Args: - text: The text to annotate - structured: If True, return structured Span objects. If False, return dict format. - - Returns: - Dictionary mapping entity types to lists of entities, or list of Span objects - """ - if len(text) <= self.text_chunk_length: - # Single chunk processing - if self.engine == "regex": - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return self.regex_annotator.annotate(text) - elif self.engine == "spacy": - if self.spacy_annotator is None: - raise ImportError( - "SpaCy engine not available. Install with: pip install datafog[nlp]" - ) - return self.spacy_annotator.annotate(text) - elif self.engine == "auto": - # Try regex first - regex_result = self.regex_annotator.annotate(text) - - # Check if regex found any entities - if any(entities for entities in regex_result.values()): - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return regex_result - - # Fall back to spacy if available - if self.spacy_annotator is not None: - return self.spacy_annotator.annotate(text) - - # Return regex result even if empty - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return regex_result - else: - # Multi-chunk processing - chunks = self._chunk_text(text) - chunk_annotations = [] - - for chunk in chunks: - chunk_result = self.annotate_text_sync(chunk, structured=False) - chunk_annotations.append(chunk_result) - - if structured: - # For structured output with chunking, we need to recalculate positions - # This is more complex, so for now return dict format - return self._combine_annotations(chunk_annotations) - - return self._combine_annotations(chunk_annotations) - - async def annotate_text_async( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text asynchronously for PII entities. - - Args: - text: The text to annotate - structured: If True, return structured Span objects. If False, return dict format. - - Returns: - Dictionary mapping entity types to lists of entities, or list of Span objects - """ - # For regex processing, we can just run synchronously since it's fast - return self.annotate_text_sync(text, structured) - - def batch_annotate_text_sync(self, texts: List[str]) -> List[Dict[str, List[str]]]: - """ - Annotate multiple texts synchronously. - - Args: - texts: List of texts to annotate - - Returns: - List of annotation dictionaries, one per input text - """ - return [self.annotate_text_sync(text) for text in texts] - - async def batch_annotate_text_async( - self, texts: List[str] - ) -> List[Dict[str, List[str]]]: - """ - Annotate multiple texts asynchronously. - - Args: - texts: List of texts to annotate - - Returns: - List of annotation dictionaries, one per input text - """ - # For better performance with many texts, we can process them concurrently - tasks = [self.annotate_text_async(text) for text in texts] - return await asyncio.gather(*tasks) diff --git a/datafog/services/text_service_original.py b/datafog/services/text_service_original.py deleted file mode 100644 index 8ad576a9..00000000 --- a/datafog/services/text_service_original.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Historical shadow text service module. - -This file is not the live DataFog 4.5 text service. Use -``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER, -auto, and smart engine boundary. This original snapshot remains importable for -legacy tests until cleanup can remove it safely. - -Text processing service for PII annotation. - -Provides synchronous and asynchronous methods for annotating text with personally identifiable information (PII) using SpaCy or regex patterns. Supports chunking long texts and batch processing. -""" - -import asyncio -from typing import Dict, List, Union - -from datafog.processing.text_processing.regex_annotator.regex_annotator import ( - RegexAnnotator, - Span, -) -from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator - - -class TextService: - """ - Service for annotating text with PII entities. - - This service provides methods to detect and annotate personally identifiable information (PII) - in text using different annotation engines. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. - """ - - def __init__(self, text_chunk_length: int = 1000, engine: str = "auto"): - """ - Initialize the TextService with specified chunk length and annotation engine. - - Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. - engine: The annotation engine to use. Options are: - - "regex": Use only the RegexAnnotator for pattern-based entity detection - - "spacy": Use only the SpacyPIIAnnotator for NLP-based entity detection - - "auto": (Default) Try RegexAnnotator first and fall back to SpacyPIIAnnotator if no entities are found - - Raises: - AssertionError: If an invalid engine type is provided - """ - assert engine in {"regex", "spacy", "auto"}, "Invalid engine" - self.engine = engine - self.spacy_annotator = SpacyPIIAnnotator.create() - self.regex_annotator = RegexAnnotator() - self.text_chunk_length = text_chunk_length - - def _chunk_text(self, text: str) -> List[str]: - """Split the text into chunks of specified length.""" - return [ - text[i : i + self.text_chunk_length] - for i in range(0, len(text), self.text_chunk_length) - ] - - def _combine_annotations( - self, annotations: List[Dict[str, List[str]]] - ) -> Dict[str, List[str]]: - """Combine annotations from multiple chunks.""" - combined: Dict[str, List[str]] = {} - for annotation in annotations: - for key, value in annotation.items(): - if key not in combined: - combined[key] = [] - combined[key].extend(value) - return combined - - def _annotate_with_engine( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text using the selected engine based on the engine parameter. - - This method implements the engine selection logic: - - For "regex" mode: Uses only the RegexAnnotator - - For "spacy" mode: Uses only the SpacyPIIAnnotator - - For "auto" mode: Tries RegexAnnotator first and falls back to SpacyPIIAnnotator if no entities are found - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary of annotations by entity type where keys are entity types (e.g., "EMAIL", "PERSON", "ORG") - and values are lists of detected entities of that type - If structured=True: List of Span objects with entity information - """ - if structured: - # Handle structured output mode - if self.engine == "regex": - _, annotation_result = self.regex_annotator.annotate_with_spans(text) - return annotation_result.spans - elif self.engine == "spacy": - # For spaCy, we need to convert the dictionary format to spans - spacy_dict = self.spacy_annotator.annotate(text) - spacy_spans: List[Span] = [] - for label, entities in spacy_dict.items(): - for entity in entities: - # Find the start and end positions of the entity in the text - start = text.find(entity) - if start >= 0: - end = start + len(entity) - span = Span(start=start, end=end, label=label, text=entity) - spacy_spans.append(span) - return spacy_spans - else: # "auto" mode - # Try regex first - _, annotation_result = self.regex_annotator.annotate_with_spans(text) - if annotation_result.spans: - return annotation_result.spans - - # If regex found nothing, fall back to spaCy - spacy_dict = self.spacy_annotator.annotate(text) - auto_spans: List[Span] = [] - for label, entities in spacy_dict.items(): - for entity in entities: - # Find the start and end positions of the entity in the text - start = text.find(entity) - if start >= 0: - end = start + len(entity) - span = Span(start=start, end=end, label=label, text=entity) - auto_spans.append(span) - return auto_spans - else: - # Handle legacy dictionary output mode - if self.engine == "regex": - return self.regex_annotator.annotate(text) - elif self.engine == "spacy": - return self.spacy_annotator.annotate(text) - else: # auto mode - # Try regex first - regex_dict = self.regex_annotator.annotate(text) - - # Check if any VALID entities were found (ignore empty strings) - has_entities = any( - any(entity.strip() for entity in entities) - for entities in regex_dict.values() - ) - - # If regex found entities, return those results - if has_entities: - return regex_dict - - # Otherwise, fall back to spaCy - return self.spacy_annotator.annotate(text) - - def annotate_text_sync( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Synchronously annotate a text string. - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary mapping entity types to lists of strings - If structured=True: List of Span objects with entity information - """ - if not text: - return [] if structured else {} - - chunks = self._chunk_text(text) - - if structured: - # Handle structured output mode - all_spans: List[Span] = [] - chunk_offset = 0 # Track the offset for each chunk in the original text - - for chunk in chunks: - # Process each chunk and get spans - chunk_spans = self._annotate_with_engine(chunk, structured=True) - if not isinstance(chunk_spans, list): - continue # Skip if not a list of spans - - # Adjust span positions based on chunk offset in the original text - for span in chunk_spans: - if not isinstance(span, Span): - continue # Skip if not a Span object - span.start += chunk_offset - span.end += chunk_offset - # Verify the span text matches the text at the adjusted position - if span.start < len(text) and span.end <= len(text): - span.text = text[span.start : span.end] - all_spans.append(span) - - # Update offset for the next chunk - chunk_offset += len(chunk) - - print(f"Done processing {text.split()[0]}") - return all_spans - else: - # Handle legacy dictionary output mode - annotations: List[Dict[str, List[str]]] = [] - for chunk in chunks: - res = self._annotate_with_engine(chunk) - if isinstance(res, dict): - annotations.append(res) - combined = self._combine_annotations(annotations) - print(f"Done processing {text.split()[0]}") - return combined - - def batch_annotate_text_sync( - self, texts: List[str], structured: bool = False - ) -> Dict[str, Union[Dict[str, List[str]], List[Span]]]: - """ - Synchronously annotate a list of text input. - - Args: - texts: List of text strings to annotate - structured: If True, return structured output (list of Span objects) for each text - - Returns: - Dictionary mapping each input text to its annotation result - """ - results = [ - self.annotate_text_sync(text, structured=structured) for text in texts - ] - return dict(zip(texts, results, strict=True)) - - async def annotate_text_async( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Asynchronously annotate a text string. - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary mapping entity types to lists of strings - If structured=True: List of Span objects with entity information - """ - if not text: - return [] if structured else {} - - chunks = self._chunk_text(text) - - if structured: - # Handle structured output mode asynchronously - all_spans: List[Span] = [] - chunk_offset = 0 # Track the offset for each chunk in the original text - - for chunk in chunks: - # We can't easily parallelize this due to the need to track offsets sequentially - # In a production environment, you might want a more sophisticated approach - chunk_spans = self._annotate_with_engine(chunk, structured=True) - if not isinstance(chunk_spans, list): - continue # Skip if not a list of spans - - # Adjust span positions based on chunk offset in the original text - for span in chunk_spans: - if not isinstance(span, Span): - continue # Skip if not a Span object - span.start += chunk_offset - span.end += chunk_offset - # Verify the span text matches the text at the adjusted position - if span.start < len(text) and span.end <= len(text): - span.text = text[span.start : span.end] - all_spans.append(span) - - # Update offset for the next chunk - chunk_offset += len(chunk) - - return all_spans - else: - # Handle legacy dictionary output mode asynchronously - tasks = [ - asyncio.to_thread(self._annotate_with_engine, chunk) for chunk in chunks - ] - results = await asyncio.gather(*tasks) - annotations: List[Dict[str, List[str]]] = [ - r for r in results if isinstance(r, dict) - ] - return self._combine_annotations(annotations) - - async def batch_annotate_text_async( - self, texts: List[str], structured: bool = False - ) -> Dict[str, Union[Dict[str, List[str]], List[Span]]]: - """ - Asynchronously annotate a list of text input. - - Args: - texts: List of text strings to annotate - structured: If True, return structured output (list of Span objects) for each text - - Returns: - Dictionary mapping each input text to its annotation result - """ - tasks = [ - self.annotate_text_async(text, structured=structured) for text in texts - ] - results = await asyncio.gather(*tasks) - return dict(zip(texts, results, strict=True)) diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md index 9a730c3b..6ea5db00 100644 --- a/docs/migration-4.10.md +++ b/docs/migration-4.10.md @@ -40,7 +40,7 @@ The native engine finds things the Python regex engine never did: `API_KEY`, stay opt-in. These differences go the other way and are recorded, not hidden, in -[tests/contracts/rust-1.3.0.json](../tests/contracts/rust-1.3.0.json): +[tests/contracts/rust-1.4.0.json](../tests/contracts/rust-1.4.0.json): | Input | Python | Native | Why | | --- | --- | --- | --- | @@ -105,5 +105,5 @@ a contract bump is a Core MAJOR and raises the floor here. ## What is not changing yet `detect()`, `process()`, OCR and Spark all still work and still warn. They are -removed in 5.0, not here. The last supported version is 4.10.0 — see the +removed in 5.0, not here. The last supported version in the current release train is 4.12.0 — see the [retirement schedule](retirement.rst). diff --git a/docs/retirement.rst b/docs/retirement.rst index 17c5c0af..01b2fc53 100644 --- a/docs/retirement.rst +++ b/docs/retirement.rst @@ -21,37 +21,37 @@ Removed or scheduled - Removed in - Deprecated in * - ``OCRMService`` - - ``4.10.0`` + - ``4.12.0`` - none - ``5.0.0`` - ``4.9.0`` * - ``DonutProcessor`` - - ``4.10.0`` + - ``4.12.0`` - none - ``5.0.0`` - ``4.9.0`` * - ``[ocr]`` extra (Donut, Tesseract, Pillow) - - ``4.10.0`` + - ``4.12.0`` - none - ``5.0.0`` - ``4.9.0`` * - ``[distributed]`` extra (PySpark, pandas) - - ``4.10.0`` + - ``4.12.0`` - :ref:`spark-recipe` - ``5.0.0`` - ``4.9.0`` * - ``detect()`` - - ``4.10.0`` + - ``4.12.0`` - ``scan()`` - ``5.0.0`` - ``4.9.0`` * - ``process()`` - - ``4.10.0`` + - ``4.12.0`` - ``redact()`` - ``5.0.0`` - ``4.9.0`` -Installing ``pip install "datafog==4.10.0"`` keeps every row above working. +After the 4.12.0 release is published, installing ``pip install "datafog==4.12.0"`` keeps every row above working. That version stays installable on PyPI permanently. OCR @@ -63,9 +63,9 @@ pin. .. code-block:: bash - pip install "datafog==4.10.0[ocr]" + pip install "datafog[ocr]==4.12.0" -If you need continued OCR support beyond that, stay on ``4.10.0``. Track the +If you need continued OCR support beyond that, stay on ``4.12.0``. Track the surfaces you depend on by name, not by staying on a major version range, so a lock file review shows the pin. diff --git a/docs/review-remediation.md b/docs/review-remediation.md new file mode 100644 index 00000000..2bcb8ddd --- /dev/null +++ b/docs/review-remediation.md @@ -0,0 +1,73 @@ +# November 2026 review remediation + +This candidate addresses the review of DataFog 4.11.0 and Core 1.4.0. The +review's suggested fixes were assessed against the local implementations; +regression tests cover the implemented behavior. + +## Observable behavior + +All engine modes and legacy scanning APIs share the structured scan boundary. +Optional NER adds spans with its own character offsets. TextService scans the +whole document; `text_chunk_length` remains validated for compatibility. +Unsupported labels, invalid ranges and invalid locales fail explicitly. Source +code-point offsets remain stable when lone surrogate characters are replaced +for native transport. `get_supported_entities()` reports supported labels, +including opt-in ones, rather than only currently active default labels. + +The `create_guardrail` decorator supports sync and async functions returning +strings. Non-string results raise TypeError; they cannot silently bypass the +filter. Apply a guardrail directly to the text fields of structured results. + +LiteLLM scans messages, nested tool/function arguments, prompt and input fields. +Image and binary parts are retained without OCR. Tokens are assigned per request +or response and repeated values reuse their token. Streamed response text and +tool arguments are buffered until completion, then scanned before any chunk is +emitted. This adds completion latency. Exceeding eight million text characters +or 100,000 chunks rejects the stream. Engine failures follow the explicit +`fail_policy`; deployments requiring protected egress should select `closed`. +The Claude Code hook requests review or denies oversized PreToolUse inputs; +nonblocking events receive an explicit incomplete-scan notice. Unexpected hook +failures retain the documented fail-open contract and log exception types only. + +`hash_type` selects md5, sha256 or sha3_256 on the CLI and legacy model APIs. +The engine also accepts algorithms provided by hashlib. Digests use full hex +lengths. Plain deterministic hashes are pseudonymization: SSNs, phones and +other guessable inputs can be enumerated. Use `hash_key` (at least 32 random +bytes) for keyed HMAC and keep the key outside logs and source control: + +```python +import secrets +import datafog +key = secrets.token_bytes(32) # store securely to reproduce pseudonyms +result = datafog.redact("mail jane@example.com", strategy="hash", hash_key=key) +``` + +## Release sequence and qualification + +The Python candidate is 4.12.0. Its standalone CI uses the published Core 1.4 +minor series, pinned with `~=1.4.0`. Core 2.0.0 is prepared separately because +safe mask-reveal and automatic-overlap transformation semantics are incompatible +changes under Core's version policy. Both use capability contract 1. Before +shipping the combined fixes, publish the qualified Core candidate, then raise +both Python dependency declarations to `~=2.0.0`, update the reviewed native +contract/corpus, and run Python's installed-wheel and release qualification +against the published artifacts. Source-candidate integration is additional +evidence; it does not substitute for that post-publication gate. + +Scheduled Python jobs only qualify candidates. Manual `release.yml` runs default +to `dry_run: true`; explicitly selecting false is the publication boundary. +Candidate bases must be newer than every stable Git tag and the latest stable +PyPI release. Development qualification refuses a ref missing `main` history. +The workflow atomically pushes the tested version commit and immutable tag +before uploading to PyPI, then fast-forwards `dev` after a stable publication. +The user must authorize deployment before any publishing workflow is dispatched. + +If registry upload fails after the atomic ref push, keep the tag. Inspect PyPI +for partial artifacts, compare hashes to the retained workflow artifact and tag, +and resume only the missing upload from those exact validated artifacts. Do not +move the tag, reuse the version for different bytes, or rerun version generation +with an existing release tag. A changed branch must be qualified again. + +Core's individual registries have separate availability; never infer that a +Python version is present on npm or crates.io. Native wheel coverage and source +build requirements are documented in Core's platform policy. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..cb600497 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/scripts/check_rust_install.py b/scripts/check_rust_install.py index bf84c45d..57604d3f 100644 --- a/scripts/check_rust_install.py +++ b/scripts/check_rust_install.py @@ -3,6 +3,8 @@ import importlib.metadata import sys +from packaging.requirements import Requirement + def main(): assert sys.flags.isolated, "Run with python -I" @@ -14,7 +16,20 @@ def main(): from datafog.compat import v4 installed = importlib.metadata.version("datafog-core") - assert installed.split(".")[0] == "1", installed + requirements = [ + Requirement(value) for value in importlib.metadata.requires("datafog") or [] + ] + core_requirements = [ + requirement + for requirement in requirements + if requirement.name == "datafog-core" and requirement.marker is None + ] + assert ( + core_requirements + ), "Installed DataFog must declare its Core compatibility range" + assert all( + installed in requirement.specifier for requirement in core_requirements + ), installed assert datafog_core.capabilities()["contract_version"] == 1 assert v4.Entity is datafog.Entity assert v5.Finding is datafog_core.Finding diff --git a/scripts/release_version.py b/scripts/release_version.py new file mode 100644 index 00000000..0ce2c585 --- /dev/null +++ b/scripts/release_version.py @@ -0,0 +1,68 @@ +"""Select a release version while refusing stale or already published bases.""" + +import argparse +import json +import re +import subprocess +import urllib.request +from pathlib import Path + +from packaging.version import Version + + +def select_version(current, release_type, tags, latest_stable, override=None): + if release_type not in {"alpha", "beta", "stable"}: + raise ValueError("Unknown release type") + source = Version(current) + base = override or source.base_version + if not re.fullmatch(r"\d+\.\d+\.\d+", base): + raise ValueError("Release base must be numeric major.minor.patch") + if Version(base) <= Version(latest_stable): + raise ValueError("Release base must be newer than the latest stable release") + if release_type == "stable": + candidate = base + else: + suffix = "a" if release_type == "alpha" else "b" + number = 1 + while f"v{base}{suffix}{number}" in tags: + number += 1 + candidate = f"{base}{suffix}{number}" + if f"v{candidate}" in tags: + raise ValueError( + "Release tag already exists; inspect publication state before retrying" + ) + return candidate + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--type", required=True, choices=["alpha", "beta", "stable"]) + parser.add_argument("--override", default="") + parser.add_argument( + "--latest-stable", help="Offline preflight only; workflow reads PyPI" + ) + args = parser.parse_args() + tags = subprocess.check_output(["git", "tag", "-l"], text=True).splitlines() + stable_tags = [tag[1:] for tag in tags if re.fullmatch(r"v\d+\.\d+\.\d+", tag)] + if args.latest_stable: + published = args.latest_stable + else: + with urllib.request.urlopen( + "https://pypi.org/pypi/datafog/json", timeout=30 + ) as response: + payload = json.load(response) + releases = [ + Version(version) + for version in payload["releases"] + if not Version(version).is_prerelease + ] + published = str(max(releases)) + latest = str(max(Version(version) for version in [published, *stable_tags])) + about = Path("datafog/__about__.py") + current = re.search(r'^__version__ = "([^"]+)"', about.read_text(), re.M).group(1) + candidate = select_version(current, args.type, tags, latest, args.override or None) + print(candidate) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 7ce0366d..bffb7ff5 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,13 @@ # Core lightweight dependencies only core_deps = [ - # Native engine, the default detection path. Exact upper bound: detection - # output is contract-versioned and must stay reproducible. - "datafog-core>=1.4.0,<2", - "pydantic>=2.0,<3.0", + # Core minor releases may change findings; stay on this reviewed minor. + "datafog-core~=1.4.0", + "pydantic>=2.1,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", + # Match timeouts bound untrusted allowlist expressions. + "regex>=2024.11.6", ] # Optional heavy dependencies @@ -34,7 +35,7 @@ nlp_advanced_deps = [ "gliner>=0.2.5", - "torch>=2.1.0,<2.7", + "torch>=2.1.0,<3", "transformers>=4.20.0", "huggingface-hub>=0.16.0", "sentencepiece>=0.2.0", @@ -88,9 +89,9 @@ extras_require = { # Retained for explicitness; the base install already pulls Core. - "rust": ["datafog-core>=1.4.0,<2"], + "rust": ["datafog-core~=1.4.0"], "nlp": nlp_deps, - "nlp-advanced": nlp_advanced_deps, + "nlp-advanced": nlp_deps + nlp_advanced_deps, "ocr": ocr_deps, "distributed": distributed_deps, "web": web_deps, @@ -117,6 +118,8 @@ version=version, author="Sid Mohan", author_email="sid@datafog.ai", + license="MIT", + license_files=["LICENSE"], description="Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)", long_description=long_description, long_description_content_type="text/markdown", @@ -127,14 +130,13 @@ python_requires=">=3.10", entry_points={ "console_scripts": [ - "datafog=datafog.client:app [cli]", # Requires cli extra + "datafog=datafog.cli:main", "datafog-hook=datafog.integrations.claude_code:main", # Core only ], }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/setup_lean.py b/setup_lean.py deleted file mode 100644 index 57315788..00000000 --- a/setup_lean.py +++ /dev/null @@ -1,109 +0,0 @@ -from setuptools import find_packages, setup - -# Historical shadow packaging snapshot. The live DataFog packaging input is -# setup.py. Do not use this file for builds, releases, or dependency changes. - -# Read README for the long description -with open("README.md", "r") as f: - long_description = f.read() - -# Use a single source of truth for the version -__version__ = "4.1.1" - -project_urls = { - "Homepage": "https://datafog.ai", - "Documentation": "https://docs.datafog.ai", - "Discord": "https://discord.gg/bzDth394R4", - "Twitter": "https://twitter.com/datafoginc", - "GitHub": "https://github.com/datafog/datafog-python", -} - -# Core lightweight dependencies only -core_deps = [ - "pydantic>=2.0,<3.0", - "typing-extensions>=4.0", -] - -# Optional heavy dependencies -extras_require = { - "nlp": [ - "spacy>=3.7.0,<4.0", - ], - "ocr": [ - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - ], - "distributed": [ - "pandas>=2.0.0", - "numpy>=1.24.0", - ], - "web": [ - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - ], - "cli": [ - "typer>=0.12.0", - ], - "crypto": [ - "cryptography>=40.0.0", - ], - "dev": [ - "pytest>=7.0.0", - "pytest-asyncio>=0.21.0", - "pytest-cov>=4.0.0", - "sphinx>=7.0.0", - ], - # Convenience bundles - "all": [ - "spacy>=3.7.0,<4.0", - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - "pandas>=2.0.0", - "numpy>=1.24.0", - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - "typer>=0.12.0", - "cryptography>=40.0.0", - ], -} - -setup( - name="datafog", - version=__version__, - author="Sid Mohan", - author_email="sid@datafog.ai", - description="Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)", - long_description=long_description, - long_description_content_type="text/markdown", - packages=find_packages(), - install_requires=core_deps, - extras_require=extras_require, - python_requires=">=3.10", - entry_points={ - "console_scripts": [ - "datafog=datafog.client:app [cli]", # Requires cli extra - ], - }, - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Text Processing", - "Topic :: Security", - ], - project_urls=project_urls, - keywords="pii detection anonymization privacy regex performance", -) diff --git a/setup_original.py b/setup_original.py deleted file mode 100644 index 006a3608..00000000 --- a/setup_original.py +++ /dev/null @@ -1,92 +0,0 @@ -from setuptools import find_packages, setup - -# Historical shadow packaging snapshot. The live DataFog packaging input is -# setup.py. Do not use this file for builds, releases, or dependency changes. - -# Read README for the long description -with open("README.md", "r") as f: - long_description = f.read() - -# Use a single source of truth for the version -__version__ = "4.1.1" - -project_urls = { - "Homepage": "https://datafog.ai", - "Documentation": "https://docs.datafog.ai", - "Discord": "https://discord.gg/bzDth394R4", - "Twitter": "https://twitter.com/datafoginc", - "GitHub": "https://github.com/datafog/datafog-python", -} - -setup( - name="datafog", - version=__version__, - author="Sid Mohan", - author_email="sid@datafog.ai", - description="Scan, redact, and manage PII in your documents before they get uploaded to a Retrieval Augmented Generation (RAG) system.", - long_description=long_description, - long_description_content_type="text/markdown", - packages=find_packages(), - install_requires=[ - "pandas", - "requests==2.32.3", - "spacy==3.7.5", - "pydantic", - "Pillow", - "sentencepiece", - "protobuf", - "pytesseract", - "aiohttp", - "pytest-asyncio", - "numpy", - "fastapi", - "asyncio", - "setuptools", - "pydantic-settings==2.3.4", - "typer==0.12.3", - "sphinx", - "cryptography", - ], - python_requires=">=3.10", - entry_points={ - "console_scripts": [ - "datafog=datafog.client:app", - ], - }, - classifiers=[ - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Framework :: tox", - "Framework :: Pytest", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "Intended Audience :: Information Technology", - "Intended Audience :: System Administrators", - ], - keywords="pii, redaction, nlp, rag, retrieval augmented generation", - maintainer="DataFog", - maintainer_email="hi@datafog.ai", - url="https://datafog.ai", - project_urls=project_urls, - license="MIT", - extras_require={ - "dev": [ - "just", - "isort", - "black", - "blacken-docs", - "flake8", - "tox", - "pytest", - "pytest-codeblocks", - "pytest-cov", - "build", - "twine", - ], - }, -) diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json index e4c0835b..b3ddf293 100644 --- a/tests/contracts/wrapper-4.12.0.json +++ b/tests/contracts/wrapper-4.12.0.json @@ -1,9 +1,6 @@ { "wrapper_version": "4.12.0", - "core_versions": [ - "1.4.0", - "2.0.0" - ], + "core_versions": ["1.4.0", "2.0.0"], "source_oracle": "4.8.1.json (unchanged)", "review": "2026-11-15 PDF remediation: explicit reviewed wrapper changes, not a regenerated historical oracle", "cases": { @@ -542,9 +539,7 @@ "reason": "PY-H3/PY-L6 route TextService through shared native detection and return populated canonical labels.", "expected": { "value": { - "EMAIL": [ - "alice@example.com" - ] + "EMAIL": ["alice@example.com"] }, "warnings": [] } @@ -554,9 +549,7 @@ "reason": "PY-H3/PY-L6 route TextService through shared native detection with overlap resolution and populated canonical labels.", "expected": { "value": { - "DE_TAX_ID": [ - "12345678901" - ] + "DE_TAX_ID": ["12345678901"] }, "warnings": [] } @@ -567,9 +560,7 @@ "expected": { "value": [ { - "EMAIL": [ - "alice@example.com" - ] + "EMAIL": ["alice@example.com"] }, {} ], @@ -920,14 +911,7 @@ "default": null } ], - "fields": [ - "type", - "text", - "start", - "end", - "confidence", - "engine" - ] + "fields": ["type", "text", "start", "end", "confidence", "engine"] }, "datafog.engine:ScanResult": { "parameters": [ @@ -950,11 +934,7 @@ "default": null } ], - "fields": [ - "entities", - "text", - "engine_used" - ] + "fields": ["entities", "text", "engine_used"] }, "datafog.engine:RedactResult": { "parameters": [ @@ -977,11 +957,7 @@ "default": null } ], - "fields": [ - "redacted_text", - "mapping", - "entities" - ] + "fields": ["redacted_text", "mapping", "entities"] }, "datafog.engine:scan": { "parameters": [ diff --git a/tests/test_client.py b/tests/test_client.py index d5b042d1..24734be1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -165,7 +165,7 @@ def test_anonymizer_outputs(): ), AnnotationResult( start=33, - end=41, + end=len(text), score=1.0, entity_type=EntityTypes.LOCATION, recognition_metadata=AnnotatorMetadata(), diff --git a/tests/test_donut_lazy_import.py b/tests/test_donut_lazy_import.py index 9b2a28f1..aab7bcfc 100644 --- a/tests/test_donut_lazy_import.py +++ b/tests/test_donut_lazy_import.py @@ -20,8 +20,7 @@ def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]: def test_no_torch_import_when_donut_disabled(): """Test that torch is not imported when use_donut is False""" - _run_isolated_python( - """ + _run_isolated_python(""" import sys from datafog.services.image_service import ImageService @@ -29,8 +28,7 @@ def test_no_torch_import_when_donut_disabled(): assert "torch" not in sys.modules assert "transformers" not in sys.modules -""" - ) +""") def test_lazy_import_mechanism(): @@ -39,8 +37,7 @@ def test_lazy_import_mechanism(): # to use lazy imports. We don't need to actually test the imports themselves, # just that the structure is correct. - _run_isolated_python( - """ + _run_isolated_python(""" import sys from datafog.processing.image_processing.donut_processor import DonutProcessor @@ -50,5 +47,4 @@ def test_lazy_import_mechanism(): assert "transformers" not in sys.modules assert hasattr(processor, "extract_text_from_image") assert not hasattr(processor, "ensure_installed") -""" - ) +""") diff --git a/tests/test_image_service.py b/tests/test_image_service.py index da8e6deb..ada09646 100644 --- a/tests/test_image_service.py +++ b/tests/test_image_service.py @@ -3,9 +3,10 @@ import asyncio import pytest -from PIL import Image -from datafog.services.image_service import ImageService +Image = pytest.importorskip("PIL.Image", reason="Image tests require datafog[ocr]") + +from datafog.services.image_service import ImageService # noqa: E402 urls = [ "https://www.pdffiller.com/preview/101/35/101035394.png", diff --git a/tests/test_legacy_retirement.py b/tests/test_legacy_retirement.py index f8be98f0..ed5910a0 100644 --- a/tests/test_legacy_retirement.py +++ b/tests/test_legacy_retirement.py @@ -31,7 +31,7 @@ def unexpected_import(*args): LegacySurfaceWarning, match=( f"{surface} support is deprecated in 4.9 and will be removed in 5.0. " - 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0".' + 'Last supported version is 4.12.0. Pin it with: pip install "datafog==4.12.0".' ), ): (ImageService if surface == "OCR" else SparkService)() @@ -43,7 +43,7 @@ def unavailable(name): monkeypatch.setattr(importlib, "import_module", unavailable) with ( - pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"), + pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.12.0"), pytest.raises(ImportError, match=r"datafog\[distributed\]"), ): SparkService() @@ -85,7 +85,7 @@ def test_donut_mock_output_is_preserved(monkeypatch): monkeypatch.setattr(donut_processor, "DONUT_TESTING_ENABLED", False) with pytest.warns(LegacySurfaceWarning): processor = DonutProcessor() - with pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"): + with pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.12.0"): assert asyncio.run(processor.extract_text_from_image(object())) == ( '{"text": "Mock OCR text for testing"}' ) @@ -245,7 +245,7 @@ def test_image_cli_notice_visible_under_default_python_filters(image_argument): args.append("image.png") result = CliRunner().invoke(client.app, args) assert "deprecated in 4.9 and will be removed in 5.0" in result.stderr, result.output -assert 'Last supported version is 4.10.0' in result.stderr, result.output +assert 'Last supported version is 4.12.0' in result.stderr, result.output if sys.argv[1] == "True": assert result.exit_code == 0, result.output assert "OCR Pipeline Results: ['OCR text']" in result.stdout diff --git a/tests/test_litellm_guardrail.py b/tests/test_litellm_guardrail.py index 1d7656a4..c4c1d697 100644 --- a/tests/test_litellm_guardrail.py +++ b/tests/test_litellm_guardrail.py @@ -152,13 +152,14 @@ async def test_post_call_fail_open_returns_unredacted_response(self, monkeypatch @pytest.mark.asyncio class TestEdgeShapes: - async def test_data_without_messages_passes_through(self): + async def test_embedding_input_is_redacted(self): guardrail = DataFogGuardrail(guardrail_name="datafog-pii") data = {"input": f"embed {EMAIL}"} result = await guardrail.async_pre_call_hook( user_api_key_dict=None, cache=None, data=data, call_type="aembedding" ) - assert result == data + assert EMAIL not in result["input"] + assert result["input"].startswith("embed [EMAIL_") async def test_message_without_content_key_passes_through(self): guardrail = DataFogGuardrail(guardrail_name="datafog-pii") diff --git a/tests/test_no_network_core.py b/tests/test_no_network_core.py index f06e8360..cbdf8c08 100644 --- a/tests/test_no_network_core.py +++ b/tests/test_no_network_core.py @@ -20,8 +20,7 @@ def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]: def test_import_scan_and_redact_do_not_open_network() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import socket import urllib.request @@ -39,8 +38,7 @@ def blocked(*_args, **_kwargs): redact_result = datafog.redact("Email jane@example.com or call 415-555-1212") assert "jane@example.com" not in redact_result.redacted_text assert "415-555-1212" not in redact_result.redacted_text -""" - ) +""") def test_core_defaults_do_not_initialize_optional_engines(monkeypatch) -> None: @@ -78,8 +76,7 @@ def fail_optional_engine_probe(): def test_import_probes_do_not_load_optional_models() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import sys import types @@ -105,13 +102,11 @@ def from_pretrained(*_args, **_kwargs): import datafog assert datafog.scan("Email jane@example.com").entities -""" - ) +""") def test_core_path_does_not_import_optional_dependency_modules() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import importlib.abc import sys @@ -144,5 +139,4 @@ def find_spec(self, fullname, path=None, target=None): assert datafog.scan_prompt("Email jane@example.com").entities assert datafog.filter_output("Email jane@example.com").redacted_text == "Email [EMAIL_1]" assert datafog.create_guardrail().filter("Email jane@example.com").redacted_text == "Email [EMAIL_1]" -""" - ) +""") diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py new file mode 100644 index 00000000..7833772a --- /dev/null +++ b/tests/test_review_entrypoints.py @@ -0,0 +1,229 @@ +"""Regression contracts for the November 2026 entrypoint review.""" + +import asyncio +import hashlib +import importlib.util +import logging +from pathlib import Path +from types import SimpleNamespace + +import pytest +from typer.testing import CliRunner + +import datafog +from datafog.agent import create_guardrail +from datafog.client import app +from datafog.config import OperationType, get_config +from datafog.main import DataFog, TextPIIAnnotator +from datafog.models.annotator import AnnotationResult, AnnotatorRequest +from datafog.models.anonymizer import Anonymizer, AnonymizerType, HashType +from datafog.models.common import EntityTypes + +EMAIL = "jane@" "example.com" + + +@pytest.mark.asyncio +async def test_decorator_awaits_and_filters(): + @create_guardrail() + async def answer(): + await asyncio.sleep(0) + return EMAIL + + assert EMAIL not in await answer() + + +@pytest.mark.parametrize("output", [[EMAIL], {"text": EMAIL}, None, 1]) +def test_guarded_non_strings_rejected(output): + @create_guardrail() + def answer(): + return output + + with pytest.raises(TypeError, match="must return a string"): + answer() + + +@pytest.mark.parametrize( + "operation", [OperationType.REDACT, OperationType.REPLACE, OperationType.HASH] +) +def test_anonymizing_operations_imply_scan(operation): + service = DataFog(operations=[operation]) + assert EMAIL not in service.run_text_pipeline_sync([EMAIL])[0] + assert EMAIL not in service.process_text(EMAIL) + + +@pytest.mark.parametrize("algorithm", list(HashType)) +def test_hash_algorithm_honored_across_api_and_cli(algorithm): + digest = hashlib.new(algorithm.value, EMAIL.encode()).hexdigest() + assert ( + digest + in DataFog(hash_type=algorithm).process(EMAIL, anonymize=True, method="hash")[ + "anonymized" + ] + ) + assert ( + digest + in datafog.process( + EMAIL, anonymize=True, method="hash", hash_type=algorithm.value + )["anonymized"] + ) + result = CliRunner().invoke( + app, ["hash-text", EMAIL, "--hash-type", algorithm.value] + ) + assert result.exit_code == 0, result.output + assert digest in result.stdout + + +def test_keyed_hash_public_facades(): + import hmac + + key = b"k" * 32 + digest = hmac.new(key, EMAIL.encode(), "sha256").hexdigest() + assert digest in datafog.redact(EMAIL, strategy="hash", hash_key=key).redacted_text + assert digest in datafog.anonymize_text(EMAIL, method="hash", hash_key=key) + assert ( + digest + in datafog.process(EMAIL, anonymize=True, method="hash", hash_key=key)[ + "anonymized" + ] + ) + + +@pytest.mark.parametrize( + "factory", [lambda: datafog.process, lambda: DataFog().process] +) +def test_bad_method_rejected(factory): + with pytest.raises(ValueError, match="method"): + factory()(EMAIL, anonymize=True, method="bogus") + + +def test_legacy_detection_uses_core(): + text = "card 6011111111111117 ip 2001:db8::1" + assert {item["type"] for item in datafog.detect(text)} >= { + "CREDIT_CARD", + "IP_ADDRESS", + } + assert "6011111111111117" not in datafog.process(text, anonymize=True)["anonymized"] + assert "IP_ADDRESS" in TextPIIAnnotator().run(text) + assert "JWT" in datafog.get_supported_entities() + + +def test_annotation_defaults_and_labels(): + assert AnnotatorRequest(text="x", language="en").score_threshold is None + for label in ["SSN", "ORG", "GPE", "IP_ADDRESS"]: + assert ( + AnnotationResult(start=0, end=1, entity_type=label).entity_type != "UNKNOWN" + ) + + +def test_enum_filter_and_date_replacement(): + annotation = AnnotationResult( + start=0, end=len(EMAIL), entity_type=EntityTypes.EMAIL + ) + assert ( + Anonymizer(anonymizer_type="redact", entities=[EntityTypes.EMAIL]) + .anonymize(EMAIL, [annotation]) + .anonymized_text + == "[REDACTED]" + ) + date = AnnotationResult(start=0, end=10, entity_type="DATE") + assert ( + Anonymizer().anonymize("2024-01-02", [date]).anonymized_text + == "[REDACTED_DATE]" + ) + + +@pytest.mark.parametrize("strategy", list(AnonymizerType)) +def test_legacy_overlaps_never_leave_partial_value(strategy): + text = "abcdefghij" + spans = [ + AnnotationResult(start=0, end=6, entity_type="EMAIL"), + AnnotationResult(start=4, end=10, entity_type="SSN"), + ] + result = Anonymizer(anonymizer_type=strategy).anonymize(text, spans) + assert "ghij" not in result.anonymized_text + assert len(result.anonymized_entities) == 1 + assert result.anonymized_entities[0]["original"] == text + + +def test_cli_hides_key_and_handles_bad_operation(): + config = get_config() + old_key = config.api_key + try: + config.api_key = "private-test-key" + result = CliRunner().invoke(app, ["show-config"]) + assert result.exit_code == 0 + assert "private-test-key" not in result.stdout + assert "REDACTED" in result.stdout + finally: + config.api_key = old_key + result = CliRunner().invoke(app, ["scan-text", EMAIL, "--operations", "bogus"]) + assert result.exit_code == 2 + assert "operations must use" in result.output + assert "Traceback" not in result.output + + +def test_import_and_cli_preserve_logger_level(): + logger = logging.getLogger("datafog_logger") + old = logger.level + try: + logger.setLevel(logging.ERROR) + import importlib + + import datafog.main + + importlib.reload(datafog.main) + assert logger.level == logging.ERROR + finally: + logger.setLevel(old) + + +def test_spark_real_spacy_labels_and_oversize(monkeypatch): + from datafog.processing.spark_processing import pyspark_udfs as spark + + monkeypatch.setattr(spark, "ensure_installed", lambda _: None) + + def nlp(text): + return SimpleNamespace( + ents=[ + SimpleNamespace(label_="PERSON", text="Ann"), + SimpleNamespace(label_="ORG", text="Acme"), + ] + ) + + with pytest.warns(FutureWarning): + result = spark.pii_annotator("Ann at Acme", SimpleNamespace(value=nlp)) + assert result[spark.PII_ANNOTATION_LABELS.index("PER")] == ["Ann"] + assert result[spark.PII_ANNOTATION_LABELS.index("ORG")] == ["Acme"] + with pytest.warns(FutureWarning), pytest.raises(ValueError, match="maximum"): + spark.pii_annotator( + "x" * (spark.MAXIMAL_STRING_SIZE + 1), SimpleNamespace(value=nlp) + ) + + +def _version_selector(): + path = Path(__file__).parents[1] / "scripts/release_version.py" + spec = importlib.util.spec_from_file_location("release_version", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.select_version + + +@pytest.mark.parametrize( + "current, kind, override", + [ + ("4.9.0a2", "alpha", None), + ("4.11.0", "beta", None), + ("4.12.0", "stable", "4.10.0"), + ], +) +def test_release_rejects_stale_bases(current, kind, override): + with pytest.raises(ValueError, match="newer than"): + _version_selector()(current, kind, [], "4.11.0", override) + + +def test_release_version_positive_and_duplicate_paths(): + select = _version_selector() + assert select("4.12.0", "alpha", ["v4.12.0a1"], "4.11.0") == "4.12.0a2" + assert select("4.12.0", "stable", [], "4.11.0") == "4.12.0" + with pytest.raises(ValueError, match="already exists"): + select("4.12.0", "stable", ["v4.12.0"], "4.11.0") diff --git a/tests/test_runtime_dependency_safety.py b/tests/test_runtime_dependency_safety.py index d34ceb8a..b311fff4 100644 --- a/tests/test_runtime_dependency_safety.py +++ b/tests/test_runtime_dependency_safety.py @@ -42,8 +42,7 @@ def test_runtime_code_does_not_install_packages() -> None: def test_ocr_and_spark_public_services_do_not_require_optional_imports() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import importlib.abc import sys @@ -74,8 +73,7 @@ def find_spec(self, fullname, path=None, target=None): assert TextService is not None assert datafog.ImageService is ImageService assert datafog.SparkService is SparkService -""" - ) +""") def test_spacy_pii_missing_model_requires_explicit_download( diff --git a/tox.ini b/tox.ini index 83d3f793..a7a1565c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py310,py311,py312 +envlist = py310,py311,py312,py313,py314 isolated_build = True [testenv:integration] From dd01fa9ee9068ce44ad0ba80997466f16aad08b5 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:03:11 -0800 Subject: [PATCH 13/23] docs: align contributor release guidance with safe qualification --- AGENTS.md | 14 +++++++------- VERSIONING.md | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 13439692..871e8012 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,9 +13,9 @@ ## Current Project Status -**Stable version: 4.9.0** +**Stable version: 4.11.0** -**Development version: 4.9.0** +**Development candidate: 4.12.0** **Next major target: 5.0.0** @@ -242,14 +242,14 @@ except ImportError: | Workflow | Purpose | Trigger | | --------------- | ----------------------------------- | -------------------------- | | `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | -| `release.yml` | Alpha/Beta/Stable publishing | Schedule + manual dispatch | +| `release.yml` | Candidate qualification and publishing | Schedule validates; manual dispatch publishes | | `benchmark.yml` | Performance benchmarks | Push/PR/weekly | ### Release Cadence -- **Alpha** (Mon-Wed 2AM UTC): Automatic from `dev`, date+commit versioning -- **Beta** (Thursday 2AM UTC): Automatic from `dev`, incremental beta numbers -- **Stable** (manual dispatch): From `main`, base version or override +- **Alpha** (Mon-Wed 2AM UTC): Candidate qualification from `dev`, without publication +- **Beta** (Thursday 2AM UTC): Candidate qualification from `dev`, without publication +- **Publication** (manual dispatch): Dry run by default; explicit non-dry runs publish alpha/beta from `dev` or stable from `main` ### Release Pipeline @@ -320,7 +320,7 @@ Before beginning any task please checkout a branch from `dev` and create a pull ### Release Preparation -- Alpha/beta releases are automated via `release.yml` schedule +- Alpha/beta candidate qualification is automated via `release.yml`; publication requires an explicit manual dispatch - Stable releases: merge `dev` → `main`, then trigger `release.yml` with `stable` type - Use `dry_run: true` to validate before actual publish - Performance validation on realistic data sets diff --git a/VERSIONING.md b/VERSIONING.md index 9f511c41..802af079 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -96,9 +96,9 @@ for `datafog-core`, and requires a matching floor in `datafog`. Both are exposed separately so a bug report is attributable: ```python -import datafog, datafog.core +import datafog, datafog.native datafog.__version__ # application version -datafog.core.__version__ # engine version +datafog.native.version() # engine version ``` ## Yanking From bd42280c063c197d12d00af755d0cd7a8bcbd27a Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:04:39 -0800 Subject: [PATCH 14/23] fix(guardrails): enforce matching stream group counts --- datafog/integrations/litellm_guardrail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafog/integrations/litellm_guardrail.py b/datafog/integrations/litellm_guardrail.py index 5611bec9..74a56169 100644 --- a/datafog/integrations/litellm_guardrail.py +++ b/datafog/integrations/litellm_guardrail.py @@ -404,7 +404,7 @@ async def async_post_call_streaming_iterator_hook( ] session = _RedactionSession(texts) changes = [] - for group, text in zip(groups.values(), texts): + for group, text in zip(groups.values(), texts, strict=True): redacted, _ = self._process_content(text, session) changes.extend( (target, field, redacted if i == 0 else "") From 2cd5127c8ec96c2bdac6d35d80187dce727545b0 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:05:09 -0800 Subject: [PATCH 15/23] fix(release): account for untagged publications and preserve release target --- .github/workflows/release.yml | 1 + scripts/release_version.py | 15 +++++++++++---- tests/test_review_entrypoints.py | 5 +++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32952a23..0e2b8c63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -382,6 +382,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_PAT }} VERSION: ${{ steps.version.outputs.version }} TYPE: ${{ needs.determine-release.outputs.release_type }} + BRANCH: ${{ needs.determine-release.outputs.target_branch }} run: | PRERELEASE_FLAG="" TITLE="" diff --git a/scripts/release_version.py b/scripts/release_version.py index 0ce2c585..f78afd28 100644 --- a/scripts/release_version.py +++ b/scripts/release_version.py @@ -10,10 +10,13 @@ from packaging.version import Version -def select_version(current, release_type, tags, latest_stable, override=None): +def select_version( + current, release_type, tags, latest_stable, override=None, published=() +): if release_type not in {"alpha", "beta", "stable"}: raise ValueError("Unknown release type") source = Version(current) + occupied = set(tags) | {f"v{Version(version)}" for version in published} base = override or source.base_version if not re.fullmatch(r"\d+\.\d+\.\d+", base): raise ValueError("Release base must be numeric major.minor.patch") @@ -24,10 +27,10 @@ def select_version(current, release_type, tags, latest_stable, override=None): else: suffix = "a" if release_type == "alpha" else "b" number = 1 - while f"v{base}{suffix}{number}" in tags: + while f"v{base}{suffix}{number}" in occupied: number += 1 candidate = f"{base}{suffix}{number}" - if f"v{candidate}" in tags: + if f"v{candidate}" in occupied: raise ValueError( "Release tag already exists; inspect publication state before retrying" ) @@ -46,6 +49,7 @@ def main(): stable_tags = [tag[1:] for tag in tags if re.fullmatch(r"v\d+\.\d+\.\d+", tag)] if args.latest_stable: published = args.latest_stable + published_versions = [published] else: with urllib.request.urlopen( "https://pypi.org/pypi/datafog/json", timeout=30 @@ -57,10 +61,13 @@ def main(): if not Version(version).is_prerelease ] published = str(max(releases)) + published_versions = list(payload["releases"]) latest = str(max(Version(version) for version in [published, *stable_tags])) about = Path("datafog/__about__.py") current = re.search(r'^__version__ = "([^"]+)"', about.read_text(), re.M).group(1) - candidate = select_version(current, args.type, tags, latest, args.override or None) + candidate = select_version( + current, args.type, tags, latest, args.override or None, published_versions + ) print(candidate) diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py index 7833772a..d05c40ab 100644 --- a/tests/test_review_entrypoints.py +++ b/tests/test_review_entrypoints.py @@ -227,3 +227,8 @@ def test_release_version_positive_and_duplicate_paths(): assert select("4.12.0", "stable", [], "4.11.0") == "4.12.0" with pytest.raises(ValueError, match="already exists"): select("4.12.0", "stable", ["v4.12.0"], "4.11.0") + + +def test_release_skips_published_prereleases_without_tags(): + select = _version_selector() + assert select("4.12.0", "alpha", [], "4.11.0", published=["4.12.0a1"]) == "4.12.0a2" From c6824095ddd91820cb64709d6ed047abded662d0 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:05:34 -0800 Subject: [PATCH 16/23] style: format updated release guidance --- AGENTS.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 871e8012..921d2d55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -239,11 +239,11 @@ except ImportError: ### Workflow Architecture (3 workflows) -| Workflow | Purpose | Trigger | -| --------------- | ----------------------------------- | -------------------------- | -| `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | +| Workflow | Purpose | Trigger | +| --------------- | -------------------------------------- | --------------------------------------------- | +| `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | | `release.yml` | Candidate qualification and publishing | Schedule validates; manual dispatch publishes | -| `benchmark.yml` | Performance benchmarks | Push/PR/weekly | +| `benchmark.yml` | Performance benchmarks | Push/PR/weekly | ### Release Cadence From 0eaa6e7d3e9f682d0263c6f6bae69171ec7b9f55 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:09:42 -0800 Subject: [PATCH 17/23] fix(ci): provision benchmark models and validate current annotation contracts --- .github/workflows/benchmark.yml | 9 +-- tests/benchmark_text_service.py | 4 +- tests/simple_performance_test.py | 2 +- tests/test_gliner_annotator.py | 114 +++++++++++++++---------------- tests/test_image_service.py | 4 +- tests/test_ocr_integration.py | 3 +- 6 files changed, 64 insertions(+), 72 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index bcc51e1b..944cd756 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,6 +27,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[nlp]" + python -m spacy download en_core_web_lg pip install -r requirements-benchmark.txt # Verify pytest-benchmark is installed and working python -c "import pytest_benchmark; print('pytest-benchmark version:', pytest_benchmark.__version__)" @@ -54,13 +55,7 @@ jobs: # Run benchmarks with optimal performance settings (no memory debugging) echo "Running benchmarks with performance-optimized settings..." - # Try pytest-benchmark first - if python -m pytest tests/benchmark_text_service.py -v --benchmark-autosave --benchmark-json=benchmark-results.json --tb=short -p no:cacheprovider; then - echo "✅ pytest-benchmark tests completed successfully" - else - echo "⚠️ pytest-benchmark failed, running simple performance test as fallback" - python tests/simple_performance_test.py - fi + python -m pytest tests/benchmark_text_service.py -v --benchmark-autosave --benchmark-json=benchmark-results.json --tb=short -p no:cacheprovider - name: Check for performance regression run: | diff --git a/tests/benchmark_text_service.py b/tests/benchmark_text_service.py index 3178c61e..e802c40e 100644 --- a/tests/benchmark_text_service.py +++ b/tests/benchmark_text_service.py @@ -107,7 +107,7 @@ def test_spacy_performance(benchmark, sample_text_10kb, spacy_service): # Verify spaCy found expected entities assert "PERSON" in result or "PER" in result - assert "ORG" in result + assert "ORGANIZATION" in result # Print some stats about the results entity_counts = {key: len(values) for key, values in result.items() if values} @@ -178,7 +178,7 @@ def test_auto_engine_fallback_performance(benchmark, spacy_only_text, auto_servi ) # Check if we have spaCy entities (depends on spaCy availability) - spacy_entities = ["PERSON", "ORG", "GPE", "CARDINAL", "DATE", "TIME"] + spacy_entities = ["PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"] has_spacy_entities = any(entity in result for entity in spacy_entities) # If no spaCy entities, check if spaCy is available diff --git a/tests/simple_performance_test.py b/tests/simple_performance_test.py index 6da82d88..589ef59b 100644 --- a/tests/simple_performance_test.py +++ b/tests/simple_performance_test.py @@ -90,7 +90,7 @@ def test_simple_spacy_performance(): # Verify functionality assert "PERSON" in stats["result"] or "PER" in stats["result"] - assert "ORG" in stats["result"] + assert "ORGANIZATION" in stats["result"] return stats diff --git a/tests/test_gliner_annotator.py b/tests/test_gliner_annotator.py index fbf0bbdf..b9a82ad5 100644 --- a/tests/test_gliner_annotator.py +++ b/tests/test_gliner_annotator.py @@ -11,8 +11,8 @@ import pytest -@pytest.fixture(scope="session", autouse=True) -def mock_gliner_module(): +@pytest.fixture(autouse=True) +def mock_gliner_module(monkeypatch): """Mock the gliner module for all tests.""" # Create a mock gliner module mock_gliner = MagicMock() @@ -20,22 +20,42 @@ def mock_gliner_module(): # Configure the mock model behavior mock_model = MagicMock() - mock_model.predict_entities.return_value = [ - {"label": "person", "text": "John Doe", "start": 0, "end": 8}, - {"label": "email", "text": "john@example.com", "start": 20, "end": 36}, - {"label": "phone number", "text": "555-123-4567", "start": 50, "end": 62}, - ] + + def predict(text, labels): + findings = [] + for label, value in [ + ("person", "John Doe"), + ("email", "john@example.com"), + ("phone number", "555-123-4567"), + ]: + start = text.find(value) + if start >= 0 and label in labels: + findings.append( + { + "label": label, + "text": value, + "start": start, + "end": start + len(value), + } + ) + return findings + + mock_model.predict_entities.side_effect = predict mock_gliner_class.from_pretrained.return_value = mock_model mock_gliner.GLiNER = mock_gliner_class - # Add to sys.modules to make import work - sys.modules["gliner"] = mock_gliner + from datafog import engine + gliner_factory, spacy_factory = ( + engine._get_gliner_annotator, + engine._get_spacy_annotator, + ) + gliner_factory.cache_clear() + spacy_factory.cache_clear() + monkeypatch.setitem(sys.modules, "gliner", mock_gliner) yield mock_gliner_class, mock_model - - # Cleanup - if "gliner" in sys.modules: - del sys.modules["gliner"] + gliner_factory.cache_clear() + spacy_factory.cache_clear() class TestGLiNERAnnotatorWithDependencies: @@ -119,15 +139,9 @@ def test_gliner_annotate_long_text(self, mock_gliner_module): # Create text longer than MAXIMAL_STRING_SIZE long_text = "A" * 1000001 # Exceeds 1M character limit - with patch( - "datafog.processing.text_processing.gliner_annotator.logging" - ) as mock_logging: - result = annotator.annotate(long_text) - - # Should log a warning about truncation - mock_logging.warning.assert_called_once() - # Should still return a valid result - assert isinstance(result, dict) + with pytest.raises(ValueError, match="exceeds 1000000"): + annotator.annotate(long_text) + mock_model.predict_entities.assert_not_called() def test_gliner_download_model(self, mock_gliner_module): """Test GLiNER model download functionality.""" @@ -326,21 +340,19 @@ def test_text_service_gliner_engine_without_dependencies(self): ): TextService(engine="gliner") - def test_text_service_smart_engine_without_dependencies(self): - """Test smart engine degrades gracefully when GLiNER dependencies are missing.""" + def test_text_service_smart_engine_without_dependencies(self, monkeypatch): + """Missing optional engines preserve the structured native layer.""" + from datafog import engine from datafog.services.text_service import TextService - with patch.object(TextService, "_create_gliner_annotator", return_value=None): - with patch.object( - TextService, "_create_spacy_annotator", return_value=None - ): - service = TextService(engine="smart") - with pytest.warns(UserWarning, match="GLiNER not available"): - result = service.annotate_text_sync( - "John Doe from Acme Corporation needs follow up." - ) - assert "EMAIL" in result - assert result["EMAIL"] == [] + unavailable = engine._UnavailableAnnotator("optional model unavailable") + monkeypatch.setattr(engine, "_get_gliner_annotator", lambda: unavailable) + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: unavailable) + with pytest.warns(UserWarning, match="GLiNER unavailable"): + result = TextService(engine="smart").annotate_text_sync( + "mail john@example.com" + ) + assert result == {"EMAIL": ["john@example.com"]} def test_text_service_valid_engines(self): """Test that all valid engines are accepted.""" @@ -382,10 +394,10 @@ def test_text_service_valid_engines(self): assert service.engine == engine def test_text_service_invalid_engine(self): - """Test that invalid engines raise AssertionError.""" + """Test that invalid engines raise a configuration error.""" from datafog.services.text_service import TextService - with pytest.raises(AssertionError, match="Invalid engine"): + with pytest.raises(ValueError, match="engine must be one of"): TextService(engine="invalid_engine") @pytest.mark.parametrize( @@ -414,30 +426,14 @@ def test_cascade_should_stop_logic(self, engine, expected_count): result_above_threshold = {"TYPE1": ["entity1"] * (expected_count + 1)} assert service._cascade_should_stop(engine, result_above_threshold) - def test_smart_cascade_flow(self, mock_gliner_annotator): - """Test the smart cascading flow.""" + def test_smart_engine_combines_gliner_with_structured_detection(self): + """GLiNER adds exact NER spans without removing structured findings.""" from datafog.services.text_service import TextService - # Inject annotators directly to keep this cascade test deterministic - # across Python versions and import ordering. - mock_regex = Mock() - mock_regex.annotate.return_value = {"EMAIL": []} - - mock_spacy = Mock() - mock_spacy.annotate.return_value = {"PERSON": ["John Doe"]} - - service = TextService(engine="smart") - service._regex_annotator = mock_regex - service._gliner_annotator = mock_gliner_annotator - service._gliner_import_attempted = True - service._spacy_annotator = mock_spacy - service._spacy_import_attempted = True - - service.annotate_text_sync("John Doe works at john@example.com") - - # Should have tried regex first, then GLiNER. - mock_regex.annotate.assert_called_once() - mock_gliner_annotator.annotate.assert_called_once() + text = "John Doe owns 6011111111111117 and john@example.com" + result = TextService(engine="smart").annotate_text_sync(text, structured=True) + assert {item.label for item in result} == {"PERSON", "CREDIT_CARD", "EMAIL"} + assert all(text[item.start : item.end] == item.text for item in result) # Test CLI updates as well diff --git a/tests/test_image_service.py b/tests/test_image_service.py index ada09646..03d472b1 100644 --- a/tests/test_image_service.py +++ b/tests/test_image_service.py @@ -4,9 +4,9 @@ import pytest -Image = pytest.importorskip("PIL.Image", reason="Image tests require datafog[ocr]") +from datafog.services.image_service import ImageService -from datafog.services.image_service import ImageService # noqa: E402 +Image = pytest.importorskip("PIL.Image", reason="Image tests require datafog[ocr]") urls = [ "https://www.pdffiller.com/preview/101/35/101035394.png", diff --git a/tests/test_ocr_integration.py b/tests/test_ocr_integration.py index 9ceef4a2..c7333dbd 100644 --- a/tests/test_ocr_integration.py +++ b/tests/test_ocr_integration.py @@ -9,11 +9,12 @@ from unittest.mock import patch import pytest -from PIL import Image from datafog.processing.image_processing.donut_processor import DonutProcessor from datafog.services.image_service import ImageService +Image = pytest.importorskip("PIL.Image", reason="OCR tests require datafog[ocr]") + # Mark all tests in this file as integration tests pytestmark = pytest.mark.integration From ad36d6a9aa7857bd282b1c4fade793d1b1e8db98 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:11:14 -0800 Subject: [PATCH 18/23] docs: record every review finding and its regression proof --- docs/review-remediation.md | 44 ++++++++++++++++++++++++++++++++ tests/test_review_entrypoints.py | 15 +++++++++++ 2 files changed, 59 insertions(+) diff --git a/docs/review-remediation.md b/docs/review-remediation.md index 2bcb8ddd..86e37cc4 100644 --- a/docs/review-remediation.md +++ b/docs/review-remediation.md @@ -71,3 +71,47 @@ with an existing release tag. A changed branch must be qualified again. Core's individual registries have separate availability; never infer that a Python version is present on npm or crates.io. Native wheel coverage and source build requirements are documented in Core's platform policy. + +## Python finding disposition + +All 35 Python findings have a concrete resolution below. Core-owned fixes are +separately tracked in Core's `docs/reviews/2026-11-core-findings.md`; their combined +release still requires the Core publication and Python dependency update above. + +| Finding | Resolution and proof | +| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| PY-H1 | Replace lone surrogates only for native transport, preserving source coordinates. Engine and hook/LiteLLM block regressions cover mixed payloads. | +| PY-H2 | Interval queries replace pairwise overlap checks; redaction and legacy assembly are linear after sorting. Dense scan/redaction benchmarks and overlap tests cover the change. | +| PY-H3 | TextService scans complete documents through the shared engine. Boundary-crossing email/SSN tests pass. | +| PY-H4 | spaCy/GLiNER return actual character spans. Stub and real-spaCy tests distinguish Ann from Annual and repeated names. | +| PY-H5 | Every engine and legacy/CLI path uses the same structured layer before optional NER. Discover/IPv6/credential tests cover the paths. | +| PY-M1 | Coroutine decorators await results; non-string results raise a documented TypeError rather than silently escaping inspection. | +| PY-M2 | Redact, replace and hash pipeline operations imply scanning. All three are tested without an explicit SCAN operation. | +| PY-M3 | Fallback email consumes each lexical run once; German separator expressions avoid ambiguous repetition. Adversarial 80K-character measurements and positive matches pass. | +| PY-M4 | Native backend selection also supplies the structured stage of smart/spaCy/GLiNER. CLI commands work with DATAFOG_BACKEND=rust. | +| PY-M5 | Request-scoped tokens, nested tool/function arguments, prompt/input fields and buffered streaming protect all tested request/response shapes. Live proxy deployment is not claimed. | +| PY-M6 | Bare strings, invalid labels and labels unavailable to the selected backend/engine raise explicit errors. Supported aliases remain accepted. | +| PY-M7 | APIs and CLI honor hash_type, use full digests and support keyed HMAC. Plain-hash guessing limitations are documented; algorithm/key tests pass. | +| PY-M8 | Hook per-string/total budget exhaustion emits review/deny or an incomplete-scan notice. Tests put PII past the cap. | +| PY-M9 | Allowlist expressions run with match timeouts; timed-out expressions retain findings. Adversarial patterns and safe quantified groups are tested. | +| PY-M10 | Dependency resolution stays within the explicitly reviewed Core minor, replacing the broad less-than-2 range. Exact output locks should still pin patch versions. | +| PY-M11 | Reconcile stable history, reject stale/PyPI-occupied versions, gate 3.13, sync version metadata and push immutable refs before upload. Scheduled runs are nonpublishing; selector tests and actionlint pass. | +| PY-M12 | Spark maps ordinary PERSON/ORG/GPE/NORP/DATE labels into its fixed output schema. Stub tests cover real spaCy label spellings and oversized input errors. | +| PY-M13 | Optional Pydantic fields have defaults, labels use canonical names, enum filters work and overlaps merge before replacement. Model/anonymizer regressions pass. | +| PY-M14 | Preserve supported GLiNER labels, use exact offsets, exclude relative spaCy DATE mentions, propagate model failures and reject oversized annotator input explicitly. | +| PY-M15 | Embedded model calls use bounded overlapping chunks. Large PERSON/ADDRESS plus structured-PII tests succeed with published and candidate Core. Native model limits remain enforced. | +| PY-M16 | Core 2 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | +| PY-L1 | Console bootstrap prints the CLI-extra install hint on a base install. A clean installed-wheel check confirms no traceback. | +| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 2 suppresses the labelled-tax-ID PHONE false positive. | +| PY-L3 | Retirement notices derive the current supported release version, and the native install hint no longer names an older release. | +| PY-L4 | Supported-entity reporting follows the selected native/Python capability inventory, including opt-in labels. Tests separate supported from default-active labels. | +| PY-L5 | Document actual token/pseudonym outputs and reject invalid methods in both process APIs. | +| PY-L6 | Legacy APIs and TextPIIAnnotator route through the shared engine; service/engine integration covers native-only entities and locale behavior. Explicit Python fallback remains a narrower documented detector set. | +| PY-L7 | Tokens reuse values and avoid existing markers; invalid explicit ranges/text raise errors. Collision, repeated-value and validation tests pass. | +| PY-L8 | Remove unused implementation copies, declare an isolated build, include test packages/corpora/scripts in sdists, align formatter pins and extend tox through 3.14. Optional OCR imports skip cleanly when absent. | +| PY-L9 | CLI configuration hides API keys; import and commands preserve host logging levels. CLI/logger regressions pass. | +| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 2 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | +| PY-L11 | TextService validates positive chunk lengths and engine names with explicit exceptions, including optimized Python execution. | +| PY-L12 | Remove incompatible Torch upper bound and include spaCy in nlp-advanced. Resolving it with Core GLiNER2 succeeds with Torch 2.12.0. | +| PY-L13 | Declare Pydantic 2.1 minimum. A clean Python 3.10 base install with Pydantic 2.1 imports and scans successfully. | +| PY-L14 | Correct native-default documentation, normalize missing native metadata to ImportError, clarify missing-model hints, validate locales consistently and surface invalid CLI operations as usage errors. | diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py index d05c40ab..fb7cc555 100644 --- a/tests/test_review_entrypoints.py +++ b/tests/test_review_entrypoints.py @@ -232,3 +232,18 @@ def test_release_version_positive_and_duplicate_paths(): def test_release_skips_published_prereleases_without_tags(): select = _version_selector() assert select("4.12.0", "alpha", [], "4.11.0", published=["4.12.0a1"]) == "4.12.0a2" + + +def test_native_version_uses_import_error_when_metadata_is_missing(monkeypatch): + import importlib.metadata + + import datafog.native + + assert datafog.native.version() + + def missing(_): + raise importlib.metadata.PackageNotFoundError("datafog-core") + + monkeypatch.setattr(importlib.metadata, "version", missing) + with pytest.raises(ImportError, match="metadata is missing"): + datafog.native.version() From fb262c8dcc1888ab0141bbea91a446c7508dafe3 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:19:36 -0800 Subject: [PATCH 19/23] fix(hash): preserve existing pseudonyms while honoring algorithms and HMAC --- CHANGELOG.MD | 7 +- datafog/__init__.py | 41 +++---- datafog/engine.py | 31 +++++- docs/review-remediation.md | 12 +- tests/contracts/wrapper-4.12.0.json | 163 +--------------------------- tests/test_review_engine.py | 17 ++- tests/test_review_entrypoints.py | 21 +++- 7 files changed, 96 insertions(+), 196 deletions(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 6bf46b28..ab2e41b6 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -16,9 +16,10 @@ #### Changed -- Hash APIs honor the selected algorithm and emit full digests. Optional - `hash_key` enables HMAC with a key of at least 32 bytes. Plain hashes provide - deterministic pseudonyms and do not resist guessing of low-entropy PII. +- Hash APIs honor the selected algorithm while preserving existing plain-hash + marker widths and defaults. Optional `hash_key` enables full-length HMAC with + a key of at least 32 bytes. Plain hashes provide deterministic pseudonyms and + do not resist guessing of low-entropy PII. - Reuse tokens for repeated values, avoid existing token collisions, reject invalid explicit ranges and unsupported entity selections, and report backend entity capabilities. Async decorators await results before filtering. diff --git a/datafog/__init__.py b/datafog/__init__.py index f9a1695e..467f5a70 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -290,7 +290,7 @@ def process( anonymize: bool = False, method: str = "redact", *, - hash_type: str = "sha256", + hash_type: str | None = None, hash_key: bytes | None = None, ) -> dict: """ @@ -300,6 +300,9 @@ def process( text: Input text to process anonymize: Whether to anonymize detected PII method: Anonymization method ('redact', 'replace', 'hash') + hash_type: Optional algorithm. The legacy plain-hash default is MD5; + keyed hashes default to SHA256. + hash_key: At least 32 bytes for a full keyed HMAC digest. Returns: Dictionary with original text, anonymized text (if requested), and findings @@ -321,32 +324,32 @@ def process( if method not in {"redact", "replace", "hash"}: raise ValueError("method must be 'redact', 'replace', or 'hash'") + if anonymize and method == "hash": + from .engine import _hash_value, _validate_hash_parameters + + algorithm = ( + hash_type + if hash_type is not None + else ("sha256" if hash_key is not None else "md5") + ) + _validate_hash_parameters("hash", algorithm, hash_key) findings = _detect_impl(text) result = {"original": text, "findings": findings} - if anonymize and method == "hash": - entities = [ - Entity( - item["type"], item["value"], item["start"], item["end"], 1.0, "regex" - ) - for item in findings - ] - result["anonymized"] = redact( - text, - entities=entities, - strategy="hash", - hash_type=hash_type, - hash_key=hash_key, - ).redacted_text - elif anonymize: + if anonymize: parts = [] cursor = 0 suffix = "REDACTED" if method == "redact" else "XXXXX" for finding in sorted(findings, key=lambda item: item["start"]): - parts.extend( - (text[cursor : finding["start"]], f"[{finding['type']}_{suffix}]") - ) + if method == "hash": + digest = _hash_value( + finding["value"], algorithm, hash_key, plain_length=8 + ) + replacement = f"[{finding['type']}_{digest}]" + else: + replacement = f"[{finding['type']}_{suffix}]" + parts.extend((text[cursor : finding["start"]], replacement)) cursor = finding["end"] parts.append(text[cursor:]) result["anonymized"] = "".join(parts) diff --git a/datafog/engine.py b/datafog/engine.py index 884bb1de..0d4a787e 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -809,6 +809,26 @@ def _validate_hash_parameters( raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None +def _hash_value( + text: str, + hash_type: str, + hash_key: bytes | None, + *, + plain_length: int = 12, +) -> str: + """Retain legacy plain-hash widths; keyed HMAC always uses its full digest. + + Unkeyed hashes are deterministic pseudonyms and remain vulnerable to + dictionary attacks regardless of digest width. Supply a private key to + prevent direct enumeration of low-entropy identifiers. + """ + _validate_hash_parameters("hash", hash_type, hash_key) + encoded = text.encode("utf-8", "surrogatepass") + if hash_key is not None: + return hmac.new(hash_key, encoded, hash_type).hexdigest() + return hashlib.new(hash_type, encoded).hexdigest()[:plain_length] + + def _validated_redaction_entities(text: str, entities: list[Entity]) -> list[Entity]: valid_entities = [] for entity in entities: @@ -842,6 +862,10 @@ def redact( type priority and then confidence. Suppressed spans are omitted from ``RedactResult.entities`` and ``mapping``. + Plain hashes retain the historical 12-character digest prefix and provide + deterministic pseudonymization. With ``hash_key`` (at least 32 bytes), HMAC + uses the complete digest. Unkeyed hashes do not resist dictionary attacks. + Invalid ranges, mismatched source text, and malformed labels raise ValueError. ``mapping`` for the ``mask`` strategy is keyed by per-type indexed keys @@ -881,12 +905,7 @@ def redact( break occupied_tokens.add(mapping_key) elif strategy == "hash": - encoded = original.encode("utf-8", "surrogatepass") - digest = ( - hmac.new(hash_key, encoded, hash_type).hexdigest() - if hash_key is not None - else hashlib.new(hash_type, encoded).hexdigest() - ) + digest = _hash_value(original, hash_type, hash_key) key = (entity.type, original) if key not in pseudonym_by_value: candidate = f"[{entity.type}_{digest}]" diff --git a/docs/review-remediation.md b/docs/review-remediation.md index 86e37cc4..04e010ea 100644 --- a/docs/review-remediation.md +++ b/docs/review-remediation.md @@ -30,8 +30,14 @@ nonblocking events receive an explicit incomplete-scan notice. Unexpected hook failures retain the documented fail-open contract and log exception types only. `hash_type` selects md5, sha256 or sha3_256 on the CLI and legacy model APIs. -The engine also accepts algorithms provided by hashlib. Digests use full hex -lengths. Plain deterministic hashes are pseudonymization: SSNs, phones and +The engine also accepts fixed-length algorithms provided by hashlib. Existing +plain-hash marker formats remain compatible: engine, class facade and CLI +markers use 12 hex characters with SHA256 as their default; deprecated top-level +`process` uses eight hex characters and retains its MD5 default. Its explicit +`hash_type` option uses that same legacy width. The legacy Anonymizer model keeps +its existing full plain digests. Explicit keyed hashes use the full HMAC digest; +an omitted algorithm selects SHA256 for this new keyed mode. +Plain deterministic hashes are pseudonymization: SSNs, phones and other guessable inputs can be enumerated. Use `hash_key` (at least 32 random bytes) for keyed HMAC and keep the key outside logs and source control: @@ -91,7 +97,7 @@ release still requires the Core publication and Python dependency update above. | PY-M4 | Native backend selection also supplies the structured stage of smart/spaCy/GLiNER. CLI commands work with DATAFOG_BACKEND=rust. | | PY-M5 | Request-scoped tokens, nested tool/function arguments, prompt/input fields and buffered streaming protect all tested request/response shapes. Live proxy deployment is not claimed. | | PY-M6 | Bare strings, invalid labels and labels unavailable to the selected backend/engine raise explicit errors. Supported aliases remain accepted. | -| PY-M7 | APIs and CLI honor hash_type, use full digests and support keyed HMAC. Plain-hash guessing limitations are documented; algorithm/key tests pass. | +| PY-M7 | APIs and CLI honor hash_type, preserve existing plain-hash formats and support full-length keyed HMAC. Plain-hash guessing limitations are documented; algorithm/key and compatibility tests pass. | | PY-M8 | Hook per-string/total budget exhaustion emits review/deny or an incomplete-scan notice. Tests put PII past the cap. | | PY-M9 | Allowlist expressions run with match timeouts; timed-out expressions retain findings. Adversarial patterns and safe quantified groups are tested. | | PY-M10 | Dependency resolution stays within the explicitly reviewed Core minor, replacing the broad less-than-2 range. Exact output locks should still pin patch versions. | diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json index b3ddf293..3f47b3d6 100644 --- a/tests/contracts/wrapper-4.12.0.json +++ b/tests/contracts/wrapper-4.12.0.json @@ -131,151 +131,6 @@ "warnings": [] } }, - "redact-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", - "expected": { - "value": { - "result_type": "RedactResult", - "fields": { - "redacted_text": "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c] [EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [SSN_01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0]", - "mapping": { - "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]": "a@example.com", - "[EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c]": "b@example.com", - "[SSN_01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0]": "123-45-6789" - }, - "entities": [ - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "a@example.com", - "start": 0, - "end": 13, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "b@example.com", - "start": 14, - "end": 27, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "a@example.com", - "start": 28, - "end": 41, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "SSN", - "text": "123-45-6789", - "start": 42, - "end": 53, - "confidence": 1.0, - "engine": "regex" - } - } - ] - } - }, - "warnings": [] - } - }, - "engine-redact-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", - "expected": { - "value": { - "result_type": "RedactResult", - "fields": { - "redacted_text": "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c] [EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]", - "mapping": { - "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]": "a@example.com", - "[EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c]": "b@example.com" - }, - "entities": [ - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "a@example.com", - "start": 0, - "end": 13, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "b@example.com", - "start": 14, - "end": 27, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "a@example.com", - "start": 28, - "end": 41, - "confidence": 1.0, - "engine": "regex" - } - } - ] - } - }, - "warnings": [] - } - }, - "preset-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", - "expected": { - "value": { - "result_type": "RedactResult", - "fields": { - "redacted_text": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]", - "mapping": { - "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]": "alice@example.com" - }, - "entities": [ - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "alice@example.com", - "start": 0, - "end": 17, - "confidence": 1.0, - "engine": "regex" - } - } - ] - } - }, - "warnings": [] - } - }, "explicit-invalid-bounds": { "classification": "validation-difference", "reason": "PY-L7 rejects malformed explicit spans rather than silently skipping them.", @@ -495,8 +350,8 @@ } }, "process-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 replaces legacy truncated MD5 with the complete selected digest; PY-L3 updates the retirement notice.", + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release. Legacy default MD5 and its eight-character prefix remain compatible; explicit algorithms and keyed HMAC are additional options.", "expected": { "value": { "original": "alice@example.com", @@ -508,7 +363,7 @@ "end": 17 } ], - "anonymized": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]" + "anonymized": "[EMAIL_c160f8cc]" }, "warnings": [ { @@ -569,7 +424,7 @@ }, "public-signatures": { "classification": "signature-addition", - "reason": "PY-M7 adds keyword-only hash_type/hash_key; TextService custom models use the engine gliner_model keyword; backend remains keyword-only.", + "reason": "PY-M7 adds keyword-only hash_type/hash_key while preserving existing unkeyed output widths and omitted legacy hash defaults; TextService custom models use the engine gliner_model keyword; backend remains keyword-only.", "expected": { "value": { "datafog:scan": { @@ -762,7 +617,7 @@ "name": "hash_type", "kind": "KEYWORD_ONLY", "required": false, - "default": "sha256" + "default": null }, { "name": "hash_key", @@ -1112,14 +967,6 @@ }, "warnings": [] } - }, - "anonymize-text-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 emits the complete chosen digest rather than a truncated unsalted token.", - "expected": { - "value": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]", - "warnings": [] - } } } } diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py index 9f47318a..b65342a7 100644 --- a/tests/test_review_engine.py +++ b/tests/test_review_engine.py @@ -236,7 +236,7 @@ def test_hash_algorithm_and_hmac_are_honored(algorithm): text = "a@b.com" entities = engine.scan(text, engine="regex").entities plain = engine.redact(text, entities, strategy="hash", hash_type=algorithm) - assert hashlib.new(algorithm, text.encode()).hexdigest() in plain.redacted_text + assert hashlib.new(algorithm, text.encode()).hexdigest()[:12] in plain.redacted_text key = bytes(range(32)) keyed = engine.redact( text, entities, strategy="hash", hash_type=algorithm, hash_key=key @@ -298,7 +298,7 @@ def test_fallback_adversarial_patterns_complete(): def test_hash_marker_cannot_collide_with_literal_input(): value = "a@b.com" - digest = hashlib.sha256(value.encode()).hexdigest() + digest = hashlib.sha256(value.encode()).hexdigest()[:12] marker = f"[EMAIL_{digest}]" text = f"{marker} {value} {value}" result = engine.scan_and_redact(text, engine="regex", strategy="hash") @@ -329,3 +329,16 @@ def test_fallback_email_context_is_trimmed_without_losing_mailboxes(text): def test_spacy_selection_rejects_gliner_only_labels(): with pytest.raises(ValueError, match="PASSPORT_NUMBER"): engine.scan("passport SECRET", engine="spacy", entity_types=["PASSPORT_NUMBER"]) + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256", "sha3_256"]) +def test_shared_hash_helper_retains_legacy_widths_and_full_hmac(algorithm): + value = "a@b.com" + digest = hashlib.new(algorithm, value.encode()).hexdigest() + assert engine._hash_value(value, algorithm, None) == digest[:12] + assert engine._hash_value(value, algorithm, None, plain_length=8) == digest[:8] + key = bytes(range(32)) + assert ( + engine._hash_value(value, algorithm, key, plain_length=8) + == hmac.new(key, value.encode(), algorithm).hexdigest() + ) diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py index fb7cc555..c471bf06 100644 --- a/tests/test_review_entrypoints.py +++ b/tests/test_review_entrypoints.py @@ -55,14 +55,14 @@ def test_anonymizing_operations_imply_scan(operation): def test_hash_algorithm_honored_across_api_and_cli(algorithm): digest = hashlib.new(algorithm.value, EMAIL.encode()).hexdigest() assert ( - digest - in DataFog(hash_type=algorithm).process(EMAIL, anonymize=True, method="hash")[ + f"[EMAIL_{digest[:12]}]" + == DataFog(hash_type=algorithm).process(EMAIL, anonymize=True, method="hash")[ "anonymized" ] ) assert ( - digest - in datafog.process( + f"[EMAIL_{digest[:8]}]" + == datafog.process( EMAIL, anonymize=True, method="hash", hash_type=algorithm.value )["anonymized"] ) @@ -70,7 +70,18 @@ def test_hash_algorithm_honored_across_api_and_cli(algorithm): app, ["hash-text", EMAIL, "--hash-type", algorithm.value] ) assert result.exit_code == 0, result.output - assert digest in result.stdout + assert result.stdout.strip() == f"[EMAIL_{digest[:12]}]" + + +def test_existing_plain_hash_defaults_remain_compatible(): + assert ( + datafog.redact(EMAIL, strategy="hash").redacted_text + == f"[EMAIL_{hashlib.sha256(EMAIL.encode()).hexdigest()[:12]}]" + ) + assert ( + datafog.process(EMAIL, anonymize=True, method="hash")["anonymized"] + == f"[EMAIL_{hashlib.md5(EMAIL.encode()).hexdigest()[:8]}]" + ) def test_keyed_hash_public_facades(): From e4e751ecd28feb4b6c5e8469664936754b08050e Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:25:19 -0800 Subject: [PATCH 20/23] fix(engine): retain structured findings when NER spans overlap --- datafog/engine.py | 4 +++- tests/corpus/unstructured_pii.json | 4 ++-- tests/test_review_engine.py | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/datafog/engine.py b/datafog/engine.py index 0d4a787e..c8041acf 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -789,7 +789,9 @@ def scan( filtered = _filter_entity_types(combined, entity_types) filtered = _apply_allowlist(filtered, allowlist, allowlist_patterns) - deduped = _suppress_overlapping_entities(_dedupe_entities(filtered)) + # Optional NER adds candidates without suppressing structured findings. + # Transformation resolves overlaps after selection, preserving scan labels. + deduped = _dedupe_entities(filtered) return ScanResult( entities=deduped, text=text, diff --git a/tests/corpus/unstructured_pii.json b/tests/corpus/unstructured_pii.json index ad91c35b..1e5ddf79 100644 --- a/tests/corpus/unstructured_pii.json +++ b/tests/corpus/unstructured_pii.json @@ -168,8 +168,8 @@ { "type": "LOCATION", "text": "Jordan", - "start": 0, - "end": 6 + "start": 33, + "end": 39 } ] }, diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py index b65342a7..5e44c461 100644 --- a/tests/test_review_engine.py +++ b/tests/test_review_engine.py @@ -342,3 +342,19 @@ def test_shared_hash_helper_retains_legacy_widths_and_full_hmac(algorithm): engine._hash_value(value, algorithm, key, plain_length=8) == hmac.new(key, value.encode(), algorithm).hexdigest() ) + + +@pytest.mark.parametrize("mode", ["smart", "spacy", "gliner"]) +def test_optional_ner_cannot_remove_overlapping_structured_scan_findings( + monkeypatch, mode +): + text = "Ping 127.0.0.1" + candidates = [engine.Entity("PERSON", text, 0, len(text), 0.8, mode)] + monkeypatch.setattr(engine, "_spacy_entities", lambda _: candidates) + monkeypatch.setattr(engine, "_gliner_entities", lambda _: candidates) + result = engine.scan(text, engine=mode) + assert any( + item.type == "IP_ADDRESS" and item.text == "127.0.0.1" + for item in result.entities + ) + assert any(item.type == "PERSON" for item in result.entities) From 260b04f76c78b18485f0182d8b78a2b860c001bd Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:40:16 -0800 Subject: [PATCH 21/23] fix(anonymizer): preserve protection with legacy empty entity filters --- datafog/models/anonymizer.py | 2 +- tests/test_review_entrypoints.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datafog/models/anonymizer.py b/datafog/models/anonymizer.py index 30d079bb..3ec36664 100644 --- a/datafog/models/anonymizer.py +++ b/datafog/models/anonymizer.py @@ -55,7 +55,7 @@ def _selected_annotations(self, text, annotations): selected = ( None - if self.entities is None + if not self.entities else { CANONICAL_TYPE_MAP.get(entity.name, entity.name) for entity in self.entities diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py index c471bf06..519c9cf6 100644 --- a/tests/test_review_entrypoints.py +++ b/tests/test_review_entrypoints.py @@ -143,6 +143,16 @@ def test_enum_filter_and_date_replacement(): ) +@pytest.mark.parametrize("strategy", list(AnonymizerType)) +def test_legacy_empty_entity_selection_keeps_protecting_all(strategy): + annotation = AnnotationResult(start=0, end=len(EMAIL), entity_type="EMAIL") + result = Anonymizer(anonymizer_type=strategy, entities=[]).anonymize( + EMAIL, [annotation] + ) + assert EMAIL not in result.anonymized_text + assert len(result.anonymized_entities) == 1 + + @pytest.mark.parametrize("strategy", list(AnonymizerType)) def test_legacy_overlaps_never_leave_partial_value(strategy): text = "abcdefghij" From d6b8a2da8799a23560b35b9f0ad6ad488e90337a Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:41:39 -0800 Subject: [PATCH 22/23] test: recognize exact restored GLiNER URL label coverage --- tests/corpus/negative_cases.json | 22 ++++++++++- tests/test_detection_accuracy.py | 64 ++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/tests/corpus/negative_cases.json b/tests/corpus/negative_cases.json index 44eb1fcf..c5b67b76 100644 --- a/tests/corpus/negative_cases.json +++ b/tests/corpus/negative_cases.json @@ -49,7 +49,16 @@ { "id": "markdown-link", "input": "See [RFC 1918](https://example.com/rfc1918)", - "expected_entities": [] + "expected_entities": [], + "permitted_gliner_entities": [ + { + "type": "URL", + "text": "https://example.com/rfc1918", + "start": 15, + "end": 42 + } + ], + "permitted_gliner_reason": "PY-M14 restores the existing GLiNER URL label. This exact public URL is valid URL detection; it must not become EMAIL, PHONE or IP_ADDRESS." }, { "id": "code-symbol", @@ -74,6 +83,15 @@ { "id": "url-with-at", "input": "https://example.com/@user/profile is a URL path", - "expected_entities": [] + "expected_entities": [], + "permitted_gliner_entities": [ + { + "type": "URL", + "text": "https://example.com/@user/profile", + "start": 0, + "end": 33 + } + ], + "permitted_gliner_reason": "PY-M14 restores the existing GLiNER URL label. This exact public URL is valid URL detection; it must not become EMAIL, PHONE or IP_ADDRESS." } ] diff --git a/tests/test_detection_accuracy.py b/tests/test_detection_accuracy.py index 6e7d8dfe..8cff1e82 100644 --- a/tests/test_detection_accuracy.py +++ b/tests/test_detection_accuracy.py @@ -453,6 +453,33 @@ def _assert_expected_found( return actual, expected +def _unexpected_negative_entities( + case: dict[str, Any], actual: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Keep legacy negative PII checks while recognizing two reviewed URL spans. + + PY-M14 restores GLiNER's URL output. Only exact, source-aligned URL candidates + from that model are permitted; other labels and malformed offsets still fail. + These added label checks are outside the historical PII metric inventory. + """ + permitted = case.get("permitted_gliner_entities", []) + return [ + entity + for entity in actual + if not ( + entity.get("engine") == "gliner" + and any( + all( + entity.get(key) == span[key] + for key in ("type", "text", "start", "end") + ) + and case["input"][span["start"] : span["end"]] == span["text"] + for span in permitted + ) + ) + ] + + def _compute_metrics( engines: list[str], corpora: list[tuple[str, list[dict[str, Any]]]] ) -> dict[str, Any]: @@ -471,6 +498,8 @@ def _compute_metrics( locales=case.get("locales"), backend=_corpus_backend(engine), ) + if corpus_kind == "negative": + actual = _unexpected_negative_entities(case, actual) expected = _required_expected( case["expected_entities"], engine, corpus_kind ) @@ -569,6 +598,7 @@ def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None: for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) } actual = [e for e in actual if (e["type"], e["text"]) not in added] + actual = _unexpected_negative_entities(case, actual) assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -591,6 +621,7 @@ def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None: for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) } actual = [e for e in actual if (e["type"], e["text"]) not in added] + actual = _unexpected_negative_entities(case, actual) assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -666,3 +697,36 @@ def test_accuracy_metrics_snapshot() -> None: output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(json.dumps(metrics, indent=2), encoding="utf-8") assert "overall" in metrics and metrics["overall"] + + +def test_reviewed_url_candidates_require_exact_label_origin_and_offsets(): + case = next( + case + for case in load_corpus("negative_cases.json") + if case["id"] == "markdown-link" + ) + permitted = dict(case["permitted_gliner_entities"][0], engine="gliner") + assert _unexpected_negative_entities(case, [permitted]) == [] + for altered in [ + dict(permitted, type="EMAIL"), + dict(permitted, type="PHONE"), + dict(permitted, type="IP_ADDRESS"), + dict(permitted, start=14), + dict(permitted, end=41), + dict(permitted, engine="spacy"), + dict(permitted, text="example.com"), + ]: + assert _unexpected_negative_entities(case, [altered]) == [altered] + + +def test_reviewed_url_permission_is_confined_to_the_two_named_cases(): + permitted_cases = [ + case + for case in load_corpus("negative_cases.json") + if case.get("permitted_gliner_entities") + ] + assert {case["id"] for case in permitted_cases} == {"markdown-link", "url-with-at"} + for case in permitted_cases: + assert case["permitted_gliner_reason"].startswith("PY-M14") + assert len(case["permitted_gliner_entities"]) == 1 + assert case["permitted_gliner_entities"][0]["type"] == "URL" From 5476c74716ac27356d7029f3b4007058e41b3859 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:55:19 -0800 Subject: [PATCH 23/23] chore: prepare Core 1.5 compatibility qualification --- README.md | 2 +- docs/review-remediation.md | 14 +++++++------- .../contracts/{rust-2.0.0.json => rust-1.5.0.json} | 4 ++-- tests/contracts/wrapper-4.12.0.json | 2 +- ...0_integration.py => test_core15_integration.py} | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) rename tests/contracts/{rust-2.0.0.json => rust-1.5.0.json} (91%) rename tests/{test_core20_integration.py => test_core15_integration.py} (88%) diff --git a/README.md b/README.md index 375cbc53..73749bed 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ Telemetry does not include input text or detected PII values. The native engine is the default, with a warned Python fallback. The 4.12 review fixes preserve structured detection across every engine and legacy API. The dependency stays within the reviewed Core 1.4 minor series (`~=1.4.0`); -Core 2.0 is a separate release candidate that must be published and qualified +Core 1.5 is a separate release candidate that must be published and qualified before this SDK raises its floor. See [release preparation](docs/review-remediation.md). Entity labels, locales and activation settings come from the installed Core. diff --git a/docs/review-remediation.md b/docs/review-remediation.md index 04e010ea..7618b363 100644 --- a/docs/review-remediation.md +++ b/docs/review-remediation.md @@ -51,11 +51,11 @@ result = datafog.redact("mail jane@example.com", strategy="hash", hash_key=key) ## Release sequence and qualification The Python candidate is 4.12.0. Its standalone CI uses the published Core 1.4 -minor series, pinned with `~=1.4.0`. Core 2.0.0 is prepared separately because -safe mask-reveal and automatic-overlap transformation semantics are incompatible -changes under Core's version policy. Both use capability contract 1. Before +minor series, pinned with `~=1.4.0`. Core 1.5.0 is prepared separately as a minor +release correcting detection and transformation behavior through the existing +APIs. Public APIs remain available and capability contract 1 is unchanged. Before shipping the combined fixes, publish the qualified Core candidate, then raise -both Python dependency declarations to `~=2.0.0`, update the reviewed native +both Python dependency declarations to `~=1.5.0`, update the reviewed native contract/corpus, and run Python's installed-wheel and release qualification against the published artifacts. Source-candidate integration is additional evidence; it does not substitute for that post-publication gate. @@ -106,9 +106,9 @@ release still requires the Core publication and Python dependency update above. | PY-M13 | Optional Pydantic fields have defaults, labels use canonical names, enum filters work and overlaps merge before replacement. Model/anonymizer regressions pass. | | PY-M14 | Preserve supported GLiNER labels, use exact offsets, exclude relative spaCy DATE mentions, propagate model failures and reject oversized annotator input explicitly. | | PY-M15 | Embedded model calls use bounded overlapping chunks. Large PERSON/ADDRESS plus structured-PII tests succeed with published and candidate Core. Native model limits remain enforced. | -| PY-M16 | Core 2 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | +| PY-M16 | Core 1.5 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | | PY-L1 | Console bootstrap prints the CLI-extra install hint on a base install. A clean installed-wheel check confirms no traceback. | -| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 2 suppresses the labelled-tax-ID PHONE false positive. | +| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 1.5 suppresses the labelled-tax-ID PHONE false positive. | | PY-L3 | Retirement notices derive the current supported release version, and the native install hint no longer names an older release. | | PY-L4 | Supported-entity reporting follows the selected native/Python capability inventory, including opt-in labels. Tests separate supported from default-active labels. | | PY-L5 | Document actual token/pseudonym outputs and reject invalid methods in both process APIs. | @@ -116,7 +116,7 @@ release still requires the Core publication and Python dependency update above. | PY-L7 | Tokens reuse values and avoid existing markers; invalid explicit ranges/text raise errors. Collision, repeated-value and validation tests pass. | | PY-L8 | Remove unused implementation copies, declare an isolated build, include test packages/corpora/scripts in sdists, align formatter pins and extend tox through 3.14. Optional OCR imports skip cleanly when absent. | | PY-L9 | CLI configuration hides API keys; import and commands preserve host logging levels. CLI/logger regressions pass. | -| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 2 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | +| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 1.5 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | | PY-L11 | TextService validates positive chunk lengths and engine names with explicit exceptions, including optimized Python execution. | | PY-L12 | Remove incompatible Torch upper bound and include spaCy in nlp-advanced. Resolving it with Core GLiNER2 succeeds with Torch 2.12.0. | | PY-L13 | Declare Pydantic 2.1 minimum. A clean Python 3.10 base install with Pydantic 2.1 imports and scans successfully. | diff --git a/tests/contracts/rust-2.0.0.json b/tests/contracts/rust-1.5.0.json similarity index 91% rename from tests/contracts/rust-2.0.0.json rename to tests/contracts/rust-1.5.0.json index 324f9cfa..13f39961 100644 --- a/tests/contracts/rust-2.0.0.json +++ b/tests/contracts/rust-1.5.0.json @@ -1,5 +1,5 @@ { - "core_version": "2.0.0", + "core_version": "1.5.0", "cases": { "scan-observation-invalid-card": { "classification": "detector-difference", @@ -73,5 +73,5 @@ } } }, - "review": "Core 2.0 candidate qualifications retain the four reviewed historical differences below. Wrapper 4.12 fixes are layered separately in wrapper-4.12.0.json." + "review": "Core 1.5 candidate qualification retains reviewed historical differences and documents the contextual tax-ID correction. Wrapper 4.12 fixes are layered separately in wrapper-4.12.0.json." } diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json index 3f47b3d6..cd78ae0d 100644 --- a/tests/contracts/wrapper-4.12.0.json +++ b/tests/contracts/wrapper-4.12.0.json @@ -1,6 +1,6 @@ { "wrapper_version": "4.12.0", - "core_versions": ["1.4.0", "2.0.0"], + "core_versions": ["1.4.0", "1.5.0"], "source_oracle": "4.8.1.json (unchanged)", "review": "2026-11-15 PDF remediation: explicit reviewed wrapper changes, not a regenerated historical oracle", "cases": { diff --git a/tests/test_core20_integration.py b/tests/test_core15_integration.py similarity index 88% rename from tests/test_core20_integration.py rename to tests/test_core15_integration.py index 290f8e0d..ed9bcdf4 100644 --- a/tests/test_core20_integration.py +++ b/tests/test_core15_integration.py @@ -1,4 +1,4 @@ -"""Core 2 candidate seam regressions; exercised before the dependency floor moves.""" +"""Core 1.5 candidate seam regressions; exercised before the dependency floor moves.""" import importlib.metadata @@ -7,8 +7,8 @@ import datafog pytestmark = pytest.mark.skipif( - importlib.metadata.version("datafog-core").split(".")[0] != "2", - reason="qualifies the separately built Core 2 candidate before publication", + importlib.metadata.version("datafog-core").split(".")[:2] != ["1", "5"], + reason="qualifies the separately built Core 1.5 candidate before publication", )