From 4ad165ff821ee26a6252894bd005e2b53b03f644 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:02:48 +0000 Subject: [PATCH 1/8] chore: bump version to 4.9.0a1 [skip ci] --- datafog/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafog/__about__.py b/datafog/__about__.py index 454a2eda..5642cf5b 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.9.0" +__version__ = "4.9.0a1" From ce5fb8cad704af2462c7e97f3361d8acb09fcfc8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:32:17 +0000 Subject: [PATCH 2/8] chore: bump version to 4.9.0b1 [skip ci] --- datafog/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafog/__about__.py b/datafog/__about__.py index 5642cf5b..680a4969 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.9.0a1" +__version__ = "4.9.0b1" From cf81370dfd7a667d05969f38b88ebad9e38c94c1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 08:34:54 +0000 Subject: [PATCH 3/8] chore: bump version to 4.9.0a2 [skip ci] --- datafog/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafog/__about__.py b/datafog/__about__.py index 680a4969..21d77306 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.9.0b1" +__version__ = "4.9.0a2" From 2b2d114eab01b972302240f144a70ffa4310930d Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:53:51 -0800 Subject: [PATCH 4/8] feat: native engine by default with Python fallback (4.10.0) Resolve backend through 'auto': the native engine when datafog-core imports, the pure-Python engine otherwise with a single warning. DATAFOG_BACKEND and backend="python" force the fallback; non-regex engines stay on the Python ML path. Serve prose PERSON and ADDRESS from the embedded model in datafog-core, so names and addresses need no spaCy, GLiNER, download or interpreter. Expose the new credential, account and UUID entity types through the alias API and add datafog.native for the native schema. Deprecation warnings now name 4.10.0 as the last supported version with an exact pin, and VERSIONING.md plus docs/retirement.rst make the window and the removal schedule auditable. Nothing is removed in this release. --- CHANGELOG.MD | 26 +++ README.md | 53 +++-- RELEASE_NOTES_4.10.0.md | 61 ++++++ VERSIONING.md | 123 +++++++++++ datafog/__about__.py | 2 +- datafog/__init__.py | 16 +- datafog/_core_capabilities.py | 13 +- datafog/_legacy_retirement.py | 6 +- datafog/compat/v4.py | 15 +- datafog/engine.py | 194 ++++++++++++++++-- datafog/exceptions.py | 4 + datafog/native.py | 60 ++++++ docs/cli.rst | 3 +- docs/important-concepts.rst | 2 +- docs/index.rst | 28 ++- docs/migration-4.10.md | 92 +++++++++ docs/migration-4.9.md | 5 +- docs/optional-surfaces.rst | 3 +- docs/python-sdk.rst | 20 +- docs/retirement.rst | 115 +++++++++++ scripts/check_rust_install.py | 10 +- setup.py | 8 +- .../{rust-0.4.0.json => rust-1.2.0.json} | 17 +- tests/rust_contract.py | 2 +- tests/test_allowlist.py | 14 +- tests/test_api_bridge_49.py | 2 +- tests/test_backend_resolution.py | 111 ++++++++++ tests/test_contract_481.py | 17 +- tests/test_core04_integration.py | 38 ++-- tests/test_de_pii_regex.py | 17 +- tests/test_legacy_retirement.py | 11 +- tests/test_rust_backend.py | 15 +- tests/test_rust_contract.py | 4 +- 33 files changed, 999 insertions(+), 108 deletions(-) create mode 100644 RELEASE_NOTES_4.10.0.md create mode 100644 VERSIONING.md create mode 100644 datafog/native.py create mode 100644 docs/migration-4.10.md create mode 100644 docs/retirement.rst rename tests/contracts/{rust-0.4.0.json => rust-1.2.0.json} (70%) create mode 100644 tests/test_backend_resolution.py diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 576597a3..1049239e 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,5 +1,31 @@ # ChangeLog +## [4.10.0] + +#### Added + +- Native engine is the default: `backend` resolves through `auto`, which uses + `datafog-core` when it imports and the pure-Python engine otherwise. One + warning on fallback, `DATAFOG_BACKEND` and `backend="python"` to opt out. +- Prose `PERSON` and `ADDRESS` from the model embedded in `datafog-core`; no + spaCy, GLiNER, download or interpreter. +- `API_KEY`, `BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, + `ACCOUNT_NUMBER`, `US_ROUTING_NUMBER`, `NPI` and `UUID` entity types, plus the + German set behind the `de` locale. +- `datafog.native` native schema access and `VERSIONING.md`. + +#### Changed + +- `datafog-core>=1.2.0,<2` is a normal dependency; `[rust]` remains as an + explicit spelling. +- `Development Status: 5 - Production/Stable`. +- Retirement warnings name `4.10.0` as the last supported version with an exact + pin, replacing "the final 4.x release". + +#### Fixed + +- `redact(..., entities=[...])` validates `backend` instead of ignoring it. + ## [4.9.0] #### Added diff --git a/README.md b/README.md index 4fd354a3..0f70fb23 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,9 @@ values never echoed into logs or transcripts: - **Claude Code hook** (`datafog-hook`): gates agent tool calls (shell commands, web requests, file writes, MCP tools) and warns the model when - prompts or tool results carry PII. ~70–90ms per invocation including - process startup. Easiest install is the + prompts or tool results carry PII. ~105–110ms per invocation including + process startup and the native engine load (`DATAFOG_BACKEND=python` trims + ~20ms at the cost of detection coverage). Easiest install is the [Claude Code plugin](https://github.com/DataFog/datafog-claude-plugin): ``` @@ -32,8 +33,8 @@ values never echoed into logs or transcripts: - **LiteLLM guardrail** (`DataFogGuardrail`): redacts or blocks PII in requests and responses at the gateway, for any LiteLLM-proxied provider. - In-process (~40µs per message scanned; a request clears the guardrail in - well under a millisecond), no sidecar service. Setup: + In-process (tens of microseconds per message scanned on the native engine), + no sidecar service. Setup: [examples/litellm_guardrail/](examples/litellm_guardrail/). Both default to the high-precision entity set (`EMAIL`, `PHONE`, @@ -51,7 +52,7 @@ NER live in [benchmarks/](benchmarks/). ## Installation ```bash -# Core install (regex engine) +# Core install: native engine, no extra packages needed pip install datafog # Add spaCy support @@ -70,9 +71,33 @@ pip install datafog[distributed] pip install datafog[all] ``` +`pip install datafog` pulls `datafog-core`, which is the native engine. Wheels +exist for Linux (glibc and musl, x86_64/aarch64/s390x/ppc64le), macOS and +Windows (amd64/arm64); anywhere else the source build needs a Rust toolchain. +If Core cannot be imported, detection falls back to the pure-Python engine and +warns once — `DATAFOG_BACKEND=python` selects that path deliberately. + The `nlp-advanced` extra includes SentencePiece and protobuf for GLiNER's multilingual tokenizer; installing the OCR extra is not required for GLiNER. +## Backends + +`backend` defaults to `auto`: the native engine when `datafog-core` is +importable, the pure-Python regex engine otherwise. `DATAFOG_BACKEND` sets the +default and `backend="python"` / `backend="rust"` override it per call. +`auto` never selects the native engine for `engine="spacy"`, `"gliner"` or +`"smart"`, which stay on the Python ML path. + +The native engine adds detectors the Python engine never had — `API_KEY`, +`BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, +`US_ROUTING_NUMBER`, `NPI`, `UUID` and a German set — and serves prose `PERSON` +and `ADDRESS` from an embedded model, so name and address detection needs no +spaCy and no model download. Findings keep `engine="regex"` provenance and +`confidence=1.0` as compatibility values, not native confidence estimates. +Reviewed differences from the Python engine are listed in +[tests/contracts/rust-1.2.0.json](tests/contracts/rust-1.2.0.json). +`datafog.native` exposes the native schema directly. + Python 3.13 support is certified for the core SDK, CLI, `nlp`, `nlp-advanced`, and `ocr` install profiles. Donut OCR still requires a model that is available locally before runtime use. `distributed` and `all` remain @@ -188,10 +213,12 @@ scan/redact helpers, or guardrail helpers. model. - A Java runtime is required by PySpark. -DataFog 4.9.0 deprecates OCR and Spark with visible use-time -warnings; their APIs and extras will be removed in 5.0. They remain functional -in 4.9. Users who need these features can stay on the final 4.x release. See -the [4.9 migration guide](docs/migration-4.9.md) for the transition plan. +DataFog deprecates OCR and Spark with visible use-time +warnings; their APIs and extras will be removed in 5.0. The last supported +version is 4.10.0 — pin it with `pip install "datafog==4.10.0"` — and every +retired surface is tracked in the [retirement schedule](docs/retirement.rst). +See the [4.9 migration guide](docs/migration-4.9.md) for the transition plan, +and [VERSIONING.md](VERSIONING.md) for what a version number promises. ## Backward-Compatible APIs @@ -261,11 +288,9 @@ The [4.9 migration guide](docs/migration-4.9.md) explains opt-in Rust detection, the native `datafog.v5` preview, and the revised 5.0 retirement schedule for `detect`/`process`, OCR, and Spark. The Python detector remains the default. -The experimental Rust adapter in 4.9.0 requires Core `>=0.4.0,<0.5` and capability -contract 1. Upgrade with `python -m pip install --upgrade "datafog[rust]==4.9.0"` -to evaluate it. Base-only users can install `datafog==4.9.0` without Core; -installing the Rust extra does not change the default backend. See the -[4.9.0 release notes](RELEASE_NOTES_4.9.0.md). +4.10.0 requires Core `>=1.2.0,<2` and capability contract 1, and installs it by +default. See the [4.10.0 release notes](RELEASE_NOTES_4.10.0.md) and the +[4.10 migration guide](docs/migration-4.10.md). Entity labels, locales, and activation settings come from the installed Core, allowing compatible releases to add detectors without a Python update. German detection is opt-in through locale or entity selection; UUID is opt-in diff --git a/RELEASE_NOTES_4.10.0.md b/RELEASE_NOTES_4.10.0.md new file mode 100644 index 00000000..f5c592d4 --- /dev/null +++ b/RELEASE_NOTES_4.10.0.md @@ -0,0 +1,61 @@ +# DataFog Python 4.10.0 + +4.10.0 turns on the native engine. `pip install datafog` installs `datafog-core` +and detection uses it. Nothing is removed in this release and no call changes +shape; `backend="python"` restores the previous behaviour exactly. + +Read the [4.10 migration guide](docs/migration-4.10.md) before upgrading a +pipeline whose output is under test. + +## Upgrade + +```bash +python -m pip install --upgrade "datafog==4.10.0" +``` + +Wheels cover Linux (glibc and musl, x86_64/aarch64/s390x/ppc64le), macOS and +Windows (amd64/arm64). Elsewhere the source build needs Rust 1.88+. If Core +cannot be imported, detection falls back to the pure-Python engine and warns +once. + +## What changes + +- `backend` defaults to `auto`: native when `datafog-core` imports, Python + otherwise. `DATAFOG_BACKEND` sets the default; `backend="python"` forces the + fallback. Non-regex engines stay on the Python path. +- `PERSON` and `ADDRESS` for prose now come from an embedded model compiled into + `datafog-core` — no spaCy, no GLiNER, no model download, no interpreter. +- The native engine adds `API_KEY`, `BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, + `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, `US_ROUTING_NUMBER`, `NPI`, `UUID` and the + German identifiers. Credential detectors are on by default; German and `UUID` + stay opt-in. +- `datafog.native` exposes the native schema, `capabilities()`, the embedded + detector and version reporting. Experimental until 5.0. +- `datafog-core` is pinned `>=1.2.0,<2`. Detection output is contract-versioned. + +## Detection differences + +Four reviewed differences from the Python engine are recorded in +[tests/contracts/rust-1.2.0.json](tests/contracts/rust-1.2.0.json) and asserted +by the suite: Core validates card checksums, requires non-alphanumeric SSN +boundaries, requires a North American area code starting 2-9 (which is what +keeps Unix timestamps out of `PHONE`), and rejects unadvertised locales instead +of ignoring them. + +`(555) 123-4567` and similar form data now match: Core keeps the area-code check +and accepts exchange codes starting with 1. + +## Deprecations unchanged + +`detect()`, `process()`, OCR and Spark still work and still warn; all are removed +in 5.0. Their messages now name `4.10.0` as the last supported version and give +the exact pin. See the [retirement schedule](docs/retirement.rst). + +## Notes + +- `VERSIONING.md` is new and defines what a version number promises here, with a + deprecation window, a detection-output rule and a support window. +- Hook invocations cost ~105–110ms including process startup, of which ~20ms is + the native engine load. `DATAFOG_BACKEND=python` trades that back for coverage. +- `datafog[rust]` remains as an explicit spelling of a dependency the base + install already has. diff --git a/VERSIONING.md b/VERSIONING.md new file mode 100644 index 00000000..9f511c41 --- /dev/null +++ b/VERSIONING.md @@ -0,0 +1,123 @@ +# Versioning and compatibility policy + +DataFog Python follows [Semantic Versioning 2.0.0](https://semver.org/spec/v2.0.0.html). +This document defines what that means here, so a version number is a promise +rather than a guess. + +DataFog versions independently of `datafog-core`. See +[Independent versioning](#independent-versioning). + +## What the numbers mean + +- **MAJOR** (`5.0.0`) — a public API is removed or renamed, an import path moves, + or a default changes such that working caller code stops working. +- **MINOR** (`4.10.0`) — new capability. New entities, new backends, new extras, + new commands, new options, and **changes to detection output**. +- **PATCH** (`4.10.1`) — fixes that do not change findings for a given input, + configuration, and locked dependency set. + +### Detection output is a minor change, never a patch + +The finding set is part of the contract. A patch release must not start finding +more email addresses, or fewer, for input that already worked. If a fix changes +what is found, it ships as a MINOR and the changelog says which entities moved +and in which direction. People write golden files against this output. + +### Dependency-default changes are minor + +Raising a floor, dropping support for a Python version, or changing which +backend is used by default is a MINOR. It is never a PATCH. + +## Deprecation window + +A public API is deprecated in a MINOR, warning at the call boundary, and removed +no earlier than the second following MINOR or 90 days, whichever is longer. + +A deprecation warning must name: + +1. the exact last version that supports the surface, as an installable pin + (`pip install "datafog==4.10.0"`), and +2. the MAJOR that removes it. + +Warnings fire at the public call boundary, before any optional import, so a plain +`import datafog` never pays for a deprecated surface. + +At the removal MAJOR, the removed name stays present as a stub raising +`ImportError` with the same pin, rather than disappearing into an +`AttributeError`. + +## Experimental surface + +Experimental code is marked as such in its documentation and docstrings. It may +break in a MINOR without a deprecation cycle. Anything unmarked is covered by +this policy. + +Currently experimental: + +- `datafog.v5` (native Core preview) +- `datafog.core` (from 4.10.0) +- GLiNER2 and other model sessions +- Browser/WASM entry points + +## Scope + +Covered by this policy: + +- top-level `datafog` callables listed in `datafog.__all__` +- `datafog.compat.v4` and its result classes +- the `datafog` CLI and the `datafog-hook` entry point +- `DataFogGuardrail` and the Claude Code hook integration +- documented configuration keys and environment variables +- the finding set for a given engine, entity selection, and locale + +Not covered: + +- private modules and names beginning with `_` +- `main_lean.py`, `main_original.py`, `__init___original.py` and other + transitional duplicates +- logging text, warning text, and internal error strings +- model accuracy targets for optional ML engines + +## Support window + +The current MAJOR receives fixes for its latest two MINOR releases. Older MINORs +receive security fixes only. Every published version stays installable on PyPI +permanently; a retired surface has a final supported version documented in +[`docs/retirement.rst`](docs/retirement.rst), and that version is the answer. + +## Independent versioning + +`datafog` and `datafog-core` are separate distributions on separate tracks. +`datafog-core` carries the engine; `datafog` carries the application surface. +The transport between them is `capabilities()` and its `contract_version`. +Bumping either `contract_version` or `workspace_contract_version` is MAJOR work +for `datafog-core`, and requires a matching floor in `datafog`. + +Both are exposed separately so a bug report is attributable: + +```python +import datafog, datafog.core +datafog.__version__ # application version +datafog.core.__version__ # engine version +``` + +## Yanking + +A broken PATCH may be yanked on PyPI and replaced by the next PATCH. A broken +MINOR or MAJOR is superseded, not yanked, because dependents are often already +pinned to it; the changelog opens with the replacement version. + +## Enforcement + +- `RELEASE_NOTES_.md` is required for stable releases. +- The changelog classifies every entry as Added / Changed / Deprecated / Removed + / Fixed and names the version at which each deprecation ends. +- `scripts/check_rust_install.py` and the parity suite run against the installed + wheel, not the checkout. + +## History + +Before 4.10.0 this policy was not written down, and the versioning record shows +it: the 5.0 retirement plan was revised mid-stream (commit `5f5bdbb3`), and the +4.9.0 final publish preceded its own `a1` and `a2` tags. 4.10.0 is the first +release prepared under this policy. diff --git a/datafog/__about__.py b/datafog/__about__.py index 21d77306..897e6be2 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.9.0a2" +__version__ = "4.10.0" diff --git a/datafog/__init__.py b/datafog/__init__.py index 591e0ef4..12ecc115 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -11,6 +11,7 @@ import warnings +from ._legacy_retirement import LAST_SUPPORTED as _LAST_SUPPORTED from .__about__ import __version__ from .agent import create_guardrail, filter_output, sanitize, scan_prompt @@ -81,6 +82,13 @@ def __getattr__(name: str): elif name in _LAZY_EXPORTS: module_path, attr_name, extra_name = _LAZY_EXPORTS[name] + if attr_name is None: + # Submodule export (``datafog.native``): return the module itself. + import importlib + + value = importlib.import_module(module_path) + globals()[name] = value + return value try: module = __import__(module_path, fromlist=[attr_name]) value = getattr(module, attr_name) @@ -104,6 +112,7 @@ def _missing_dependency(*args, **kwargs): _LAZY_EXPORTS = { + "native": ("datafog.native", None, None), "app": ("datafog.client", "app", None), "DataFog": ("datafog.main", "DataFog", None), "TextPIIAnnotator": ("datafog.main", "TextPIIAnnotator", None), @@ -131,7 +140,8 @@ def _missing_dependency(*args, **kwargs): def _warn_v5_replacement(old_api: str, replacement: str) -> None: warnings.warn( f"datafog.{old_api}() is deprecated and will be removed in 5.0. Use {replacement} instead. " - "The earlier promise to retain this shim through 5.x has been revised.", + f'Last supported version is {_LAST_SUPPORTED}. Pin it with: pip install "datafog=={_LAST_SUPPORTED}". ' + "The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst.", FutureWarning, stacklevel=3, ) @@ -145,7 +155,7 @@ def scan( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> ScanResult: """Scan using the 4.x compatibility API; Rust detection is opt-in.""" return _compat_v4.scan( @@ -170,7 +180,7 @@ def redact( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> RedactResult: """Redact using the 4.x compatibility API and legacy strategies.""" return _compat_v4.redact( diff --git a/datafog/_core_capabilities.py b/datafog/_core_capabilities.py index a5a0df4b..77cd6c9d 100644 --- a/datafog/_core_capabilities.py +++ b/datafog/_core_capabilities.py @@ -85,7 +85,7 @@ def _requested_locales(locales: Any, lookup: dict[str, str]) -> set[str]: return {_resolve_locale(locale, lookup) for locale in locales or []} -def _activation_config(label: str, metadata: dict) -> dict: +def _activation_config(label: str, metadata: dict) -> dict | None: details = metadata.get(label) if not isinstance(details, dict): raise _invalid(f"missing entity metadata for {label!r}") @@ -95,10 +95,9 @@ def _activation_config(label: str, metadata: dict) -> dict: ): raise _invalid(f"invalid scopes for {label!r}") if "text" not in scopes: - raise ValueError( - f"Entity {label!r} is not available for Rust text scanning " - "(structured-only entity)" - ) + # Structured-only labels are served by the caller, usually from the + # embedded model, so a mixed selection is not an error. + return None activation = details.get("activation") if not isinstance(activation, dict): raise _invalid(f"missing activation metadata for {label!r}") @@ -120,13 +119,15 @@ def scan_configs(core: Any, requested: set[str], locales: Any) -> list[dict]: capabilities = getattr(core, "capabilities", None) if not callable(capabilities): raise _invalid( - "contract version 1 is required; install a compatible Core 0.4.x" + "contract version 1 is required; install a compatible datafog-core" ) supported, lookup, metadata = _read_capabilities(capabilities) selected_locales = _requested_locales(locales, lookup) common: dict[str, Any] = {} for label in sorted(requested & supported): config = _activation_config(label, metadata) + if config is None: + continue for key, value in config.items(): if key == "locale": selected_locales.add(_resolve_locale(value, lookup)) diff --git a/datafog/_legacy_retirement.py b/datafog/_legacy_retirement.py index 5bc96c13..e86506db 100644 --- a/datafog/_legacy_retirement.py +++ b/datafog/_legacy_retirement.py @@ -2,6 +2,9 @@ import warnings +#: Final release on which retired optional surfaces still work. +LAST_SUPPORTED = "4.10.0" + class LegacySurfaceWarning(FutureWarning): """An optional DataFog surface scheduled for removal in 5.0.""" @@ -11,7 +14,8 @@ def warn_legacy_surface(surface: str) -> None: """Warn at the caller's public API boundary, before optional imports.""" warnings.warn( f"DataFog {surface} support is deprecated in 4.9 and will be removed in 5.0. " - "Remain on the final 4.x release if you need continued support.", + f'Last supported version is {LAST_SUPPORTED}. Pin it with: pip install "datafog=={LAST_SUPPORTED}". ' + "See docs/retirement.rst.", LegacySurfaceWarning, stacklevel=3, ) diff --git a/datafog/compat/v4.py b/datafog/compat/v4.py index 0be6b4bb..49d32e2c 100644 --- a/datafog/compat/v4.py +++ b/datafog/compat/v4.py @@ -5,6 +5,7 @@ """ from ..engine import Entity, RedactResult, ScanResult +from ..engine import resolve_backend from ..engine import redact as _redact_entities from ..engine import scan as _scan from ..engine import scan_and_redact as _scan_and_redact @@ -29,13 +30,13 @@ def scan( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> ScanResult: """ - Scan with the legacy result schema; Rust detection is opt-in. + Scan with the legacy result schema. - Defaults to the lightweight regex engine so the core install works without - optional dependency fallback warnings. + Backend selection follows ``datafog.engine.resolve_backend``: native when + ``datafog-core`` is importable, otherwise Python with a one-time warning. ``allowlist`` exempts exact entity texts (your own support address, doc placeholders); ``allowlist_patterns`` exempts entities whose full text @@ -64,7 +65,7 @@ def redact( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> RedactResult: """ Redact with legacy strategies and result schema. @@ -75,8 +76,8 @@ def redact( full-text regex match respectively); they apply to the scan path and are rejected when explicit ``entities`` are supplied. """ - if backend not in ("python", "rust"): - raise ValueError("backend must be one of: python, rust") + # Validate the backend even when explicit entities skip the scan path. + resolve_backend(backend, engine) if preset is not None: try: diff --git a/datafog/engine.py b/datafog/engine.py index 0e2a0fb4..fa6ad926 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -3,13 +3,14 @@ from __future__ import annotations import hashlib +import os import re import warnings from dataclasses import dataclass from functools import lru_cache from typing import Optional -from .exceptions import EngineNotAvailable +from .exceptions import CoreUnavailableWarning, EngineNotAvailable from .processing.text_processing.regex_annotator import RegexAnnotator CANONICAL_TYPE_MAP = { @@ -27,6 +28,13 @@ # Presidio-compatible aliases, so configs migrate without renames. "EMAIL_ADDRESS": "EMAIL", "US_SSN": "SSN", + # Credential detectors are native-only; these spellings keep one vocabulary. + "APIKEY": "API_KEY", + "API-KEY": "API_KEY", + "TOKEN": "BEARER_TOKEN", + "BEARER": "BEARER_TOKEN", + "PRIVATEKEY": "PRIVATE_KEY", + "CREDENTIAL_URI": "CREDENTIAL_URI", } ALL_ENTITY_TYPES = { @@ -48,8 +56,28 @@ "ORGANIZATION", "LOCATION", "ADDRESS", + "API_KEY", + "BEARER_TOKEN", + "JWT", + "PRIVATE_KEY", + "CREDENTIAL_URI", + "ACCOUNT_NUMBER", + "STREET_ADDRESS", + "US_ROUTING_NUMBER", + "NPI", + "UUID", } +#: Labels the native rule engine reports as structured-only, served for text by +#: the embedded model instead. +MODEL_ENTITY_TYPES = {"PERSON", "ADDRESS"} + +#: Environment override for the default backend: ``auto``, ``python`` or ``rust``. +BACKEND_ENV = "DATAFOG_BACKEND" + +_core_warned = False +_cnn_cache: tuple[object, object] | None = None + NER_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} ENTITY_TYPE_PRIORITY = { @@ -209,6 +237,81 @@ def _regex_entities( return _suppress_overlapping_entities(entities) +def _core_module_or_none() -> object | None: + """Return the importable ``datafog_core`` module, or ``None``. + + Deliberately uncached: after the first import this is a ``sys.modules`` + lookup, and not caching keeps ``sys.modules`` patching in tests honest. + """ + try: + import datafog_core + except ImportError: + return None + return datafog_core + + +def _require_core() -> object: + core = _core_module_or_none() + if core is None: + raise ImportError( + "The Rust backend requires datafog-core. " + 'Install with: pip install "datafog[rust]"' + ) + return core + + +def core_available() -> bool: + """Whether the native engine can serve this interpreter.""" + return _core_module_or_none() is not None + + +def _set_core(module: object | None) -> None: + """Test seam: reset cached state derived from the core module.""" + global _cnn_cache + _cnn_cache = None + + +def _cnn_detector_or_none() -> object | None: + """Cached embedded detector, keyed on the module object it came from.""" + global _cnn_cache + core = _core_module_or_none() + if core is None: + return None + if _cnn_cache is not None and _cnn_cache[0] is core: + return _cnn_cache[1] + factory = getattr(core, "EmbeddedCnnDetector", None) + if factory is None: + return None + detector = factory() + _cnn_cache = (core, detector) + return detector + + +def _cnn_findings( + detector: object, requested: set[str], text: str +) -> list[tuple[dict, object]]: + """Serve prose names and address blocks from the embedded model.""" + try: + result = detector.scan_model(text) + except Exception as exc: # relative-path/limit errors surface to the caller + raise _model_error(exc) from exc + return [ + ({}, finding) + for finding in result.findings + if finding.entity_type in requested + ] + + +def _model_error(exc: Exception) -> Exception: + """Re-raise a model failure as a ValueError with its native code attached.""" + code = getattr(exc, "code", "model_error") + reason = getattr(exc, "reason", None) + detail = f"{code}/{reason}" if reason else code + error = ValueError(f"Native model detection failed ({detail}): {exc}") + error.code = code # type: ignore[attr-defined] + return error + + def _rust_entities( text: str, entity_types: Optional[list[str]] = None, @@ -220,22 +323,30 @@ def _rust_entities( suppression before entity selection and allowlists, preserving that order without duplicating the native detectors. """ - try: - import datafog_core - except ImportError as exc: - raise ImportError( - "The Rust backend requires datafog-core. " - 'Install with: pip install "datafog[rust]"' - ) from exc + datafog_core = _require_core() from ._core_capabilities import scan_configs requested = {_canonical_type(value) for value in entity_types or []} - configs = scan_configs(datafog_core, requested, locales) + model_requested = requested & MODEL_ENTITY_TYPES + detector = _cnn_detector_or_none() if model_requested else None + if model_requested and detector is None: + raise ValueError( + f"Entity {', '.join(sorted(model_requested))} needs the embedded model, " + "which this datafog-core build does not provide. Upgrade it with: " + 'pip install --upgrade "datafog-core"' + ) + configs = scan_configs(datafog_core, requested - MODEL_ENTITY_TYPES, locales) entities: list[Entity] = [] seen: set[tuple[str, int, int]] = set() - for config in configs: - for finding in datafog_core.scan(text, config=config): + findings = [ + (config, finding) + for config in configs + for finding in datafog_core.scan(text, config=config) + ] + if detector is not None: + findings.extend(_cnn_findings(detector, model_requested, text)) + for config, finding in findings: canonical_type = _canonical_type(finding.entity_type) start = finding.codepoint_range.start end = finding.codepoint_range.end @@ -404,6 +515,42 @@ def _needs_ner(entity_types: Optional[list[str]]) -> bool: return bool(requested & NER_ENTITY_TYPES) +def _warn_core_missing() -> None: + global _core_warned + if _core_warned: + return + _core_warned = True + warnings.warn( + "datafog-core is not installed, so detection falls back to the pure-Python " + f"engine. Install it to use the native engine, or set {BACKEND_ENV}=python " + "to silence this warning.", + CoreUnavailableWarning, + stacklevel=4, + ) + + +def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: + """Resolve ``auto`` to a concrete backend without importing Core eagerly. + + An explicit ``backend`` wins over ``DATAFOG_BACKEND``. ``auto`` uses the + native engine whenever Core is importable and degrades to Python otherwise. + Non-``regex`` engines always take the Python path, because the native + backend serves ``regex`` and model findings only. + """ + requested = backend or os.environ.get(BACKEND_ENV) or "auto" + normalized = requested.strip().lower() + if normalized not in {"auto", "python", "rust"}: + raise ValueError( + f"backend must be one of: auto, python, rust (from {BACKEND_ENV} or the " + "backend argument)" + ) + if normalized == "auto": + if engine != "regex": + return "python" + return "rust" if core_available() else "python" + return normalized + + def _validate_backend(backend: str, engine: str) -> None: if backend not in {"python", "rust"}: raise ValueError("backend must be one of: python, rust") @@ -419,13 +566,19 @@ def scan( allowlist: Optional[list[str]] = None, allowlist_patterns: Optional[list[str]] = None, *, - backend: str = "python", + backend: Optional[str] = None, ) -> ScanResult: """Scan text for PII entities. - ``backend="rust"`` opts into experimental native detection for - ``engine="regex"``. Results retain legacy regex provenance and confidence; - these are compatibility values, not native confidence estimates. + ``backend`` defaults to ``auto``: the native engine when ``datafog-core`` is + importable, otherwise the pure-Python engine with a one-time warning. Set + ``DATAFOG_BACKEND`` or pass ``backend="python"`` to force the fallback. The + native path serves ``engine="regex"``; non-regex engines always use Python. + Probe names and address blocks come from the embedded model on the native + path, so ``PERSON`` and ``ADDRESS`` work without spaCy or a model download. + + Results retain legacy regex provenance and confidence; these are + compatibility values, not native confidence estimates. ``allowlist`` exempts exact entity texts (e.g. your own support email); ``allowlist_patterns`` exempts entities whose full text matches a regex @@ -437,7 +590,11 @@ def scan( if engine not in {"regex", "spacy", "gliner", "smart"}: raise ValueError("engine must be one of: regex, spacy, gliner, smart") + requested_backend = (backend or os.environ.get(BACKEND_ENV) or "auto").strip().lower() + backend = resolve_backend(backend, engine) _validate_backend(backend, engine) + if backend == "python" and requested_backend == "auto" and not core_available(): + _warn_core_missing() # Validate patterns up front so config errors fail fast even when the # text contains no entities. @@ -611,9 +768,12 @@ def scan_and_redact( allowlist: Optional[list[str]] = None, allowlist_patterns: Optional[list[str]] = None, *, - backend: str = "python", + backend: Optional[str] = None, ) -> RedactResult: - """Scan with the selected backend, then apply legacy Python redaction.""" + """Scan with the selected backend, then apply legacy Python redaction. + + Backend selection matches :func:`scan`. + """ scan_result = scan( text=text, engine=engine, diff --git a/datafog/exceptions.py b/datafog/exceptions.py index 98bc8d0d..7ac32340 100644 --- a/datafog/exceptions.py +++ b/datafog/exceptions.py @@ -70,6 +70,10 @@ def __init__(self, message: str): super().__init__(message, status_code=None) +class CoreUnavailableWarning(RuntimeWarning): + """Emitted once when ``backend="auto"`` falls back to the pure-Python engine.""" + + def raise_for_status_code(status_code: int, error_message: str): """ Raise the appropriate exception based on the status code. diff --git a/datafog/native.py b/datafog/native.py new file mode 100644 index 00000000..cde64934 --- /dev/null +++ b/datafog/native.py @@ -0,0 +1,60 @@ +"""Native detection API, re-exported from the installed ``datafog-core``. + +``datafog`` selects the native engine by default, so most callers never need +this module. Import it to reach the native schema directly: byte and code-point +ranges, transformation records, provider-backed tokenization, structured scans, +the embedded model, and ``capabilities()``. + +The names are re-exports, not a second implementation. Importing ``datafog`` +alone does not load them. + +Experimental: this module may change outside the deprecation policy until 5.0. +""" + +from __future__ import annotations + +import importlib +import importlib.metadata +from typing import Any + +_MODULE = "datafog_core" + +__all__ = [ + "EmbeddedCnnDetector", + "Finding", + "ModelScanResult", + "capabilities", + "contract_version", + "scan", + "scan_and_transform", + "scan_structured", + "transform", + "transform_structured", + "version", +] + + +def _core() -> Any: + try: + return importlib.import_module(_MODULE) + except ImportError as exc: # pragma: no cover - base install always has it + raise ImportError( + "datafog.native requires datafog-core. Install with: " + 'pip install "datafog==4.10.0"' + ) from exc + + +def __getattr__(name: str) -> Any: + if name in __all__: + return getattr(_core(), name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def version() -> str: + """Installed Core version, for bug reports and reproducibility locks.""" + return importlib.metadata.version("datafog-core") + + +def contract_version() -> int: + """Capability contract the installed Core speaks. Currently 1.""" + return int(_core().capabilities()["contract_version"]) diff --git a/docs/cli.rst b/docs/cli.rst index b07a080a..b704508b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -24,7 +24,8 @@ dependencies are: Spark/distributed workflows are Python SDK surfaces rather than first-path CLI commands. Spark support is also deprecated in 4.9 for removal in 5.0. Install ``datafog[distributed]`` when using ``SparkService`` during 4.x. Users needing -OCR/Spark after the cutover can remain on the final 4.x release. See +OCR/Spark after the cutover should pin ``datafog==4.10.0``, the last supported +version. See :doc:`retirement` and :doc:`optional-surfaces` and the :download:`4.9 migration guide `. diff --git a/docs/important-concepts.rst b/docs/important-concepts.rst index 239471bc..4f48c39d 100644 --- a/docs/important-concepts.rst +++ b/docs/important-concepts.rst @@ -39,7 +39,7 @@ Services ``TextService`` remains available. ``ImageService`` and ``SparkService`` are optional legacy services, deprecated in 4.9 for removal in 5.0. They remain functional throughout 4.9; users requiring them after the cutover can remain on -the final 4.x release. See :doc:`optional-surfaces`. +:doc:`retirement`. Existing services: diff --git a/docs/index.rst b/docs/index.rst index df61a539..bb9b3d9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,13 +11,30 @@ Start with :doc:`getting-started` if you want the shortest route from install to scanning text. The roadmap and historical planning pages remain available, but the live user docs are the first path for current text APIs. +4.10.0 native default +===================== + +DataFog 4.10.0 turns the native engine on by default. ``pip install datafog`` +now installs ``datafog-core``, and detection uses it whenever it imports; the +pure-Python engine remains the fallback and ``DATAFOG_BACKEND=python`` forces +it. No public API is removed in 4.10.0 and no existing call changes shape — the +``backend`` keyword is still accepted, and now also accepts ``auto``. + +:download:`Complete 4.10 migration guide `. + +Prose ``PERSON`` and ``ADDRESS`` detection now comes from an embedded model, so +names and addresses no longer need spaCy or a model download. The native engine +adds credential, account and German detectors. Reviewed differences from the +Python engine are recorded in ``tests/contracts/rust-1.2.0.json`` and the +migration guide below. + 4.9.0 migration bridge ====================== -DataFog 4.9.0 is the migration bridge to the planned Rust-backed 5.0 API. -Upgrade the base package with ``python -m pip install --upgrade datafog==4.9.0``; -the native dependency remains optional. The ``rust`` extra requires -``datafog-core>=0.4.0,<0.5`` and capability contract version 1. +DataFog 4.9.0 was the migration bridge to the Rust-backed 5.0 API. It added the +explicit ``backend="rust"`` opt-in, the ``datafog.compat.v4`` facade and the +``datafog.v5`` preview. 4.10.0 built on it by making the native path the +default. 4.9 preserves existing imports, result classes, Python detection defaults, and redaction behavior. It adds explicit experimental Rust detection through @@ -28,7 +45,7 @@ Core schema preview. See :doc:`python-sdk` and the DataFog 4.9.0 deprecates ``detect()``/``process()``, OCR, and Spark for removal in 5.0. The earlier promise to retain ``detect()``/``process()`` throughout 5.x is revised. OCR/Spark remain functional in 4.9; users needing them after the -cutover can remain on the final 4.x release. See :doc:`optional-surfaces`. +cutover can pin :doc:`retirement`. Use DataFog =========== @@ -42,6 +59,7 @@ Use DataFog cli optional-surfaces important-concepts + retirement Reference ========= diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md new file mode 100644 index 00000000..cbec85de --- /dev/null +++ b/docs/migration-4.10.md @@ -0,0 +1,92 @@ +# Migrating to DataFog 4.10.0 + +4.10.0 turns on the native engine. Nothing is removed; most callers change +nothing at all. This page covers what a version number cannot: what you will +notice, and what to do if you do not want it. + +```bash +python -m pip install --upgrade "datafog==4.10.0" +``` + +`datafog-core` is now a normal dependency, so this pulls a platform wheel. On +Linux (glibc or musl, x86_64/aarch64/s390x/ppc64le), macOS and Windows +(amd64/arm64) that is a wheel. Elsewhere it builds from source and needs Rust +1.88+. If Core is missing at runtime, detection falls back to the Python engine +and warns once instead of failing. + +## What changes + +`backend` defaults to `auto`. `auto` selects the native engine when Core +imports, and the Python engine otherwise. `backend="python"` and +`backend="rust"` work exactly as before. `DATAFOG_BACKEND` sets the default +without touching code: + +```bash +DATAFOG_BACKEND=python # always the pure-Python engine, no warning +DATAFOG_BACKEND=auto # the default +``` + +`auto` never picks the native engine for `engine="spacy"`, `"gliner"` or +`"smart"`, because the native path serves `engine="regex"` plus embedded-model +findings. Those engines still run on the Python ML stack and still need their +extras. + +## Detection differences + +The native engine finds things the Python regex engine never did: `API_KEY`, +`BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, +`US_ROUTING_NUMBER`, `NPI`, `UUID`, and the German identifier set behind the +`de` locale. Credential detectors ship enabled by default; German and `UUID` +stay opt-in. + +These differences go the other way and are recorded, not hidden, in +[tests/contracts/rust-1.2.0.json](../tests/contracts/rust-1.2.0.json): + +| Input | Python | Native | Why | +| --- | --- | --- | --- | +| `4111 1111 1111 1112` | `CREDIT_CARD` | none | Core validates the Luhn checksum | +| `x123-45-6789y` | `SSN` | none | Core requires non-alphanumeric boundaries | +| `Invoice 1234567890` | `PHONE` | none | Core requires an area code starting 2-9, which is what keeps Unix timestamps out of `PHONE` | +| `unsupported` locale | locale ignored | `ValueError` | Core validates locales against its inventory | + +`PERSON` and `ADDRESS` used to need spaCy or GLiNER through `[nlp]` and +`[nlp-advanced]`. They now come from an embedded model on the native path, with +no download and no interpreter. `PERSON` for a text scan on `backend="python"` +is still Python-engine territory: the pure-Python regex engine has no name +detection, so select `PERSON` without forcing `backend="python"`. + +## If the output must not move + +Pin the backend, not just the version: + +```python +import datafog + +datafog.scan(text, backend="python") # exact 4.9.0 output +``` + +Or pin the whole package: `pip install "datafog==4.9.0"`. + +Detection output is part of the contract. Per +[VERSIONING.md](../VERSIONING.md), a change to the finding set is a MINOR, never +a PATCH, so a patch release will not move your golden files. + +## Reproducibility + +Core versions independently. If output must be byte-identical across machines, +lock it: + +```bash +pip install "datafog==4.10.0" "datafog-core==1.2.0" +``` + +`datafog.native.version()` reports the installed engine version and +`datafog.native.contract_version()` the contract it speaks, for bug reports. +Core follows its own [versioning policy](https://github.com/DataFog/datafog-core/blob/main/VERSIONING.md); +a contract bump is a Core MAJOR and raises the floor here. + +## What is not changing yet + +`detect()`, `process()`, OCR and Spark all still work and still warn. They are +removed in 5.0, not here. The last supported version is 4.10.0 — see the +[retirement schedule](retirement.rst). diff --git a/docs/migration-4.9.md b/docs/migration-4.9.md index 3bf2b900..4e06e3fe 100644 --- a/docs/migration-4.9.md +++ b/docs/migration-4.9.md @@ -179,8 +179,9 @@ They are now scheduled for removal in 5.0. Use `scan` or `redact` in 4.9 and eva the native preview before upgrading to 5.0. `process(anonymize=True)` used older placeholder/hash semantics; migrating to `redact` can intentionally change output. -Users needing OCR/Spark can remain on the final 4.x release. No successor package -or indefinite support commitment is introduced here. Deprecation notices do not +Users needing OCR/Spark can pin `pip install "datafog==4.10.0"`, the last +supported version. No successor package is introduced here. See +[the retirement schedule](retirement.md). Deprecation notices do not trigger downloads or import heavy dependencies during ordinary text imports. ## Verification and performance diff --git a/docs/optional-surfaces.rst b/docs/optional-surfaces.rst index 96d1494d..006bec49 100644 --- a/docs/optional-surfaces.rst +++ b/docs/optional-surfaces.rst @@ -18,7 +18,8 @@ The default path is: OCR and Spark are deprecated optional surfaces in 4.9 and will be removed in 5.0. Their APIs, installation extras, and existing behavior remain available throughout -4.9. Remain on the final 4.x release if you need continued OCR or Spark support. +4.9. The last supported version is 4.10.0 — pin it with ``pip install +"datafog==4.10.0"``. See :doc:`retirement`. No replacement package is introduced by this migration. Use sites emit ``FutureWarning`` notices, visible with Python's default warning diff --git a/docs/python-sdk.rst b/docs/python-sdk.rst index 2a8a2ddf..98781f96 100644 --- a/docs/python-sdk.rst +++ b/docs/python-sdk.rst @@ -27,10 +27,11 @@ available for existing users. ``TextService(engine="regex")`` is the dependency-light service path; ``spacy``, ``gliner``, ``smart``, OCR, and Spark surfaces require their explicit extras. -4.9.0 compatibility and Core preview ------------------------------------- +4.10.0 native default and Core preview +-------------------------------------- -DataFog 4.9.0 provides an experimental Rust backend and native schema preview. +DataFog 4.10.0 uses the native engine by default and keeps the native schema +preview at ``datafog.v5`` and ``datafog.native``. Existing top-level ``scan``/``redact`` functions retain their result shapes and use the Python backend by default. They delegate through the new facade, whose classes are the same objects as the established result types: @@ -44,9 +45,9 @@ continue to work at the top level in 4.9 but warn of removal in 5.0, revising th previous promise to retain them throughout 5.x. Moving from ``process`` to ``redact`` can change old placeholder and hash output; compare results explicitly. -The adapter requires ``datafog-core>=0.4.0,<0.5``. Install with -``python -m pip install --upgrade "datafog[rust]==4.9.0"`` to evaluate Rust. -A base install of ``datafog==4.9.0`` has no native dependency; installing the +The adapter requires ``datafog-core>=1.2.0,<2``, which the base install pulls. Install with +``datafog-core`` is installed by every ``datafog`` install; ``backend="python"`` +or ``DATAFOG_BACKEND=python`` forces the pure-Python engine. Installing the extra does not change the default Python backend: .. code-block:: python @@ -89,7 +90,7 @@ a separate decision. The adapter requires capability contract version 1 and discovers supported labels, locales, and activation settings from the installed Core. Future finding labels -are preserved. Core 0.4.x updates may change detection output; lock the version +are preserved. Core 1.x updates may change detection output; lock the version when reproducibility is required. German aliases ``de``, ``de-DE``, and ``de_DE`` activate German detectors; ``en-US`` and ``fr`` activate only base detectors. Locale validation trims ASCII whitespace and ignores ASCII case; unsupported @@ -101,7 +102,7 @@ through Core metadata. Core's ``PERSON`` is structured-only and explicitly selecting it for Rust text scanning raises an error. The Python default backend keeps its existing behavior. -Core 0.4.0 adds JWT, private-key, contextual routing-number, and contextual NPI +Core 1.2 adds the same JWT, private-key, contextual routing-number and NPI findings. The legacy overlap policy can prefer ``PHONE`` over a same-span ``NPI``; filtering for NPI then returns no entities. Native ``datafog.v5.scan`` retains NPI, and native transformation can select it. Legacy result and transformation @@ -158,7 +159,8 @@ OCR and Spark remain available as optional surfaces throughout 4.9: DataFog 4.9.0 deprecates OCR and Spark for removal in 5.0. Use-time ``FutureWarning`` notices are visible under normal Python warning filters. Users -needing these features can remain on the final 4.x release; this migration does +needing these features can pin ``datafog==4.10.0``, the last supported version; +this migration does not introduce successor packages or promise indefinite maintenance. See :doc:`optional-surfaces` for install notes and limitations. diff --git a/docs/retirement.rst b/docs/retirement.rst new file mode 100644 index 00000000..17c5c0af --- /dev/null +++ b/docs/retirement.rst @@ -0,0 +1,115 @@ +=================== +Retirement schedule +=================== + +This page is the standing record of surfaces DataFog is removing. It is updated +once per release, and a row is added in the MINOR that deprecates the surface — +never retroactively. + +Ordering and removal rules are defined in +`VERSIONING.md `__. + +Removed or scheduled +==================== + +.. list-table:: + :header-rows: 1 + + * - Surface + - Last supported version + - Replacement + - Removed in + - Deprecated in + * - ``OCRMService`` + - ``4.10.0`` + - none + - ``5.0.0`` + - ``4.9.0`` + * - ``DonutProcessor`` + - ``4.10.0`` + - none + - ``5.0.0`` + - ``4.9.0`` + * - ``[ocr]`` extra (Donut, Tesseract, Pillow) + - ``4.10.0`` + - none + - ``5.0.0`` + - ``4.9.0`` + * - ``[distributed]`` extra (PySpark, pandas) + - ``4.10.0`` + - :ref:`spark-recipe` + - ``5.0.0`` + - ``4.9.0`` + * - ``detect()`` + - ``4.10.0`` + - ``scan()`` + - ``5.0.0`` + - ``4.9.0`` + * - ``process()`` + - ``4.10.0`` + - ``redact()`` + - ``5.0.0`` + - ``4.9.0`` + +Installing ``pip install "datafog==4.10.0"`` keeps every row above working. +That version stays installable on PyPI permanently. + +OCR +=== + +There is no successor package and no 5.x reimplementation. OCR quality was never +the reason anyone adopted DataFog, and a mediocre replacement is worse than a +pin. + +.. code-block:: bash + + pip install "datafog==4.10.0[ocr]" + +If you need continued OCR support beyond that, stay on ``4.10.0``. Track the +surfaces you depend on by name, not by staying on a major version range, so a +lock file review shows the pin. + +.. _spark-recipe: + +Spark +===== + +No distributed API ships in 5.0. Partition the dataframe and call ``scan`` per +partition: one Spark job with a ``mapInPandas`` step, or Dask's +``map_partitions``. Both run the local engine and need no plugin. + +.. code-block:: python + + # Spark + def scan_partition(rows): + for pdf in rows: + pdf["findings"] = pdf["text"].map(lambda t: len(datafog.scan(t))) + yield pdf + + df.mapInPandas(scan_partition, schema="text string, findings long") + +.. code-block:: python + + # Dask + ddf["findings"] = ddf["text"].map_partitions( + lambda s: s.map(lambda t: len(datafog.scan(t))) + ) + +Per-partition ``scan`` is a pure local call with no network and no shared state, +so worker fan-out is just normal framework behaviour. What you lose versus +``SparkService`` is the UDF packaging, not correctness. + +detect and process +================== + +``detect()`` and ``process()`` were the pre-4.0 entry points. +``process(anonymize=True)`` used older placeholder and hash semantics, so +migrating to ``redact()`` can intentionally change output. Read the output +before you commit the change. + +DataFog Core +============ + +``datafog-core`` retires surfaces on its own schedule, documented in that +repository. The Python surface here inherits Core's ``contract_version``; a +contract bump is a Core MAJOR and raises the floor in ``datafog`` accordingly. diff --git a/scripts/check_rust_install.py b/scripts/check_rust_install.py index 26120232..bf84c45d 100644 --- a/scripts/check_rust_install.py +++ b/scripts/check_rust_install.py @@ -1,4 +1,4 @@ -"""Smoke-test installed 4.9 bridge wheels with python -I, outside checkout imports.""" +"""Smoke-test an installed wheel with python -I, outside checkout imports.""" import importlib.metadata import sys @@ -13,18 +13,20 @@ def main(): from datafog import v5 from datafog.compat import v4 - assert importlib.metadata.version("datafog-core").startswith("0.4.") + installed = importlib.metadata.version("datafog-core") + assert installed.split(".")[0] == "1", installed assert datafog_core.capabilities()["contract_version"] == 1 assert v4.Entity is datafog.Entity assert v5.Finding is datafog_core.Finding text = "👋 Contact alice@example.com" - legacy = datafog.scan(text, backend="rust") + # No explicit backend: the native engine must be the resolved default. + legacy = datafog.scan(text) assert isinstance(legacy, datafog.ScanResult) assert legacy.entities[0].text == "alice@example.com" assert ( text[legacy.entities[0].start : legacy.entities[0].end] == "alice@example.com" ) - assert datafog.redact(text, backend="rust").redacted_text == "👋 Contact [EMAIL_1]" + assert datafog.redact(text).redacted_text == "👋 Contact [EMAIL_1]" native = v5.scan(text) assert isinstance(native[0], datafog_core.Finding) assert ( diff --git a/setup.py b/setup.py index 72cb2da3..4f012b22 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,9 @@ # Core lightweight dependencies only core_deps = [ + # Native engine, the default detection path. Exact upper bound: detection + # output is contract-versioned and must stay reproducible. + "datafog-core>=1.2.0,<2", "pydantic>=2.0,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", @@ -84,7 +87,8 @@ ] extras_require = { - "rust": ["datafog-core>=0.4.0,<0.5"], + # Retained for explicitness; the base install already pulls Core. + "rust": ["datafog-core>=1.2.0,<2"], "nlp": nlp_deps, "nlp-advanced": nlp_advanced_deps, "ocr": ocr_deps, @@ -128,7 +132,7 @@ ], }, classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", diff --git a/tests/contracts/rust-0.4.0.json b/tests/contracts/rust-1.2.0.json similarity index 70% rename from tests/contracts/rust-0.4.0.json rename to tests/contracts/rust-1.2.0.json index 541cc3a3..6f46c6b8 100644 --- a/tests/contracts/rust-0.4.0.json +++ b/tests/contracts/rust-1.2.0.json @@ -1,5 +1,5 @@ { - "core_version": "0.4.0", + "core_version": "1.2.0", "cases": { "scan-observation-invalid-card": { "classification": "detector-difference", @@ -41,6 +41,21 @@ }, "warnings": [] } + }, + "scan-observation-ambiguous-number": { + "classification": "detector-difference", + "reason": "Core's North American area-code check requires a leading digit of 2-9, so a bare 10-digit run starting with 1 stays undetected. This is the deliberate Unix-timestamp protection; Python 4.8.1 matches it.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Invoice 1234567890", + "engine_used": "regex" + } + }, + "warnings": [] + } } } } diff --git a/tests/rust_contract.py b/tests/rust_contract.py index 3b477231..85d0d6a9 100644 --- a/tests/rust_contract.py +++ b/tests/rust_contract.py @@ -9,7 +9,7 @@ from tests.contract_481 import FIXTURE, observe -DEVIATIONS = Path(__file__).parent / "contracts" / "rust-0.4.0.json" +DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.2.0.json" TARGETS = { "datafog:scan", "datafog:redact", diff --git a/tests/test_allowlist.py b/tests/test_allowlist.py index 1f3bef6e..0bd85aec 100644 --- a/tests/test_allowlist.py +++ b/tests/test_allowlist.py @@ -10,7 +10,8 @@ EMAIL = "jane.doe@" "example.com" OTHER_EMAIL = "sid@" "example.com" -TIMESTAMP_LIKE = "17830" "25668" # ten digits: matches the PHONE pattern +TIMESTAMP_LIKE = "44123" "45678" # ten digits, NANP-shaped: both backends match +LEADING_ONE_TIMESTAMP = "17830" "25668" # native reads the leading 1 as a country code class TestExactAllowlist: @@ -54,6 +55,17 @@ def test_pattern_suppresses_matching_entities(self): ) assert result.entities == [] + def test_leading_one_timestamp_is_a_documented_native_delta(self): + # The native NANP check reads a leading 1 as a country code and needs a + # nine-digit national number after it, so this misses; the Python engine + # matches. Recorded rather than silently accepted. + assert datafog.scan(f"created {LEADING_ONE_TIMESTAMP}", engine="regex").entities == [] + assert len( + datafog.scan( + f"created {LEADING_ONE_TIMESTAMP}", engine="regex", backend="python" + ).entities + ) == 1 + def test_pattern_matches_full_entity_text_only(self): result = datafog.scan( f"mail {EMAIL}", engine="regex", allowlist_patterns=[r"^jane\."] diff --git a/tests/test_api_bridge_49.py b/tests/test_api_bridge_49.py index 483aa7f8..d47ff561 100644 --- a/tests/test_api_bridge_49.py +++ b/tests/test_api_bridge_49.py @@ -32,7 +32,7 @@ def test_existing_positional_calls_match_compatibility_facade(): def test_backend_is_additive_keyword_only(function): parameter = inspect.signature(function).parameters["backend"] assert parameter.kind is inspect.Parameter.KEYWORD_ONLY - assert parameter.default == "python" + assert parameter.default is None @pytest.mark.parametrize("name", ["scan", "redact"]) diff --git a/tests/test_backend_resolution.py b/tests/test_backend_resolution.py new file mode 100644 index 00000000..729e1a54 --- /dev/null +++ b/tests/test_backend_resolution.py @@ -0,0 +1,111 @@ +"""Backend resolution: native by default, Python fallback, env override.""" + +import builtins + +import pytest + +import datafog +from datafog import engine +from datafog.exceptions import CoreUnavailableWarning + + +@pytest.fixture +def without_core(monkeypatch): + """Hide ``datafog_core`` from the import machinery.""" + original_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + if name == "datafog_core" or name.startswith("datafog_core."): + raise ImportError("hidden for test") + return original_import(name, *args, **kwargs) + + monkeypatch.delitem(__import__("sys").modules, "datafog_core", raising=False) + monkeypatch.setattr(builtins, "__import__", guarded_import) + monkeypatch.setattr(engine, "_cnn_cache", None) + + +def test_auto_resolves_to_native_when_core_is_installed(): + pytest.importorskip("datafog_core") + assert engine.resolve_backend(None, "regex") == "rust" + + +def test_auto_resolves_to_python_without_core(without_core): + assert engine.resolve_backend(None, "regex") == "python" + + +def test_auto_keeps_non_regex_engines_on_python(): + pytest.importorskip("datafog_core") + for engine_name in ("smart", "spacy", "gliner"): + assert engine.resolve_backend(None, engine_name) == "python" + + +def test_python_fallback_warns_once_then_stays_quiet(without_core): + with pytest.warns(CoreUnavailableWarning) as caught: + datafog.scan("mail jane@example.com") + datafog.scan("mail jane@example.com") + assert len(caught) == 1 + + +def test_explicit_python_does_not_warn(): + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("error", CoreUnavailableWarning) + assert datafog.scan("mail jane@example.com", backend="python").entities + + +def test_env_override_selects_the_python_engine(monkeypatch): + pytest.importorskip("datafog_core") + monkeypatch.setenv(engine.BACKEND_ENV, "python") + assert engine.resolve_backend(None, "regex") == "python" + + +def test_explicit_backend_beats_the_env(monkeypatch): + pytest.importorskip("datafog_core") + monkeypatch.setenv(engine.BACKEND_ENV, "python") + assert engine.resolve_backend("rust", "regex") == "rust" + + +def test_env_is_case_and_space_insensitive(monkeypatch): + pytest.importorskip("datafog_core") + monkeypatch.setenv(engine.BACKEND_ENV, " AUTO ") + assert engine.resolve_backend(None, "regex") == "rust" + + +def test_unknown_env_value_rejected(monkeypatch): + monkeypatch.setenv(engine.BACKEND_ENV, "gpu") + with pytest.raises(ValueError, match="backend must be one of"): + engine.scan("text", "regex") + + +def test_explicit_rust_without_core_still_raises(without_core): + with pytest.raises(ImportError, match="datafog\\[rust\\]"): + datafog.scan("mail jane@example.com", backend="rust") + + +def test_person_selection_without_model_raises_a_pin(monkeypatch): + from types import SimpleNamespace + + meta = { + "contract_version": 1, + "supported_entities": ["EMAIL", "PERSON"], + "default_entities": ["EMAIL"], + "locales": {"en-US": {"enabled_entities": []}}, + "entities": { + "EMAIL": {"scopes": ["structured", "text"], "activation": {"kind": "default"}}, + "PERSON": {"scopes": ["structured"], "activation": {"kind": "default"}}, + }, + } + stub = SimpleNamespace(capabilities=lambda: meta, scan=lambda text, config=None: []) + monkeypatch.setitem(__import__("sys").modules, "datafog_core", stub) + monkeypatch.setattr(engine, "_cnn_cache", None) + with pytest.raises(ValueError, match="embedded model"): + datafog.scan("Jane Doe", entity_types=["PERSON"], backend="rust") + + +def test_native_findings_are_classified_as_regex_provenance(): + pytest.importorskip("datafog_core") + result = datafog.scan("mail jane@example.com") + assert result.engine_used == "regex" + assert result.entities[0].engine == "regex" + assert result.entities[0].confidence == 1.0 diff --git a/tests/test_contract_481.py b/tests/test_contract_481.py index 5aa5b767..95158b07 100644 --- a/tests/test_contract_481.py +++ b/tests/test_contract_481.py @@ -6,13 +6,20 @@ import pytest from tests.contract_481 import FIXTURE, observe +from tests.rust_contract import DEVIATIONS CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) +# The default backend is native from 4.10.0, so the reviewed native +# differences are the published 4.10 observations for those cases. +REVIEWED = json.loads(DEVIATIONS.read_text(encoding="utf-8")) def expected_49(case): - """Allow only the four added backend parameters and revised shim notices.""" - expected = copy.deepcopy(case["expected"]) + """Allow only the added backend parameter, revised shim notices and the + reviewed native differences.""" + + deviation = REVIEWED["cases"].get(case["id"]) + expected = copy.deepcopy(deviation["expected"]) if deviation else copy.deepcopy(case["expected"]) if case["id"] == "public-signatures": for target in ( "datafog:scan", @@ -25,7 +32,7 @@ def expected_49(case): "name": "backend", "kind": "KEYWORD_ONLY", "required": False, - "default": "python", + "default": None, } ) if case.get("target") in {"datafog:detect", "datafog:process"}: @@ -41,7 +48,9 @@ def expected_49(case): "message": ( f"datafog.{name}() is deprecated and will be removed in 5.0. " f"Use {replacement} instead. " - "The earlier promise to retain this shim through 5.x has been revised." + 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0". ' + "The earlier promise to retain this shim through 5.x has been revised. " + "See docs/retirement.rst." ), } ] diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index 272060af..f767fd0b 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -50,7 +50,7 @@ def test_supported_wheel_version_and_capability_contract(): version = importlib.metadata.version("datafog-core") - assert version.split(".")[:2] == ["0", "4"], version + assert version.split(".")[:1] == ["1"], version capabilities = core.capabilities() assert capabilities["contract_version"] == 1 supported = capabilities["supported_entities"] @@ -174,12 +174,16 @@ def test_contextual_identifiers_do_not_activate_without_context(label, raw): assert not result.entities -def test_npi_retains_native_identity_but_legacy_overlap_prefers_phone(): - """4.x suppresses overlaps before selection; this is a reviewed difference.""" +def test_contextual_npi_survives_the_legacy_overlap_policy(): + """A leading-1 national number is not PHONE, so NPI is no longer suppressed. + + Core 1.2 accepts exchange codes starting with 1 but keeps the area-code + 2-9 check, so `1234567893` is NPI only. Under Core 0.4 the same span was + ambiguous and the legacy overlap policy dropped the NPI. + """ text = "😀 café NPI 1234567893" native = v5.scan(text) assert [(item.entity_type, item.matched_text) for item in native] == [ - ("PHONE", "1234567893"), ("NPI", "1234567893"), ] assert { @@ -187,10 +191,12 @@ def test_npi_retains_native_identity_but_legacy_overlap_prefers_phone(): } == {(11, 21)} legacy = datafog.scan(text, backend="rust") assert [(item.type, item.text) for item in legacy.entities] == [ - ("PHONE", "1234567893") + ("NPI", "1234567893") + ] + assert [item.type for item in datafog.scan(text, entity_types=["NPI"], backend="rust").entities] == [ + "NPI" ] - assert not datafog.scan(text, entity_types=["NPI"], backend="rust").entities - assert datafog.redact(text, backend="rust").redacted_text == "😀 café NPI [PHONE_1]" + assert datafog.redact(text, backend="rust").redacted_text == "😀 café NPI [NPI_1]" # Native selection occurs before transformation overlap resolution, exposing # the NPI-specific route without changing the 4.x compatibility contract. result = v5.scan_and_transform( @@ -200,9 +206,12 @@ def test_npi_retains_native_identity_but_legacy_overlap_prefers_phone(): assert result.text == "😀 café NPI [NPI]" -def test_structured_only_person_rejected_by_text_adapter(): - with pytest.raises(ValueError, match="PERSON"): - datafog.scan("Jane Doe", entity_types=["PERSON"], backend="rust") +def test_structured_only_person_served_by_the_embedded_model(): + """`PERSON` is structured-only in the rule inventory but the embedded CNN + covers prose, so text selection succeeds instead of raising.""" + assert not datafog.scan("Jane Doe", entity_types=["PERSON"], backend="python").entities + result = datafog.scan("Jane Doe\n14 Penny Lane", entity_types=["PERSON", "ADDRESS"]) + assert [item.type for item in result.entities] == ["PERSON", "ADDRESS"] def test_unsupported_locale_rejected_explicitly(): @@ -210,6 +219,9 @@ def test_unsupported_locale_rejected_explicitly(): datafog.scan("a@example.com", locales=["unsupported"], backend="rust") -def test_python_default_backend_remains_unchanged(): - assert not datafog.scan(JWT).entities - assert [item.type for item in datafog.scan(JWT, backend="rust").entities] == ["JWT"] +def test_native_backend_is_the_default(): + assert [item.type for item in datafog.scan(JWT).entities] == ["JWT"] + assert [item.type for item in datafog.scan(JWT, backend="rust").entities] == [ + "JWT" + ] + assert not datafog.scan(JWT, backend="python").entities diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py index 96901cf2..920c35f3 100644 --- a/tests/test_de_pii_regex.py +++ b/tests/test_de_pii_regex.py @@ -134,12 +134,17 @@ def test_redaction_and_service_locale_support() -> None: def test_german_vat_redaction_suppresses_inner_generic_ssn_match( text: str, vat_text: str ) -> None: - # Default (no locale): v4.4.0 parity — the bare nine-digit run still - # matches the generic SSN pattern even when prefixed by a country code. - scan_result = scan(text, engine="regex") + # Default (no locale): the Python engine keeps v4.4.0 parity and still + # matches the inner nine-digit run. The native engine rejects an SSN that is + # embedded in an alphanumeric token, a reviewed detector difference. + scan_result = scan(text, engine="regex", backend="python") assert [(entity.type, entity.text) for entity in scan_result.entities] == [ ("SSN", "123456789") ] + native_expected = [] if "DE1" in text else [("SSN", "123456789")] + assert [ + (entity.type, entity.text) for entity in scan(text, engine="regex").entities + ] == native_expected # German locale: the longer DE_VAT_ID span wins via the engine's # span-overlap suppression, so the inner SSN match is dropped. @@ -148,8 +153,12 @@ def test_german_vat_redaction_suppresses_inner_generic_ssn_match( ("DE_VAT_ID", vat_text) ] - default_redaction = scan_and_redact(text, engine="regex") + default_redaction = scan_and_redact(text, engine="regex", backend="python") assert default_redaction.redacted_text == text.replace("123456789", "[SSN_1]") + native_redaction = scan_and_redact(text, engine="regex").redacted_text + assert native_redaction == ( + text if native_expected == [] else text.replace("123456789", "[SSN_1]") + ) redaction = scan_and_redact(text, engine="regex", locales=["de"]) assert redaction.redacted_text == text.replace(vat_text, "[DE_VAT_ID_1]") diff --git a/tests/test_legacy_retirement.py b/tests/test_legacy_retirement.py index f5dd80e3..f8be98f0 100644 --- a/tests/test_legacy_retirement.py +++ b/tests/test_legacy_retirement.py @@ -29,7 +29,10 @@ def unexpected_import(*args): warnings.simplefilter("error", LegacySurfaceWarning) with pytest.raises( LegacySurfaceWarning, - match=f"{surface} support is deprecated in 4.9 and will be removed in 5.0", + match=( + f"{surface} support is deprecated in 4.9 and will be removed in 5.0. " + 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0".' + ), ): (ImageService if surface == "OCR" else SparkService)() @@ -40,7 +43,7 @@ def unavailable(name): monkeypatch.setattr(importlib, "import_module", unavailable) with ( - pytest.warns(LegacySurfaceWarning, match="final 4.x release"), + pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"), pytest.raises(ImportError, match=r"datafog\[distributed\]"), ): SparkService() @@ -82,7 +85,7 @@ def test_donut_mock_output_is_preserved(monkeypatch): monkeypatch.setattr(donut_processor, "DONUT_TESTING_ENABLED", False) with pytest.warns(LegacySurfaceWarning): processor = DonutProcessor() - with pytest.warns(LegacySurfaceWarning, match="final 4.x release"): + with pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"): assert asyncio.run(processor.extract_text_from_image(object())) == ( '{"text": "Mock OCR text for testing"}' ) @@ -242,7 +245,7 @@ def test_image_cli_notice_visible_under_default_python_filters(image_argument): args.append("image.png") result = CliRunner().invoke(client.app, args) assert "deprecated in 4.9 and will be removed in 5.0" in result.stderr, result.output -assert "final 4.x release" in result.stderr, result.output +assert 'Last supported version is 4.10.0' in result.stderr, result.output if sys.argv[1] == "True": assert result.exit_code == 0, result.output assert "OCR Pipeline Results: ['OCR text']" in result.stdout diff --git a/tests/test_rust_backend.py b/tests/test_rust_backend.py index 74922c8f..34da6a21 100644 --- a/tests/test_rust_backend.py +++ b/tests/test_rust_backend.py @@ -153,9 +153,12 @@ def test_nonregex_engine_rejected(native, name): assert not calls -def test_bad_backend_rejected(): +def test_bad_backend_rejected(monkeypatch): with pytest.raises(ValueError, match="backend must be one of"): - engine.scan("", "regex", backend="auto") + engine.scan("", "regex", backend="unknown") + monkeypatch.setenv(engine.BACKEND_ENV, "unknown") + with pytest.raises(ValueError, match="backend must be one of"): + engine.scan("", "regex") def test_no_native_import_on_python_path_and_actionable_missing_error(monkeypatch): @@ -198,7 +201,9 @@ def test_all_legacy_transformations_use_native_findings(native, strategy): finding("EMAIL", "a@example.com", 14, 27), ] ) - expected = engine.scan_and_redact(text, "regex", strategy=strategy) + expected = engine.scan_and_redact( + text, "regex", strategy=strategy, backend="python" + ) actual = engine.scan_and_redact(text, "regex", strategy=strategy, backend="rust") assert actual == expected assert calls == [text] @@ -208,7 +213,9 @@ def test_all_legacy_transformations_use_native_findings(native, strategy): def test_real_native_unicode_and_transformations(strategy): pytest.importorskip("datafog_core") text = "😀 München a@example.com and a@example.com" - python_result = engine.scan_and_redact(text, "regex", strategy=strategy) + python_result = engine.scan_and_redact( + text, "regex", strategy=strategy, backend="python" + ) native_result = engine.scan_and_redact( text, "regex", strategy=strategy, backend="rust" ) diff --git a/tests/test_rust_contract.py b/tests/test_rust_contract.py index 9bea3641..c3f16bc0 100644 --- a/tests/test_rust_contract.py +++ b/tests/test_rust_contract.py @@ -15,7 +15,9 @@ def test_native_version_and_review_inventory(): - assert importlib.metadata.version("datafog-core").startswith("0.4.") + installed = importlib.metadata.version("datafog-core") + major, minor = REVIEWED["core_version"].split(".")[:2] + assert installed.split(".")[:2] == [major, minor] assert set(REVIEWED["cases"]) <= {case["id"] for case in CASES} for item in REVIEWED["cases"].values(): assert item["classification"] in { From d0b38929695fd28f1553f437e68194303033c347 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:41:42 -0800 Subject: [PATCH 5/8] chore: require datafog-core 1.3.0 1.2.0 shipped without the exchange-code fix, so 4.10.0 depends on the release that has it. Contract file follows the engine version. --- CHANGELOG.MD | 2 +- README.md | 4 ++-- RELEASE_NOTES_4.10.0.md | 4 ++-- docs/migration-4.10.md | 4 ++-- docs/python-sdk.rst | 2 +- setup.py | 4 ++-- tests/contracts/{rust-1.2.0.json => rust-1.3.0.json} | 2 +- tests/rust_contract.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) rename tests/contracts/{rust-1.2.0.json => rust-1.3.0.json} (98%) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 1049239e..d15aa102 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -16,7 +16,7 @@ #### Changed -- `datafog-core>=1.2.0,<2` is a normal dependency; `[rust]` remains as an +- `datafog-core>=1.3.0,<2` is a normal dependency; `[rust]` remains as an explicit spelling. - `Development Status: 5 - Production/Stable`. - Retirement warnings name `4.10.0` as the last supported version with an exact diff --git a/README.md b/README.md index 0f70fb23..172c0936 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ and `ADDRESS` from an embedded model, so name and address detection needs no spaCy and no model download. Findings keep `engine="regex"` provenance and `confidence=1.0` as compatibility values, not native confidence estimates. Reviewed differences from the Python engine are listed in -[tests/contracts/rust-1.2.0.json](tests/contracts/rust-1.2.0.json). +[tests/contracts/rust-1.3.0.json](tests/contracts/rust-1.3.0.json). `datafog.native` exposes the native schema directly. Python 3.13 support is certified for the core SDK, CLI, `nlp`, @@ -288,7 +288,7 @@ The [4.9 migration guide](docs/migration-4.9.md) explains opt-in Rust detection, the native `datafog.v5` preview, and the revised 5.0 retirement schedule for `detect`/`process`, OCR, and Spark. The Python detector remains the default. -4.10.0 requires Core `>=1.2.0,<2` and capability contract 1, and installs it by +4.10.0 requires Core `>=1.3.0,<2` and capability contract 1, and installs it by default. See the [4.10.0 release notes](RELEASE_NOTES_4.10.0.md) and the [4.10 migration guide](docs/migration-4.10.md). Entity labels, locales, and activation settings come from the installed diff --git a/RELEASE_NOTES_4.10.0.md b/RELEASE_NOTES_4.10.0.md index f5c592d4..fa84344a 100644 --- a/RELEASE_NOTES_4.10.0.md +++ b/RELEASE_NOTES_4.10.0.md @@ -31,12 +31,12 @@ once. stay opt-in. - `datafog.native` exposes the native schema, `capabilities()`, the embedded detector and version reporting. Experimental until 5.0. -- `datafog-core` is pinned `>=1.2.0,<2`. Detection output is contract-versioned. +- `datafog-core` is pinned `>=1.3.0,<2`. Detection output is contract-versioned. ## Detection differences Four reviewed differences from the Python engine are recorded in -[tests/contracts/rust-1.2.0.json](tests/contracts/rust-1.2.0.json) and asserted +[tests/contracts/rust-1.3.0.json](tests/contracts/rust-1.3.0.json) and asserted by the suite: Core validates card checksums, requires non-alphanumeric SSN boundaries, requires a North American area code starting 2-9 (which is what keeps Unix timestamps out of `PHONE`), and rejects unadvertised locales instead diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md index cbec85de..2046974c 100644 --- a/docs/migration-4.10.md +++ b/docs/migration-4.10.md @@ -40,7 +40,7 @@ The native engine finds things the Python regex engine never did: `API_KEY`, stay opt-in. These differences go the other way and are recorded, not hidden, in -[tests/contracts/rust-1.2.0.json](../tests/contracts/rust-1.2.0.json): +[tests/contracts/rust-1.3.0.json](../tests/contracts/rust-1.3.0.json): | Input | Python | Native | Why | | --- | --- | --- | --- | @@ -77,7 +77,7 @@ Core versions independently. If output must be byte-identical across machines, lock it: ```bash -pip install "datafog==4.10.0" "datafog-core==1.2.0" +pip install "datafog==4.10.0" "datafog-core==1.3.0" ``` `datafog.native.version()` reports the installed engine version and diff --git a/docs/python-sdk.rst b/docs/python-sdk.rst index 98781f96..581afc91 100644 --- a/docs/python-sdk.rst +++ b/docs/python-sdk.rst @@ -45,7 +45,7 @@ continue to work at the top level in 4.9 but warn of removal in 5.0, revising th previous promise to retain them throughout 5.x. Moving from ``process`` to ``redact`` can change old placeholder and hash output; compare results explicitly. -The adapter requires ``datafog-core>=1.2.0,<2``, which the base install pulls. Install with +The adapter requires ``datafog-core>=1.3.0,<2``, which the base install pulls. Install with ``datafog-core`` is installed by every ``datafog`` install; ``backend="python"`` or ``DATAFOG_BACKEND=python`` forces the pure-Python engine. Installing the extra does not change the default Python backend: diff --git a/setup.py b/setup.py index 4f012b22..df40d56c 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ core_deps = [ # Native engine, the default detection path. Exact upper bound: detection # output is contract-versioned and must stay reproducible. - "datafog-core>=1.2.0,<2", + "datafog-core>=1.3.0,<2", "pydantic>=2.0,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", @@ -88,7 +88,7 @@ extras_require = { # Retained for explicitness; the base install already pulls Core. - "rust": ["datafog-core>=1.2.0,<2"], + "rust": ["datafog-core>=1.3.0,<2"], "nlp": nlp_deps, "nlp-advanced": nlp_advanced_deps, "ocr": ocr_deps, diff --git a/tests/contracts/rust-1.2.0.json b/tests/contracts/rust-1.3.0.json similarity index 98% rename from tests/contracts/rust-1.2.0.json rename to tests/contracts/rust-1.3.0.json index 6f46c6b8..2c63c98c 100644 --- a/tests/contracts/rust-1.2.0.json +++ b/tests/contracts/rust-1.3.0.json @@ -1,5 +1,5 @@ { - "core_version": "1.2.0", + "core_version": "1.3.0", "cases": { "scan-observation-invalid-card": { "classification": "detector-difference", diff --git a/tests/rust_contract.py b/tests/rust_contract.py index 85d0d6a9..d3be6f66 100644 --- a/tests/rust_contract.py +++ b/tests/rust_contract.py @@ -9,7 +9,7 @@ from tests.contract_481 import FIXTURE, observe -DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.2.0.json" +DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.3.0.json" TARGETS = { "datafog:scan", "datafog:redact", From 288e2c237fe0ca51b6f4ee5b304775c90ffaee0d Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:12:09 -0800 Subject: [PATCH 6/8] ci: re-run after datafog-core 1.3.0 publication From f8d737ff42a8e8abdfb799ee0e4a73e471b64b11 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:27:30 -0800 Subject: [PATCH 7/8] test: score the accuracy corpus against both backends The corpus encodes Python-engine expectations, so it pins backend=python by default and scores the native engine under DATAFOG_ACCURACY_BACKEND=rust against tests/contracts/native-corpus.json, which records the twelve known divergences with reasons. CI runs both variants; pre-commit gains a ruff per-file-ignores entry for the dynamic re-exports in datafog/native.py. --- .flake8 | 5 +- .github/workflows/ci.yml | 10 +++ .github/workflows/release.yml | 5 ++ RELEASE_NOTES_4.10.0.md | 10 +++ datafog/__init__.py | 2 +- datafog/compat/v4.py | 2 +- datafog/engine.py | 40 +++++----- docs/migration-4.10.md | 17 ++++ ruff.toml | 7 ++ tests/contracts/native-corpus.json | 100 ++++++++++++++++++++++++ tests/test_allowlist.py | 18 +++-- tests/test_backend_resolution.py | 9 ++- tests/test_contract_481.py | 6 +- tests/test_core04_integration.py | 15 ++-- tests/test_detection_accuracy.py | 121 +++++++++++++++++++++++++---- 15 files changed, 311 insertions(+), 56 deletions(-) create mode 100644 ruff.toml create mode 100644 tests/contracts/native-corpus.json diff --git a/.flake8 b/.flake8 index e2ed01d9..db0f341c 100644 --- a/.flake8 +++ b/.flake8 @@ -3,4 +3,7 @@ ignore = E203, E266, E501, W503, B006, B007, B008, F401, C416, B950, B904 max-line-length = 88 max-complexity = 18 select = B,C,E,F,W,T4,B9 -exclude = venv, .venv, tests/.datafog_env, examples/venv \ No newline at end of file +exclude = venv, .venv, tests/.datafog_env, examples/venv# datafog.native.py re-exports the installed datafog-core surface through a +# dynamic __getattr__, so its __all__ names resolve at runtime. +per-file-ignores = + datafog/native.py: F822 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be420465..ba447626 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,6 +119,16 @@ jobs: --cov-report=xml \ --cov-report=term-missing + - name: Run detection accuracy corpus on the native engine + if: matrix.python-version == '3.11' && matrix.install-profile == 'nlp-advanced' + env: + DATAFOG_ACCURACY_BACKEND: rust + run: | + pytest tests/test_detection_accuracy.py \ + --tb=short \ + --ignore=tests/test_spark_integration.py \ + --ignore=tests/test_text_service_integration.py + - name: Run tests (nlp) if: matrix.install-profile == 'nlp' run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77e7aedd..d93c6bd8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -167,6 +167,11 @@ jobs: - name: Run detection accuracy corpus run: python -m pytest tests/test_detection_accuracy.py -v --tb=short + - name: Run detection accuracy corpus on the native engine + env: + DATAFOG_ACCURACY_BACKEND: rust + run: python -m pytest tests/test_detection_accuracy.py --tb=short + - name: Run performance validation run: | OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 python tests/simple_performance_test.py diff --git a/RELEASE_NOTES_4.10.0.md b/RELEASE_NOTES_4.10.0.md index fa84344a..7608b6e0 100644 --- a/RELEASE_NOTES_4.10.0.md +++ b/RELEASE_NOTES_4.10.0.md @@ -45,6 +45,16 @@ of ignoring them. `(555) 123-4567` and similar form data now match: Core keeps the area-code check and accepts exchange codes starting with 1. +## Accuracy corpus + +The frozen corpus runs twice in CI now: once against the Python engine, whose +expectations it encodes, and once against the native engine with its twelve +recorded divergences in +[tests/contracts/native-corpus.json](tests/contracts/native-corpus.json). The +native divergences are strictness differences (Core validates more) plus email +boundary handling that Core currently exposes through `format: "env"` / +`"sql"` settings — flagged for a default decision in Core 1.4. + ## Deprecations unchanged `detect()`, `process()`, OCR and Spark still work and still warn; all are removed diff --git a/datafog/__init__.py b/datafog/__init__.py index 12ecc115..b0c7f2a8 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -11,8 +11,8 @@ import warnings -from ._legacy_retirement import LAST_SUPPORTED as _LAST_SUPPORTED from .__about__ import __version__ +from ._legacy_retirement import LAST_SUPPORTED as _LAST_SUPPORTED from .agent import create_guardrail, filter_output, sanitize, scan_prompt # Core API functions - always available (lightweight) diff --git a/datafog/compat/v4.py b/datafog/compat/v4.py index 49d32e2c..96d619aa 100644 --- a/datafog/compat/v4.py +++ b/datafog/compat/v4.py @@ -5,8 +5,8 @@ """ from ..engine import Entity, RedactResult, ScanResult -from ..engine import resolve_backend from ..engine import redact as _redact_entities +from ..engine import resolve_backend from ..engine import scan as _scan from ..engine import scan_and_redact as _scan_and_redact diff --git a/datafog/engine.py b/datafog/engine.py index fa6ad926..495c2d18 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -296,9 +296,7 @@ def _cnn_findings( except Exception as exc: # relative-path/limit errors surface to the caller raise _model_error(exc) from exc return [ - ({}, finding) - for finding in result.findings - if finding.entity_type in requested + ({}, finding) for finding in result.findings if finding.entity_type in requested ] @@ -347,23 +345,23 @@ def _rust_entities( if detector is not None: findings.extend(_cnn_findings(detector, model_requested, text)) for config, finding in findings: - canonical_type = _canonical_type(finding.entity_type) - start = finding.codepoint_range.start - end = finding.codepoint_range.end - key = (canonical_type, start, end) - if key in seen or not finding.matched_text.strip(): - continue - seen.add(key) - entities.append( - Entity( - type=canonical_type, - text=finding.matched_text, - start=start, - end=end, - confidence=1.0, - engine="regex", - ) + canonical_type = _canonical_type(finding.entity_type) + start = finding.codepoint_range.start + end = finding.codepoint_range.end + key = (canonical_type, start, end) + if key in seen or not finding.matched_text.strip(): + continue + seen.add(key) + entities.append( + Entity( + type=canonical_type, + text=finding.matched_text, + start=start, + end=end, + confidence=1.0, + engine="regex", ) + ) return _suppress_overlapping_entities(entities) @@ -590,7 +588,9 @@ def scan( if engine not in {"regex", "spacy", "gliner", "smart"}: raise ValueError("engine must be one of: regex, spacy, gliner, smart") - requested_backend = (backend or os.environ.get(BACKEND_ENV) or "auto").strip().lower() + requested_backend = ( + (backend or os.environ.get(BACKEND_ENV) or "auto").strip().lower() + ) backend = resolve_backend(backend, engine) _validate_backend(backend, engine) if backend == "python" and requested_backend == "auto" and not core_available(): diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md index 2046974c..9a730c3b 100644 --- a/docs/migration-4.10.md +++ b/docs/migration-4.10.md @@ -55,6 +55,23 @@ no download and no interpreter. `PERSON` for a text scan on `backend="python"` is still Python-engine territory: the pure-Python regex engine has no name detection, so select `PERSON` without forcing `backend="python"`. +## The accuracy corpus + +`tests/corpus` is the frozen accuracy corpus, and its expectations describe the +Python engine, which is what they were written against. `pytest +tests/test_detection_accuracy.py` pins `backend="python"`. Set +`DATAFOG_ACCURACY_BACKEND=rust` to score the native engine; its twelve known +divergences — eight misses, one added finding and three shifted email +boundaries — are enumerated with reasons in +[tests/contracts/native-corpus.json](../tests/contracts/native-corpus.json), and +CI runs both variants. + +The three shifted email boundaries are the one worth filing a follow-up for: +Core finds `alice@example.com` inside ``email=alice@example.com`` only under its +`format: "env"` setting, and inside a quoted value only under +`format: "sql"`. Log-shaped input wants that behaviour by default; Core 1.4 +should decide that rather than leaving it to each caller. + ## If the output must not move Pin the backend, not just the version: diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..536f394a --- /dev/null +++ b/ruff.toml @@ -0,0 +1,7 @@ +# Only the delta from the default ruff rules. Rule selection stays ruff's +# default so this hook behaves like pre-4.10; flake8 config is in .flake8. + +[lint.per-file-ignores] +# datafog/native.py re-exports the installed datafog-core surface through a +# dynamic __getattr__, so its __all__ names resolve at runtime. +"datafog/native.py" = ["F822"] diff --git a/tests/contracts/native-corpus.json b/tests/contracts/native-corpus.json new file mode 100644 index 00000000..26679ced --- /dev/null +++ b/tests/contracts/native-corpus.json @@ -0,0 +1,100 @@ +{ + "core_version": "1.3.0", + "cases": { + "adjacent-pii-no-separator": { + "missing": [ + [ + "SSN", + "123-45-6789" + ] + ], + "reason": "Core requires non-alphanumeric SSN boundaries; the adjacent-email case embeds it" + }, + "phone-two-values": { + "missing": [ + [ + "PHONE", + "555-000-1111" + ] + ], + "reason": "Core requires the exchange code to start 1-9; 000-1111 is a fictional exchange" + }, + "ssn-second-valid": { + "missing": [ + [ + "SSN", + "987-65-4321" + ] + ], + "reason": "Core rejects SSN after 'Backup SSN' context token in this boundary case" + }, + "date-slash-short": { + "missing": [ + [ + "DATE", + "3/5/2020" + ] + ], + "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" + }, + "date-dash-short": { + "missing": [ + [ + "DATE", + "3-5-2020" + ] + ], + "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" + }, + "date-year-only": { + "missing": [ + [ + "DATE", + "year 2024" + ] + ], + "reason": "Core DATE requires a full date; bare years are not matched" + }, + "de-vat-id-locale": { + "missing": [ + [ + "DE_VAT_ID", + "DE 123456789" + ] + ], + "reason": "Core German VAT requires the digits to follow the country prefix without a space" + }, + "de-iban-locale": { + "missing": [ + [ + "DE_IBAN", + "DE44 5001 0517 5407 3249 31" + ] + ], + "reason": "Core German IBAN accepts the grouped form but this spacing variant differs" + } + }, + "extra": { + "product-code-not-phone": { + "added": [ + [ + "PHONE", + "555-123-4567" + ] + ], + "reason": "Core accepts exchange codes starting with 1, so this product code is PHONE now; the area-code 2-9 check still keeps Unix timestamps out" + }, + "unicode-chinese-name": { + "offsets_shifted": true, + "reason": "Core's default email boundary set does not include a CJK prefix; ENV/SQL format handling covers the Latin cases it ships today" + }, + "json-escaped": { + "offsets_shifted": true, + "reason": "Core treats '=' as an assignment boundary under format: env, not by default" + }, + "markdown-code-block": { + "offsets_shifted": true, + "reason": "Core treats the opening quote as part of the token under format: sql, not by default" + } + } +} diff --git a/tests/test_allowlist.py b/tests/test_allowlist.py index 0bd85aec..f737a9f2 100644 --- a/tests/test_allowlist.py +++ b/tests/test_allowlist.py @@ -59,12 +59,18 @@ def test_leading_one_timestamp_is_a_documented_native_delta(self): # The native NANP check reads a leading 1 as a country code and needs a # nine-digit national number after it, so this misses; the Python engine # matches. Recorded rather than silently accepted. - assert datafog.scan(f"created {LEADING_ONE_TIMESTAMP}", engine="regex").entities == [] - assert len( - datafog.scan( - f"created {LEADING_ONE_TIMESTAMP}", engine="regex", backend="python" - ).entities - ) == 1 + assert ( + datafog.scan(f"created {LEADING_ONE_TIMESTAMP}", engine="regex").entities + == [] + ) + assert ( + len( + datafog.scan( + f"created {LEADING_ONE_TIMESTAMP}", engine="regex", backend="python" + ).entities + ) + == 1 + ) def test_pattern_matches_full_entity_text_only(self): result = datafog.scan( diff --git a/tests/test_backend_resolution.py b/tests/test_backend_resolution.py index 729e1a54..c6cd6d60 100644 --- a/tests/test_backend_resolution.py +++ b/tests/test_backend_resolution.py @@ -41,8 +41,8 @@ def test_auto_keeps_non_regex_engines_on_python(): def test_python_fallback_warns_once_then_stays_quiet(without_core): with pytest.warns(CoreUnavailableWarning) as caught: - datafog.scan("mail jane@example.com") - datafog.scan("mail jane@example.com") + for _ in range(2): + datafog.scan("mail jane@example.com") assert len(caught) == 1 @@ -92,7 +92,10 @@ def test_person_selection_without_model_raises_a_pin(monkeypatch): "default_entities": ["EMAIL"], "locales": {"en-US": {"enabled_entities": []}}, "entities": { - "EMAIL": {"scopes": ["structured", "text"], "activation": {"kind": "default"}}, + "EMAIL": { + "scopes": ["structured", "text"], + "activation": {"kind": "default"}, + }, "PERSON": {"scopes": ["structured"], "activation": {"kind": "default"}}, }, } diff --git a/tests/test_contract_481.py b/tests/test_contract_481.py index 95158b07..43863dfd 100644 --- a/tests/test_contract_481.py +++ b/tests/test_contract_481.py @@ -19,7 +19,11 @@ def expected_49(case): reviewed native differences.""" deviation = REVIEWED["cases"].get(case["id"]) - expected = copy.deepcopy(deviation["expected"]) if deviation else copy.deepcopy(case["expected"]) + expected = ( + copy.deepcopy(deviation["expected"]) + if deviation + else copy.deepcopy(case["expected"]) + ) if case["id"] == "public-signatures": for target in ( "datafog:scan", diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index f767fd0b..8cd218a9 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -193,9 +193,10 @@ def test_contextual_npi_survives_the_legacy_overlap_policy(): assert [(item.type, item.text) for item in legacy.entities] == [ ("NPI", "1234567893") ] - assert [item.type for item in datafog.scan(text, entity_types=["NPI"], backend="rust").entities] == [ - "NPI" - ] + assert [ + item.type + for item in datafog.scan(text, entity_types=["NPI"], backend="rust").entities + ] == ["NPI"] assert datafog.redact(text, backend="rust").redacted_text == "😀 café NPI [NPI_1]" # Native selection occurs before transformation overlap resolution, exposing # the NPI-specific route without changing the 4.x compatibility contract. @@ -209,7 +210,9 @@ def test_contextual_npi_survives_the_legacy_overlap_policy(): def test_structured_only_person_served_by_the_embedded_model(): """`PERSON` is structured-only in the rule inventory but the embedded CNN covers prose, so text selection succeeds instead of raising.""" - assert not datafog.scan("Jane Doe", entity_types=["PERSON"], backend="python").entities + assert not datafog.scan( + "Jane Doe", entity_types=["PERSON"], backend="python" + ).entities result = datafog.scan("Jane Doe\n14 Penny Lane", entity_types=["PERSON", "ADDRESS"]) assert [item.type for item in result.entities] == ["PERSON", "ADDRESS"] @@ -221,7 +224,5 @@ def test_unsupported_locale_rejected_explicitly(): def test_native_backend_is_the_default(): assert [item.type for item in datafog.scan(JWT).entities] == ["JWT"] - assert [item.type for item in datafog.scan(JWT, backend="rust").entities] == [ - "JWT" - ] + assert [item.type for item in datafog.scan(JWT, backend="rust").entities] == ["JWT"] assert not datafog.scan(JWT, backend="python").entities diff --git a/tests/test_detection_accuracy.py b/tests/test_detection_accuracy.py index 61dec7ba..6e7d8dfe 100644 --- a/tests/test_detection_accuracy.py +++ b/tests/test_detection_accuracy.py @@ -281,10 +281,13 @@ def _canon_type(entity_type: str) -> str: def _extract_entities( - text: str, engine: str, locales: list[str] | None = None + text: str, + engine: str, + locales: list[str] | None = None, + backend: str | None = None, ) -> list[dict[str, Any]]: try: - result = scan(text=text, engine=engine, locales=locales) + result = scan(text=text, engine=engine, locales=locales, backend=backend) except (ImportError, EngineNotAvailable) as exc: pytest.skip(f"{engine} engine unavailable in this environment: {exc}") @@ -336,6 +339,44 @@ def _required_expected( return expected_list +def _corpus_backend(engine: str | None = None) -> str: + """Backend under test for the frozen corpus, settable from the CLI. + + The default is ``python`` because the corpus expectations were authored + against that engine. Set ``DATAFOG_ACCURACY_BACKEND=rust`` to score the + native engine; its known divergences are then skipped through + ``tests/contracts/native-corpus.json``. + """ + import os + + value = os.environ.get("DATAFOG_ACCURACY_BACKEND", "python").strip().lower() + if engine is not None and value == "rust" and engine != "regex": + # The native path serves regex plus embedded-model findings; other + # engines are Python ML paths by definition. + value = "python" + if value not in {"python", "rust"}: + raise ValueError("DATAFOG_ACCURACY_BACKEND must be python or rust") + return "python" if value == "rust" and engine != "regex" else value + + +@lru_cache(maxsize=1) +def _native_deltas() -> dict[str, Any]: + path = Path("tests/contracts/native-corpus.json") + if not path.exists(): + return {"core_version": None, "cases": {}} + return json.loads(path.read_text(encoding="utf-8")) + + +def _native_missing(case_id: str) -> list[list[str]]: + """Findings the native engine is known to miss, asserted only as recorded.""" + return _native_deltas()["cases"].get(case_id, {}).get("missing", []) + + +def _native_offsets_shifted(case_id: str) -> bool: + """True when the native engine finds the value with different boundaries.""" + return bool(_native_deltas()["extra"].get(case_id, {}).get("offsets_shifted")) + + def _xfail_if_known_limitation( case: dict[str, Any], engine: str, corpus_kind: str ) -> None: @@ -346,11 +387,19 @@ def _xfail_if_known_limitation( def _assert_expected_found( - case: dict[str, Any], engine: str, corpus_kind: str + case: dict[str, Any], engine: str, corpus_kind: str, backend: str | None = None ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: text = case["input"] - actual = _extract_entities(text, engine, locales=case.get("locales")) + actual = _extract_entities( + text, engine, locales=case.get("locales"), backend=backend + ) expected = _required_expected(case["expected_entities"], engine, corpus_kind) + if (backend or _corpus_backend(engine)) == "rust": + expected = [ + entry + for entry in expected + if [entry["type"], entry["text"]] not in _native_missing(case["id"]) + ] for exp in expected: exp_type = _canon_type(exp["type"]) @@ -358,6 +407,15 @@ def _assert_expected_found( matches = [ ent for ent in actual if ent["type"] == exp_type and ent["text"] == exp_text ] + if _native_offsets_shifted(case["id"]): + # Same entity, different span: the native boundary rules treat the + # surrounding characters differently. Overlap is good enough. + matches = [ + ent + for ent in actual + if ent["type"] == exp_type + and not (ent["end"] <= exp["start"] or ent["start"] >= exp["end"]) + ] if not matches: matches = [ ent @@ -373,7 +431,9 @@ def _assert_expected_found( # If offsets are available from the engine output, validate exact position. with_offsets = [m for m in matches if m["start"] >= 0 and m["end"] >= 0] if with_offsets: - if engine == "regex" or exp_type in STRUCTURED_TYPES: + if ( + engine == "regex" or exp_type in STRUCTURED_TYPES + ) and not _native_offsets_shifted(case["id"]): assert any( m["start"] == exp["start"] and m["end"] == exp["end"] for m in with_offsets @@ -406,7 +466,10 @@ def _compute_metrics( for corpus_kind, cases in corpora: for case in cases: actual = _extract_entities( - case["input"], engine, locales=case.get("locales") + case["input"], + engine, + locales=case.get("locales"), + backend=_corpus_backend(engine), ) expected = _required_expected( case["expected_entities"], engine, corpus_kind @@ -475,7 +538,7 @@ def _prf(scores: dict[str, int]) -> dict[str, float]: @pytest.mark.parametrize("engine", FAST_ENGINES) def test_structured_pii_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "structured") - _assert_expected_found(case, engine, "structured") + _assert_expected_found(case, engine, "structured", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -485,7 +548,7 @@ def test_structured_pii_detection_fast(case: dict[str, Any], engine: str) -> Non @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_structured_pii_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "structured") - _assert_expected_found(case, engine, "structured") + _assert_expected_found(case, engine, "structured", backend=_corpus_backend(engine)) @pytest.mark.parametrize( @@ -494,7 +557,18 @@ def test_structured_pii_detection_slow(case: dict[str, Any], engine: str) -> Non @pytest.mark.parametrize("engine", FAST_ENGINES) def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "negative") - actual = _extract_entities(case["input"], engine, locales=case.get("locales")) + actual = _extract_entities( + case["input"], + engine, + locales=case.get("locales"), + backend=_corpus_backend(engine), + ) + if _corpus_backend(engine) == "rust": + added = { + tuple(pair) + for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) + } + actual = [e for e in actual if (e["type"], e["text"]) not in added] assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -505,7 +579,18 @@ def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "negative") - actual = _extract_entities(case["input"], engine, locales=case.get("locales")) + actual = _extract_entities( + case["input"], + engine, + locales=case.get("locales"), + backend=_corpus_backend(engine), + ) + if _corpus_backend(engine) == "rust": + added = { + tuple(pair) + for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) + } + actual = [e for e in actual if (e["type"], e["text"]) not in added] assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -515,7 +600,9 @@ def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", ["smart"]) def test_unstructured_pii_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "unstructured") - _assert_expected_found(case, engine, "unstructured") + _assert_expected_found( + case, engine, "unstructured", backend=_corpus_backend(engine) + ) @pytest.mark.slow @@ -525,14 +612,16 @@ def test_unstructured_pii_detection_fast(case: dict[str, Any], engine: str) -> N @pytest.mark.parametrize("engine", ["gliner", "spacy"]) def test_unstructured_pii_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "unstructured") - _assert_expected_found(case, engine, "unstructured") + _assert_expected_found( + case, engine, "unstructured", backend=_corpus_backend(engine) + ) @pytest.mark.parametrize("case", load_corpus("mixed_pii.json"), ids=lambda c: c["id"]) @pytest.mark.parametrize("engine", FAST_ENGINES) def test_mixed_pii_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "mixed") - _assert_expected_found(case, engine, "mixed") + _assert_expected_found(case, engine, "mixed", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -540,14 +629,14 @@ def test_mixed_pii_detection_fast(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_mixed_pii_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "mixed") - _assert_expected_found(case, engine, "mixed") + _assert_expected_found(case, engine, "mixed", backend=_corpus_backend(engine)) @pytest.mark.parametrize("case", load_corpus("edge_cases.json"), ids=lambda c: c["id"]) @pytest.mark.parametrize("engine", FAST_ENGINES) def test_edge_case_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "edge") - _assert_expected_found(case, engine, "edge") + _assert_expected_found(case, engine, "edge", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -555,7 +644,7 @@ def test_edge_case_detection_fast(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_edge_case_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "edge") - _assert_expected_found(case, engine, "edge") + _assert_expected_found(case, engine, "edge", backend=_corpus_backend(engine)) @pytest.mark.slow From e3d4300368780ba91fee4d6d6c3ae8b5f45f5ae8 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:38:48 -0800 Subject: [PATCH 8/8] style: prettier formatting for the native corpus delta --- tests/contracts/native-corpus.json | 63 +++++------------------------- 1 file changed, 9 insertions(+), 54 deletions(-) diff --git a/tests/contracts/native-corpus.json b/tests/contracts/native-corpus.json index 26679ced..e3a3bbd7 100644 --- a/tests/contracts/native-corpus.json +++ b/tests/contracts/native-corpus.json @@ -2,86 +2,41 @@ "core_version": "1.3.0", "cases": { "adjacent-pii-no-separator": { - "missing": [ - [ - "SSN", - "123-45-6789" - ] - ], + "missing": [["SSN", "123-45-6789"]], "reason": "Core requires non-alphanumeric SSN boundaries; the adjacent-email case embeds it" }, "phone-two-values": { - "missing": [ - [ - "PHONE", - "555-000-1111" - ] - ], + "missing": [["PHONE", "555-000-1111"]], "reason": "Core requires the exchange code to start 1-9; 000-1111 is a fictional exchange" }, "ssn-second-valid": { - "missing": [ - [ - "SSN", - "987-65-4321" - ] - ], + "missing": [["SSN", "987-65-4321"]], "reason": "Core rejects SSN after 'Backup SSN' context token in this boundary case" }, "date-slash-short": { - "missing": [ - [ - "DATE", - "3/5/2020" - ] - ], + "missing": [["DATE", "3/5/2020"]], "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" }, "date-dash-short": { - "missing": [ - [ - "DATE", - "3-5-2020" - ] - ], + "missing": [["DATE", "3-5-2020"]], "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" }, "date-year-only": { - "missing": [ - [ - "DATE", - "year 2024" - ] - ], + "missing": [["DATE", "year 2024"]], "reason": "Core DATE requires a full date; bare years are not matched" }, "de-vat-id-locale": { - "missing": [ - [ - "DE_VAT_ID", - "DE 123456789" - ] - ], + "missing": [["DE_VAT_ID", "DE 123456789"]], "reason": "Core German VAT requires the digits to follow the country prefix without a space" }, "de-iban-locale": { - "missing": [ - [ - "DE_IBAN", - "DE44 5001 0517 5407 3249 31" - ] - ], + "missing": [["DE_IBAN", "DE44 5001 0517 5407 3249 31"]], "reason": "Core German IBAN accepts the grouped form but this spacing variant differs" } }, "extra": { "product-code-not-phone": { - "added": [ - [ - "PHONE", - "555-123-4567" - ] - ], + "added": [["PHONE", "555-123-4567"]], "reason": "Core accepts exchange codes starting with 1, so this product code is PHONE now; the area-code 2-9 check still keeps Unix timestamps out" }, "unicode-chinese-name": {