diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e72c9e2a..46abe29a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.9.0 +current_version = 4.12.0 commit = True tag = True tag_name = v{new_version} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index bcc51e1b..944cd756 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,6 +27,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[nlp]" + python -m spacy download en_core_web_lg pip install -r requirements-benchmark.txt # Verify pytest-benchmark is installed and working python -c "import pytest_benchmark; print('pytest-benchmark version:', pytest_benchmark.__version__)" @@ -54,13 +55,7 @@ jobs: # Run benchmarks with optimal performance settings (no memory debugging) echo "Running benchmarks with performance-optimized settings..." - # Try pytest-benchmark first - if python -m pytest tests/benchmark_text_service.py -v --benchmark-autosave --benchmark-json=benchmark-results.json --tb=short -p no:cacheprovider; then - echo "✅ pytest-benchmark tests completed successfully" - else - echo "⚠️ pytest-benchmark failed, running simple performance test as fallback" - python tests/simple_performance_test.py - fi + python -m pytest tests/benchmark_text_service.py -v --benchmark-autosave --benchmark-json=benchmark-results.json --tb=short -p no:cacheprovider - name: Check for performance regression run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d93c6bd8..0e2b8c63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,9 +2,9 @@ name: Release on: schedule: - # Monday-Wednesday at 2 AM UTC: alpha builds from dev + # Scheduled runs validate candidates without publishing. - cron: "0 2 * * 1-3" - # Thursday at 2 AM UTC: beta builds from dev + # Thursday beta qualification also stops before publication. - cron: "0 2 * * 4" workflow_dispatch: inputs: @@ -19,7 +19,7 @@ on: dry_run: description: "Dry run (skip PyPI publish)" required: false - default: false + default: true type: boolean dry_run_ref: description: "Candidate ref to validate (allowed only with dry_run)" @@ -86,6 +86,15 @@ jobs: fetch-depth: 0 ref: ${{ steps.resolve.outputs.checkout_ref }} + - name: Reject a development branch missing stable changes + if: steps.resolve.outputs.release_type != 'stable' + run: | + git fetch origin main + git merge-base --is-ancestor origin/main HEAD || { + echo "Development source is missing stable changes; merge main into dev first" + exit 1 + } + - name: Freeze source commit id: source run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" @@ -125,7 +134,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v6 with: @@ -272,7 +281,7 @@ jobs: pip install build twine bump2version - name: Configure git - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true run: | git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]" @@ -282,57 +291,20 @@ jobs: env: VERSION_OVERRIDE: ${{ inputs.version_override }} run: | - set -e + set -euo pipefail git fetch --tags - TYPE="${{ needs.determine-release.outputs.release_type }}" - CURRENT=$(sed -n 's/^__version__ = "\([^"]*\)"/\1/p' datafog/__about__.py) - if [ -z "$CURRENT" ]; then - echo "Failed to parse current version from datafog/__about__.py" - exit 1 - fi - echo "Current version: $CURRENT" - - # Strip any pre-release suffix to get base version - BASE=$(echo "$CURRENT" | sed -E 's/(a|b)[0-9]+([.][0-9A-Za-z]+)?$//') - if [ -n "$VERSION_OVERRIDE" ]; then - BASE="$VERSION_OVERRIDE" - if echo "$BASE" | grep -Eq '(a|b)[0-9]+([.][0-9A-Za-z]+)?$'; then - echo "version_override must be a stable base version like 4.4.0, not a prerelease" - exit 1 - fi - fi - if ! echo "$BASE" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then - echo "Release base must have numeric major.minor.patch format" - exit 1 - fi - echo "Base version: $BASE" - - if [ "$TYPE" = "alpha" ]; then - ALPHA_NUM=1 - while git tag -l "v${BASE}a${ALPHA_NUM}" | grep -q .; do - ALPHA_NUM=$((ALPHA_NUM + 1)) - done - VERSION="${BASE}a${ALPHA_NUM}" - - elif [ "$TYPE" = "beta" ]; then - BETA_NUM=1 - while git tag -l "v${BASE}b${BETA_NUM}" | grep -q .; do - BETA_NUM=$((BETA_NUM + 1)) - done - VERSION="${BASE}b${BETA_NUM}" - - else - VERSION="$BASE" - fi - + VERSION=$(python scripts/release_version.py --type "$TYPE" --override "$VERSION_OVERRIDE") echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "Publishing version: $VERSION" - - sed -i "s/__version__ = \".*\"/__version__ = \"$VERSION\"/" datafog/__about__.py - if grep -q 'version="' setup.py 2>/dev/null; then - sed -i "s/version=\".*\"/version=\"$VERSION\"/" setup.py - fi + RELEASE_VERSION="$VERSION" python - <<'PYCODE' + import os, re + from pathlib import Path + version = os.environ["RELEASE_VERSION"] + about = Path("datafog/__about__.py") + about.write_text(re.sub(r'^__version__ = ".*"', f'__version__ = "{version}"', about.read_text(), flags=re.M)) + config = Path(".bumpversion.cfg") + config.write_text(re.sub(r'^current_version = .*', f'current_version = {version}', config.read_text(), flags=re.M)) + PYCODE - name: Generate changelog env: @@ -371,7 +343,7 @@ jobs: if-no-files-found: error - name: Verify release branch still matches tested source - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: RELEASE_BRANCH: ${{ needs.determine-release.outputs.target_branch }} TESTED_SHA: ${{ needs.determine-release.outputs.source_sha }} @@ -382,30 +354,36 @@ jobs: exit 1 fi + - name: Commit and tag the validated release before publication + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true + env: + VERSION: ${{ steps.version.outputs.version }} + BRANCH: ${{ needs.determine-release.outputs.target_branch }} + run: | + set -euo pipefail + git add datafog/__about__.py .bumpversion.cfg + if ! git diff --cached --quiet; then + git commit -m "chore: bump version to $VERSION [skip ci]" + fi + git tag -a "v$VERSION" -m "Release $VERSION" + git push --atomic origin "HEAD:$BRANCH" "v$VERSION" + - name: Publish to PyPI - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | python -m twine upload dist/* --verbose - - name: Commit version bump & create release - if: inputs.dry_run != true + - name: Create GitHub release + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: GITHUB_TOKEN: ${{ secrets.GH_PAT }} + VERSION: ${{ steps.version.outputs.version }} + TYPE: ${{ needs.determine-release.outputs.release_type }} + BRANCH: ${{ needs.determine-release.outputs.target_branch }} run: | - VERSION="${{ steps.version.outputs.version }}" - TYPE="${{ needs.determine-release.outputs.release_type }}" - BRANCH="${{ needs.determine-release.outputs.target_branch }}" - - git add datafog/__about__.py setup.py - git commit -m "chore: bump version to $VERSION [skip ci]" || echo "No version changes to commit" - git push origin "HEAD:$BRANCH" - - git tag -a "v$VERSION" -m "Release $VERSION" - git push origin "v$VERSION" - PRERELEASE_FLAG="" TITLE="" if [ "$TYPE" = "alpha" ]; then @@ -426,7 +404,7 @@ jobs: dist/* - name: Dry run summary - if: inputs.dry_run == true + if: github.event_name != 'workflow_dispatch' || inputs.dry_run == true run: | echo "DRY RUN COMPLETE" echo "Would have published: ${{ steps.version.outputs.version }}" @@ -434,9 +412,25 @@ jobs: ls -la dist/ # ── 4. Cleanup old pre-releases ─────────────────────────────────────── + sync-development: + needs: [determine-release, publish] + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.determine-release.outputs.release_type == 'stable' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: dev + fetch-depth: 0 + token: ${{ secrets.GH_PAT }} + - name: Bring stable release metadata back to development + run: | + git fetch origin main + git merge --ff-only origin/main + git push origin HEAD:dev + cleanup: needs: [determine-release, publish] - if: needs.determine-release.outputs.release_type != 'stable' && inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.determine-release.outputs.release_type != 'stable' runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/.gitignore b/.gitignore index 46214511..a594afa1 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ docs/* !docs/migration-4.8.1-contract.md !docs/migration-4.9.md !docs/core-0.4-integration.md +!docs/review-remediation.md !docs/agents/ !docs/agents/** !docs/audit/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4063257e..f6b76791 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort args: [--profile=black] exclude: .venv - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 26.3.1 hooks: - id: black language_version: python3 diff --git a/AGENTS.md b/AGENTS.md index 13439692..921d2d55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,9 +13,9 @@ ## Current Project Status -**Stable version: 4.9.0** +**Stable version: 4.11.0** -**Development version: 4.9.0** +**Development candidate: 4.12.0** **Next major target: 5.0.0** @@ -239,17 +239,17 @@ except ImportError: ### Workflow Architecture (3 workflows) -| Workflow | Purpose | Trigger | -| --------------- | ----------------------------------- | -------------------------- | -| `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | -| `release.yml` | Alpha/Beta/Stable publishing | Schedule + manual dispatch | -| `benchmark.yml` | Performance benchmarks | Push/PR/weekly | +| Workflow | Purpose | Trigger | +| --------------- | -------------------------------------- | --------------------------------------------- | +| `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | +| `release.yml` | Candidate qualification and publishing | Schedule validates; manual dispatch publishes | +| `benchmark.yml` | Performance benchmarks | Push/PR/weekly | ### Release Cadence -- **Alpha** (Mon-Wed 2AM UTC): Automatic from `dev`, date+commit versioning -- **Beta** (Thursday 2AM UTC): Automatic from `dev`, incremental beta numbers -- **Stable** (manual dispatch): From `main`, base version or override +- **Alpha** (Mon-Wed 2AM UTC): Candidate qualification from `dev`, without publication +- **Beta** (Thursday 2AM UTC): Candidate qualification from `dev`, without publication +- **Publication** (manual dispatch): Dry run by default; explicit non-dry runs publish alpha/beta from `dev` or stable from `main` ### Release Pipeline @@ -320,7 +320,7 @@ Before beginning any task please checkout a branch from `dev` and create a pull ### Release Preparation -- Alpha/beta releases are automated via `release.yml` schedule +- Alpha/beta candidate qualification is automated via `release.yml`; publication requires an explicit manual dispatch - Stable releases: merge `dev` → `main`, then trigger `release.yml` with `stable` type - Use `dry_run: true` to validate before actual publish - Performance validation on realistic data sets diff --git a/CHANGELOG.MD b/CHANGELOG.MD index ddc47dd9..86116707 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,5 +1,46 @@ # ChangeLog +## [Unreleased] + +## [4.12.0] + +#### Security + +- Preserve native structured PII detection in smart, spaCy, GLiNER, legacy and + CLI paths. Normalize lone surrogates without moving source spans; retain + structured findings on bounded model inputs. Require review when hook payloads + exceed scan budgets, rather than silently allowing unscanned tails. +- Bound user allowlist matching with timeouts and remove quadratic overlap and + Python fallback regex behavior. Redact exact NER offsets and whole TextService + inputs so repeated names and chunk boundaries cannot expose PII. +- Cover LiteLLM tool arguments, prompt/input requests and complete streamed + responses, with request-scoped tokens. Hide configured API keys in CLI output. + +#### Changed + +- Hash APIs honor the selected algorithm while preserving existing plain-hash + marker widths and defaults. Optional `hash_key` enables full-length HMAC with + a key of at least 32 bytes. Plain hashes provide deterministic pseudonyms and + do not resist guessing of low-entropy PII. +- Reuse tokens for repeated values, avoid existing token collisions, reject + invalid explicit ranges and unsupported entity selections, and report backend + entity capabilities. Async decorators await results before filtering. +- Require the reviewed Core 1.5 minor series (`datafog-core~=1.5.0`) in the base + package and `rust` extra; review native differences in `rust-1.5.0.json`. + Core's complete mapped-IPv6 spans, email boundaries, Unicode mailboxes and + phone-plus-ZIP classification are available through the wrapper. Published + Core artifacts passed dependent installed-wheel qualification. +- Raise the Pydantic floor to 2.1, + include spaCy in `nlp-advanced`, and allow Torch 2.x versions compatible with + `datafog-core[gliner2]`. Add an isolated build declaration and runnable sdist + tests; remove unused duplicate implementation modules. +- Repair Pydantic optional defaults, legacy anonymizer enum filters and overlaps, + Spark label aliases, implicit pipeline scanning and invalid CLI operations. +- Scheduled release jobs qualify candidates without publication. Manual runs + default to dry runs, reject stale version bases and development history, gate + Python 3.13, record release refs before upload, and synchronize stable metadata + back to development after publication. + ## [4.11.0] #### Changed diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..75a7cea8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include LICENSE README.md pyproject.toml tox.ini requirements-test.txt +recursive-include tests *.py *.json *.txt +recursive-include scripts *.py +prune tests/.datafog_env +global-exclude __pycache__ *.py[cod] diff --git a/README.md b/README.md index 172c0936..8c647a1e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DataFog Python +# DataFog Python DataFog is a Python library for detecting and redacting personally identifiable information (PII). @@ -85,8 +85,9 @@ multilingual tokenizer; installing the OCR extra is not required for GLiNER. `backend` defaults to `auto`: the native engine when `datafog-core` is importable, the pure-Python regex engine otherwise. `DATAFOG_BACKEND` sets the default and `backend="python"` / `backend="rust"` override it per call. -`auto` never selects the native engine for `engine="spacy"`, `"gliner"` or -`"smart"`, which stay on the Python ML path. +Every engine uses this backend for structured PII. `engine="spacy"`, +`"gliner"` and `"smart"` add model spans to the same structured findings; +selecting an ML engine does not disable native card or credential detection. The native engine adds detectors the Python engine never had — `API_KEY`, `BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, @@ -95,7 +96,7 @@ and `ADDRESS` from an embedded model, so name and address detection needs no spaCy and no model download. Findings keep `engine="regex"` provenance and `confidence=1.0` as compatibility values, not native confidence estimates. Reviewed differences from the Python engine are listed in -[tests/contracts/rust-1.3.0.json](tests/contracts/rust-1.3.0.json). +[tests/contracts/rust-1.5.0.json](tests/contracts/rust-1.5.0.json). `datafog.native` exposes the native schema directly. Python 3.13 support is certified for the core SDK, CLI, `nlp`, @@ -151,10 +152,8 @@ import datafog text = "Steuer-ID 12345678901 liegt vor." -print(datafog.scan(text, engine="regex").entities) -# [] - -print(datafog.scan(text, engine="regex", locales=["de"]).entities) +print(datafog.scan(text, engine="regex", locales=["de"], + entity_types=["DE_TAX_ID"]).entities) # [Entity(type='DE_TAX_ID', text='12345678901', ...)] ``` @@ -215,7 +214,8 @@ scan/redact helpers, or guardrail helpers. DataFog deprecates OCR and Spark with visible use-time warnings; their APIs and extras will be removed in 5.0. The last supported -version is 4.10.0 — pin it with `pip install "datafog==4.10.0"` — and every +version in this release train is 4.12.0 — pin it with +`pip install "datafog==4.12.0"` — and every retired surface is tracked in the [retirement schedule](docs/retirement.rst). See the [4.9 migration guide](docs/migration-4.9.md) for the transition plan, and [VERSIONING.md](VERSIONING.md) for what a version number promises. @@ -245,6 +245,8 @@ print(result["PHONE"]) ## CLI +Install the command dependencies with `pip install "datafog[cli]"`. + ```bash # Scan text datafog scan-text "john@example.com" @@ -284,20 +286,20 @@ Telemetry does not include input text or detected PII values. ## Development -The [4.9 migration guide](docs/migration-4.9.md) explains opt-in Rust detection, -the native `datafog.v5` preview, and the revised 5.0 retirement schedule for -`detect`/`process`, OCR, and Spark. The Python detector remains the default. - -4.10.0 requires Core `>=1.3.0,<2` and capability contract 1, and installs it by -default. See the [4.10.0 release notes](RELEASE_NOTES_4.10.0.md) and the -[4.10 migration guide](docs/migration-4.10.md). -Entity labels, locales, and activation settings come from the installed -Core, allowing compatible releases to add detectors without a Python update. -German detection is opt-in through locale or entity selection; UUID is opt-in -through `entity_types=["UUID"]`. Core's structured-only `PERSON` is unavailable -for explicit Rust text selection. The legacy overlap policy can suppress NPI in -favor of PHONE; the migration guide explains native alternatives. Lock the Core -version if detection output must remain reproducible. +The native engine is the default, with a warned Python fallback. The 4.12 +review fixes preserve structured detection across every engine and legacy API. +DataFog 4.12 requires the reviewed Core 1.5 minor series +(`datafog-core~=1.5.0`) in the base package and the `rust` extra. The published +Core 1.5.0 wheel passed installed integration and compatibility qualification +with this release. Python and Core have independent release versions. +See [release preparation](docs/review-remediation.md). + +Entity labels, locales and activation settings come from the installed Core. +`get_supported_entities()` reports that backend's supported inventory, including +opt-in labels; it does not mean every label is enabled by default. German +detection is opt-in through locale or entity selection. `UUID`, `PERSON` and +`ADDRESS` can be selected explicitly. Unsupported labels and invalid arguments +raise errors instead of silently disabling detection. The [4.8.1 compatibility contract](docs/migration-4.8.1-contract.md) records published Python behavior for the Rust migration, with frozen fixtures and diff --git a/RELEASE_NOTES_4.12.0.md b/RELEASE_NOTES_4.12.0.md new file mode 100644 index 00000000..ad598fd8 --- /dev/null +++ b/RELEASE_NOTES_4.12.0.md @@ -0,0 +1,56 @@ +# DataFog 4.12.0 + +DataFog 4.12.0 carries structured PII detection through every engine, legacy +API and CLI anonymization path. Smart, spaCy and GLiNER modes retain the same +native card, IP and credential findings as the regex mode, then add NER spans. +TextService scans complete documents, and spaCy/GLiNER character offsets prevent +an earlier substring or repeated name from receiving the wrong redaction. + +This release requires `datafog-core~=1.5.0` in the base package and the `rust` +extra. Core 1.5 supplies complete IPv4-mapped IPv6 redaction, corrected email +field boundaries and Unicode mailboxes, and phone-plus-ZIP classification that +does not suppress the phone behind a false card finding. Native differences +are recorded in `tests/contracts/rust-1.5.0.json`. Capability contract 1 remains +unchanged; Python and Core have independent version numbers. + +Lone surrogate characters no longer disable native scans. Dense overlap +suppression, Python fallback patterns and allowlist matching have bounded work; +typoed entity labels, invalid ranges and invalid locales raise explicit errors. +Anonymizing pipeline operations imply scanning. Async guardrail decorators +await their result, and non-string results raise TypeError. + +LiteLLM scans nested tool/function arguments and prompt/input requests with a +shared token assignment for the request. Repeated values reuse tokens, and +existing markers do not collide. Streaming buffers the complete upstream +response before emitting its first redacted chunk, including text and tool +arguments split across deltas. This adds the full upstream completion latency +to first output. Streams exceeding eight million text characters or 100,000 +chunks are rejected. Engine errors retain the configured `open` or `closed` +failure policy. Image and binary parts remain opaque and receive no OCR. + +The Claude Code hook asks for review, or denies according to configuration, +when input exceeds its scan budget; it no longer silently permits an unscanned +tail. Unexpected hook failures retain the documented fail-open policy. + +Hash APIs honor the selected algorithm while preserving existing plain-hash +formats. Engine APIs, the DataFog class and CLI retain SHA-256 with 12 hex +characters by default. Deprecated top-level `datafog.process(method="hash")` +retains MD5 with eight hex characters. The legacy Anonymizer model retains its +existing full plain digest. Explicit `hash_key` adds full-length HMAC with at +least 32 key bytes; omitting the algorithm in keyed mode selects SHA-256. +Unkeyed hashes remain guessable deterministic pseudonyms, not protection +against enumerating SSNs, phone numbers or other low-entropy values. + +The minimum Pydantic version is 2.1. The `nlp-advanced` extra includes spaCy and +permits Torch 2.x versions compatible with Core's GLiNER2 extra. Base installs +show a CLI-extra installation hint instead of a missing-Typer traceback. +Unused duplicate implementations are removed, source distributions include +runnable tests and corpora, and CLI configuration output hides API keys. + +Scheduled release jobs qualify candidates without publishing. Manual jobs +default to dry runs, reject stale or occupied versions, and record immutable +release refs before uploading artifacts. + +The published Core 1.5.0 dependency passed installed-wheel qualification with +this Python release, including the native bridge and compatibility contracts. +The Python and Core versions are published independently. diff --git a/VERSIONING.md b/VERSIONING.md index 9f511c41..802af079 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -96,9 +96,9 @@ for `datafog-core`, and requires a matching floor in `datafog`. Both are exposed separately so a bug report is attributable: ```python -import datafog, datafog.core +import datafog, datafog.native datafog.__version__ # application version -datafog.core.__version__ # engine version +datafog.native.version() # engine version ``` ## Yanking diff --git a/datafog/__about__.py b/datafog/__about__.py index bdb8bda6..ade1e2b4 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.11.0" +__version__ = "4.12.0" diff --git a/datafog/__init__.py b/datafog/__init__.py index b0c7f2a8..467f5a70 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -157,7 +157,7 @@ def scan( *, backend: str | None = None, ) -> ScanResult: - """Scan using the 4.x compatibility API; Rust detection is opt-in.""" + """Scan using the 4.x compatibility API; native detection is the default.""" return _compat_v4.scan( text, engine, @@ -181,6 +181,8 @@ def redact( allowlist_patterns: list[str] | None = None, *, backend: str | None = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Redact using the 4.x compatibility API and legacy strategies.""" return _compat_v4.redact( @@ -194,6 +196,8 @@ def redact( allowlist, allowlist_patterns, backend=backend, + hash_type=hash_type, + hash_key=hash_key, ) @@ -242,18 +246,15 @@ def _detect_impl(text: str) -> list: _start = _time.monotonic() - _lazy_import_regex_annotator() - annotator = RegexAnnotator() - # Use the structured output to get proper positions - _, result = annotator.annotate_with_spans(text) + result = scan(text) # Convert to simple format, filtering out empty matches entities = [] - for span in result.spans: + for span in result.entities: if span.text.strip(): # Only include non-empty matches entities.append( { - "type": span.label, + "type": span.type, "value": span.text, "start": span.start, "end": span.end, @@ -284,7 +285,14 @@ def _detect_impl(text: str) -> list: return entities -def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: +def process( + text: str, + anonymize: bool = False, + method: str = "redact", + *, + hash_type: str | None = None, + hash_key: bytes | None = None, +) -> dict: """ Process text to detect and optionally anonymize PII. @@ -292,6 +300,9 @@ def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: text: Input text to process anonymize: Whether to anonymize detected PII method: Anonymization method ('redact', 'replace', 'hash') + hash_type: Optional algorithm. The legacy plain-hash default is MD5; + keyed hashes default to SHA256. + hash_key: At least 32 bytes for a full keyed HMAC digest. Returns: Dictionary with original text, anonymized text (if requested), and findings @@ -311,31 +322,37 @@ def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: _start = _time.monotonic() + if method not in {"redact", "replace", "hash"}: + raise ValueError("method must be 'redact', 'replace', or 'hash'") + if anonymize and method == "hash": + from .engine import _hash_value, _validate_hash_parameters + + algorithm = ( + hash_type + if hash_type is not None + else ("sha256" if hash_key is not None else "md5") + ) + _validate_hash_parameters("hash", algorithm, hash_key) findings = _detect_impl(text) result = {"original": text, "findings": findings} if anonymize: - anonymized = text - # Simple anonymization - replace from end to start to preserve positions - for finding in sorted(findings, key=lambda x: x["start"], reverse=True): - start, end = finding["start"], finding["end"] - entity_type = finding["type"] - - if method == "redact": - replacement = f"[{entity_type}_REDACTED]" - elif method == "replace": - replacement = f"[{entity_type}_XXXXX]" - elif method == "hash": - import hashlib - - replacement = f"[{entity_type}_{hashlib.md5(finding['value'].encode()).hexdigest()[:8]}]" + parts = [] + cursor = 0 + suffix = "REDACTED" if method == "redact" else "XXXXX" + for finding in sorted(findings, key=lambda item: item["start"]): + if method == "hash": + digest = _hash_value( + finding["value"], algorithm, hash_key, plain_length=8 + ) + replacement = f"[{finding['type']}_{digest}]" else: - replacement = f"[{entity_type}]" - - anonymized = anonymized[:start] + replacement + anonymized[end:] - - result["anonymized"] = anonymized + replacement = f"[{finding['type']}_{suffix}]" + parts.extend((text[cursor : finding["start"]], replacement)) + cursor = finding["end"] + parts.append(text[cursor:]) + result["anonymized"] = "".join(parts) try: from .telemetry import _get_duration_bucket, track_function_call diff --git a/datafog/__init___lean.py b/datafog/__init___lean.py deleted file mode 100644 index 3f1165b8..00000000 --- a/datafog/__init___lean.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Historical shadow package export module. - -This file is not the live DataFog 4.5 package export surface. Use -``datafog/__init__.py`` for current package exports, lazy optional imports, -top-level helpers, and compatibility shims. This snapshot remains importable -only as historical reference until legacy cleanup can remove it safely. - -DataFog: Lightning-fast PII detection and anonymization library. - -Core package provides regex-based PII detection, 100x+ faster than NER-based -detection on identical payloads (reproduce: python benchmarks/run.py). -Optional extras available for advanced features: -- pip install datafog[nlp] - for spaCy integration -- pip install datafog[ocr] - for image/OCR processing -- pip install datafog[all] - for all features -""" - -from .__about__ import __version__ - -# Core imports - always available -from .models.annotator import AnnotationResult, AnnotatorRequest -from .models.anonymizer import ( - AnonymizationResult, - Anonymizer, - AnonymizerRequest, - AnonymizerType, -) -from .models.common import EntityTypes -from .processing.text_processing.regex_annotator import RegexAnnotator - -# Optional imports with graceful fallback -try: - from .client import app -except ImportError: - app = None - -try: - from .main import DataFog, TextPIIAnnotator -except ImportError: - DataFog = None - TextPIIAnnotator = None - -try: - from .services.text_service import TextService -except ImportError: - TextService = None - - -# Optional heavy features - only import if dependencies available -def _optional_import(name, module_path, extra_name): - """Helper to import optional modules with helpful error messages.""" - try: - module = __import__(module_path, fromlist=[name]) - return getattr(module, name) - except ImportError: - - def _missing_dependency(*args, **kwargs): - raise ImportError( - f"{name} requires additional dependencies. " - f"Install with: pip install datafog[{extra_name}]" - ) - - return _missing_dependency - - -# OCR/Image processing - requires 'ocr' extra -DonutProcessor = _optional_import( - "DonutProcessor", "datafog.processing.image_processing.donut_processor", "ocr" -) -PytesseractProcessor = _optional_import( - "PytesseractProcessor", - "datafog.processing.image_processing.pytesseract_processor", - "ocr", -) -ImageService = _optional_import("ImageService", "datafog.services.image_service", "ocr") - -# NLP processing - requires 'nlp' extra -SpacyPIIAnnotator = _optional_import( - "SpacyPIIAnnotator", "datafog.processing.text_processing.spacy_pii_annotator", "nlp" -) - -# Distributed processing - requires 'distributed' extra -SparkService = _optional_import( - "SparkService", "datafog.services.spark_service", "distributed" -) - - -# Simple API for core functionality -def detect(text: str) -> list: - """ - Detect PII in text using regex patterns. - - Args: - text: Input text to scan for PII - - Returns: - List of detected PII entities - - Example: - >>> from datafog import detect - >>> detect("Contact john@example.com") - [{'type': 'EMAIL', 'value': 'john@example.com', 'start': 8, 'end': 24}] - """ - annotator = RegexAnnotator() - result = annotator.annotate(text) - - # Convert to simple format - entities = [] - for entity_type, matches in result.items(): - for match in matches: - entities.append( - { - "type": entity_type, - "value": match, - "start": text.find(match), - "end": text.find(match) + len(match), - } - ) - - return entities - - -def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: - """ - Process text to detect and optionally anonymize PII. - - Args: - text: Input text to process - anonymize: Whether to anonymize detected PII - method: Anonymization method ('redact', 'replace', 'hash') - - Returns: - Dictionary with original text, anonymized text (if requested), and findings - - Example: - >>> from datafog import process - >>> process("Contact john@example.com", anonymize=True) - { - 'original': 'Contact john@example.com', - 'anonymized': 'Contact [EMAIL_REDACTED]', - 'findings': [{'type': 'EMAIL', 'value': 'john@example.com', ...}] - } - """ - findings = detect(text) - - result = {"original": text, "findings": findings} - - if anonymize: - anonymized = text - # Simple anonymization - replace from end to start to preserve positions - for finding in sorted(findings, key=lambda x: x["start"], reverse=True): - start, end = finding["start"], finding["end"] - entity_type = finding["type"] - - if method == "redact": - replacement = f"[{entity_type}_REDACTED]" - elif method == "replace": - replacement = f"[{entity_type}_XXXXX]" - elif method == "hash": - import hashlib - - replacement = f"[{entity_type}_{hashlib.md5(finding['value'].encode()).hexdigest()[:8]}]" - else: - replacement = f"[{entity_type}]" - - anonymized = anonymized[:start] + replacement + anonymized[end:] - - result["anonymized"] = anonymized - - return result - - -# Core exports -__all__ = [ - "__version__", - "detect", - "process", - "AnnotationResult", - "AnnotatorRequest", - "AnonymizationResult", - "Anonymizer", - "AnonymizerRequest", - "AnonymizerType", - "EntityTypes", - "RegexAnnotator", - # Optional exports (may be None if dependencies missing) - "DataFog", - "TextPIIAnnotator", - "TextService", - "app", - "DonutProcessor", - "PytesseractProcessor", - "ImageService", - "SpacyPIIAnnotator", - "SparkService", -] diff --git a/datafog/__init___original.py b/datafog/__init___original.py deleted file mode 100644 index 380511b1..00000000 --- a/datafog/__init___original.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Historical shadow package export module. - -This file is not the live DataFog 4.5 package export surface. Use -``datafog/__init__.py`` for current package exports, lazy optional imports, -top-level helpers, and compatibility shims. This eager-export snapshot remains -importable only as historical reference until legacy cleanup can remove it -safely. -""" - -from .__about__ import __version__ -from .client import app -from .config import OperationType, get_config -from .main import DataFog, TextPIIAnnotator -from .models.annotator import ( - AnalysisExplanation, - AnnotationResult, - AnnotationResultWithAnaysisExplanation, - AnnotatorRequest, -) -from .models.anonymizer import ( - AnonymizationResult, - Anonymizer, - AnonymizerRequest, - AnonymizerType, -) -from .models.common import AnnotatorMetadata, EntityTypes, Pattern, PatternRecognizer -from .models.spacy_nlp import SpacyAnnotator -from .processing.image_processing.donut_processor import DonutProcessor -from .processing.image_processing.image_downloader import ImageDownloader -from .processing.image_processing.pytesseract_processor import PytesseractProcessor -from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator -from .services.image_service import ImageService -from .services.spark_service import SparkService -from .services.text_service import TextService - -__all__ = [ - "DonutProcessor", - "DataFog", - "ImageService", - "OperationType", - "SparkService", - "TextPIIAnnotator", - "TextService", - "SpacyPIIAnnotator", - "ImageDownloader", - "PytesseractProcessor", - "__version__", - "app", - "AnalysisExplanation", - "AnnotationResult", - "AnnotationResultWithAnaysisExplanation", - "AnnotatorRequest", - "AnnotatorMetadata", - "EntityTypes", - "Pattern", - "PatternRecognizer", - "get_config", - "SpacyAnnotator", - "AnonymizerType", - "AnonymizerRequest", - "AnonymizationResult", - "Anonymizer", -] diff --git a/datafog/_legacy_retirement.py b/datafog/_legacy_retirement.py index e86506db..41753955 100644 --- a/datafog/_legacy_retirement.py +++ b/datafog/_legacy_retirement.py @@ -2,8 +2,10 @@ import warnings +from .__about__ import __version__ + #: Final release on which retired optional surfaces still work. -LAST_SUPPORTED = "4.10.0" +LAST_SUPPORTED = __version__ class LegacySurfaceWarning(FutureWarning): diff --git a/datafog/agent.py b/datafog/agent.py index e3cd7bdb..d8bfe56e 100644 --- a/datafog/agent.py +++ b/datafog/agent.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect import warnings from contextlib import contextmanager from dataclasses import dataclass @@ -101,14 +102,27 @@ def filter(self, text: str) -> RedactResult: return result def __call__(self, fn: F) -> F: - """Decorator that applies guardrail filtering to string return values.""" + """Filter string results, awaiting coroutine functions before scanning. + + Other result types raise TypeError rather than escaping unscanned. + """ + + def filtered(output: Any) -> str: + if not isinstance(output, str): + raise TypeError("A guarded function must return a string") + return self.filter(output).redacted_text + + if inspect.iscoroutinefunction(fn): + + @wraps(fn) + async def async_wrapped(*args: Any, **kwargs: Any) -> str: + return filtered(await fn(*args, **kwargs)) + + return async_wrapped # type: ignore[return-value] @wraps(fn) def wrapped(*args: Any, **kwargs: Any) -> Any: - output = fn(*args, **kwargs) - if isinstance(output, str): - return self.filter(output).redacted_text - return output + return filtered(fn(*args, **kwargs)) return wrapped # type: ignore[return-value] diff --git a/datafog/cli.py b/datafog/cli.py new file mode 100644 index 00000000..d5a86302 --- /dev/null +++ b/datafog/cli.py @@ -0,0 +1,17 @@ +"""Console bootstrap with an actionable optional dependency error.""" + + +def main(): + try: + import typer # noqa: F401 + except ImportError: + import sys + + print( + 'The datafog command requires the CLI extra: pip install "datafog[cli]"', + file=sys.stderr, + ) + raise SystemExit(1) from None + from .client import app + + app() diff --git a/datafog/client.py b/datafog/client.py index fd9a303d..cde33431 100644 --- a/datafog/client.py +++ b/datafog/client.py @@ -46,6 +46,15 @@ def list_entities(): app = typer.Typer() +def _operations(value: str) -> list[OperationType]: + try: + return [OperationType(op.strip()) for op in value.split(",")] + except ValueError: + raise typer.BadParameter( + "operations must use scan, extract, redact, replace, or hash" + ) from None + + @app.command() def scan_image( image_urls: List[str] = typer.Argument( @@ -70,9 +79,8 @@ def scan_image( typer.echo("No image URLs or file paths provided. Please provide at least one.") raise typer.Exit(code=1) - logging.basicConfig(level=logging.INFO) # Convert comma-separated string operations to a list of OperationType objects - operation_list = [OperationType(op.strip()) for op in operations.split(",")] + operation_list = _operations(operations) ocr_client = DataFog(operations=operation_list) try: results = asyncio.run(ocr_client.run_ocr_pipeline(image_urls=image_urls)) @@ -130,9 +138,8 @@ def scan_text( typer.echo("No texts provided.") raise typer.Exit(code=1) - logging.basicConfig(level=logging.INFO) # Convert comma-separated string operations to a list of OperationType objects - operation_list = [OperationType(op.strip()) for op in operations.split(",")] + operation_list = _operations(operations) text_client = DataFog(operations=operation_list, locales=locale) try: results = text_client.run_text_pipeline_sync(str_list=str_list) @@ -179,7 +186,11 @@ def show_config(): Prints the current DataFog configuration. """ - typer.echo(get_config()) + config = get_config() + values = dict(config) if isinstance(config, dict) else config.model_dump() + if values.get("api_key"): + values["api_key"] = "[REDACTED]" + typer.echo(values) @app.command() @@ -432,12 +443,11 @@ def hash_text( typer.echo("No text provided to hash.") raise typer.Exit(code=1) - # HashType is retained for backward-compatible CLI signature. - _ = hash_type result = scan_and_redact( text=text, engine="smart", strategy="hash", + hash_type=hash_type.value, locales=locale, ) typer.echo(result.redacted_text) diff --git a/datafog/compat/v4.py b/datafog/compat/v4.py index 96d619aa..ce6f00f4 100644 --- a/datafog/compat/v4.py +++ b/datafog/compat/v4.py @@ -66,6 +66,8 @@ def redact( allowlist_patterns: list[str] | None = None, *, backend: str | None = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """ Redact with legacy strategies and result schema. @@ -92,7 +94,13 @@ def redact( "allowlist/allowlist_patterns cannot be combined with explicit " "entities; filter the entities before calling redact" ) - return _redact_entities(text=text, entities=entities, strategy=strategy) + return _redact_entities( + text=text, + entities=entities, + strategy=strategy, + hash_type=hash_type, + hash_key=hash_key, + ) return _scan_and_redact( text=text, @@ -103,4 +111,6 @@ def redact( allowlist=allowlist, allowlist_patterns=allowlist_patterns, backend=backend, + hash_type=hash_type, + hash_key=hash_key, ) diff --git a/datafog/core.py b/datafog/core.py index e56c2aa5..2153ee94 100644 --- a/datafog/core.py +++ b/datafog/core.py @@ -89,6 +89,9 @@ def anonymize_text( text: str, method: Union[str, AnonymizerType] = "redact", locales: List[str] | None = None, + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> str: """ Simple text anonymization using lightweight regex engine. @@ -103,7 +106,7 @@ def anonymize_text( Example: >>> result = anonymize_text("Contact john@example.com", method="redact") >>> print(result) - "Contact [EMAIL_REDACTED]" + "Contact [EMAIL_1]" """ import time as _time @@ -129,6 +132,8 @@ def anonymize_text( engine=REGEX_ENGINE, strategy=strategy_map[method], locales=locales, + hash_type=hash_type, + hash_key=hash_key, ) try: @@ -211,18 +216,29 @@ def scan_text( return result -def get_supported_entities(locales: List[str] | None = None) -> List[str]: +def get_supported_entities( + locales: List[str] | None = None, *, backend: str | None = None +) -> List[str]: """ - Get list of PII entity types supported by the regex engine. + Get the selected backend's supported entity types, including opt-in labels. Returns: List of supported entity type names Example: >>> entities = get_supported_entities() - >>> print(entities) - ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DATE', 'ZIP_CODE'] + >>> "EMAIL" in entities + True """ + from datafog.engine import _require_core, resolve_backend + + if resolve_backend(backend, "regex") == "rust": + from datafog._core_capabilities import _read_capabilities, scan_configs + + core = _require_core() + supported, _, _ = _read_capabilities(core.capabilities) + scan_configs(core, set(), locales) + return sorted(supported) annotator = RegexAnnotator(locales=locales) legacy_map = {"DOB": "DATE", "ZIP": "ZIP_CODE"} result = [] diff --git a/datafog/engine.py b/datafog/engine.py index 495c2d18..c8041acf 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import hmac import os import re import warnings @@ -10,6 +11,8 @@ from functools import lru_cache from typing import Optional +import regex as bounded_re + from .exceptions import CoreUnavailableWarning, EngineNotAvailable from .processing.text_processing.regex_annotator import RegexAnnotator @@ -66,6 +69,10 @@ "US_ROUTING_NUMBER", "NPI", "UUID", + "PASSPORT_NUMBER", + "MEDICAL_RECORD_NUMBER", + "LICENSE_NUMBER", + "URL", } #: Labels the native rule engine reports as structured-only, served for text by @@ -78,7 +85,18 @@ _core_warned = False _cnn_cache: tuple[object, object] | None = None -NER_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} +SPACY_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} + +NER_ENTITY_TYPES = { + "PERSON", + "ORGANIZATION", + "LOCATION", + "ADDRESS", + "PASSPORT_NUMBER", + "MEDICAL_RECORD_NUMBER", + "LICENSE_NUMBER", + "URL", +} ENTITY_TYPE_PRIORITY = { "DE_IBAN": 100, @@ -155,9 +173,13 @@ def _find_all_occurrences(text: str, needle: str) -> list[tuple[int, int]]: def _entities_from_dict( text: str, payload: dict[str, list[str]], engine: str, confidence: float ) -> list[Entity]: + """Compatibility for third-party annotators without offset support. + + Whole-word matching avoids interpreting "Ann" inside "Annual" as a name. + First-party annotators use their original character ranges instead. + """ entities: list[Entity] = [] value_offsets: dict[str, int] = {} - for raw_type, values in payload.items(): canonical_type = _canonical_type(raw_type) if canonical_type not in ALL_ENTITY_TYPES: @@ -165,25 +187,47 @@ def _entities_from_dict( for value in values: if not isinstance(value, str) or not value.strip(): continue - search_start = value_offsets.get(value, 0) - idx = text.find(value, search_start) - if idx < 0: - idx = text.find(value) - end = idx + len(value) if idx >= 0 else -1 - value_offsets[value] = end if end >= 0 else search_start + 1 + pattern = re.compile(r"(? list[Entity]: + method = getattr(annotator, "annotate_with_spans", None) + if not callable(method): + return _entities_from_dict(text, annotator.annotate(text), engine, confidence) + _, result = method(text) + entities = [] + for span in result.spans: + label = _canonical_type(span.label) + if label not in ALL_ENTITY_TYPES: + continue + if ( + not 0 <= span.start < span.end <= len(text) + or text[span.start : span.end] != span.text + ): + raise ValueError(f"{engine} returned an invalid entity span") + entities.append( + Entity(label, span.text, span.start, span.end, confidence, engine) + ) + return entities + + def _entity_length(entity: Entity) -> int: return max(entity.end - entity.start, 0) @@ -195,7 +239,31 @@ def _entities_overlap(left: Entity, right: Entity) -> bool: def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]: - selected: list[Entity] = [] + """Preserve longest-span precedence with O(n log n) interval queries.""" + coordinates = { + value: index + 1 + for index, value in enumerate( + sorted( + {value for entity in entities for value in (entity.start, entity.end)} + ) + ) + } + starts = [0] * (len(coordinates) + 1) + ends = [0] * (len(coordinates) + 1) + + def add(tree: list[int], index: int) -> None: + while index < len(tree): + tree[index] += 1 + index += index & -index + + def count(tree: list[int], index: int) -> int: + total = 0 + while index > 0: + total += tree[index] + index -= index & -index + return total + + selected = [] for entity in sorted( entities, key=lambda item: ( @@ -207,9 +275,12 @@ def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]: item.type, ), ): - if any(_entities_overlap(entity, kept) for kept in selected): + start, end = coordinates[entity.start], coordinates[entity.end] + if count(starts, end - 1) > count(ends, start): continue selected.append(entity) + add(starts, start) + add(ends, end) return sorted(selected, key=lambda item: (item.start, item.end, item.type)) @@ -289,15 +360,28 @@ def _cnn_detector_or_none() -> object | None: def _cnn_findings( detector: object, requested: set[str], text: str -) -> list[tuple[dict, object]]: - """Serve prose names and address blocks from the embedded model.""" - try: - result = detector.scan_model(text) - except Exception as exc: # relative-path/limit errors surface to the caller - raise _model_error(exc) from exc - return [ - ({}, finding) for finding in result.findings if finding.entity_type in requested - ] +) -> list[tuple[int, object]]: + """Bound model work while retaining document character coordinates. + + 4096 characters fit the native byte/token caps even for four-byte Unicode. + Overlap gives names and address blocks context on either side of a seam. + """ + chunk_length, overlap = 4096, 256 + findings = [] + for offset in range(0, max(len(text), 1), chunk_length - overlap): + chunk = text[offset : offset + chunk_length] + try: + result = detector.scan_model(chunk) + except Exception as exc: + raise _model_error(exc) from exc + findings.extend( + (offset, finding) + for finding in result.findings + if finding.entity_type in requested + ) + if offset + chunk_length >= len(text): + break + return findings def _model_error(exc: Exception) -> Exception: @@ -337,17 +421,18 @@ def _rust_entities( configs = scan_configs(datafog_core, requested - MODEL_ENTITY_TYPES, locales) entities: list[Entity] = [] seen: set[tuple[str, int, int]] = set() + native_text = re.sub(r"[\ud800-\udfff]", "\ufffd", text) findings = [ - (config, finding) + (0, finding) for config in configs - for finding in datafog_core.scan(text, config=config) + for finding in datafog_core.scan(native_text, config=config) ] if detector is not None: - findings.extend(_cnn_findings(detector, model_requested, text)) - for config, finding in findings: + findings.extend(_cnn_findings(detector, model_requested, native_text)) + for offset, finding in findings: canonical_type = _canonical_type(finding.entity_type) - start = finding.codepoint_range.start - end = finding.codepoint_range.end + start = offset + finding.codepoint_range.start + end = offset + finding.codepoint_range.end key = (canonical_type, start, end) if key in seen or not finding.matched_text.strip(): continue @@ -355,7 +440,7 @@ def _rust_entities( entities.append( Entity( type=canonical_type, - text=finding.matched_text, + text=text[start:end], start=start, end=end, confidence=1.0, @@ -369,16 +454,22 @@ def _spacy_entities(text: str) -> list[Entity]: annotator = _get_spacy_annotator() if isinstance(annotator, _UnavailableAnnotator): raise EngineNotAvailable(annotator.message) - payload = annotator.annotate(text) - return _entities_from_dict(text, payload, engine="spacy", confidence=0.7) + return [ + entity + for entity in _annotator_entities(text, annotator, "spacy", 0.7) + if entity.type in SPACY_ENTITY_TYPES + ] -def _gliner_entities(text: str) -> list[Entity]: - annotator = _get_gliner_annotator() +def _gliner_entities(text: str, model_name: str | None = None) -> list[Entity]: + annotator = ( + _get_gliner_annotator() + if model_name is None + else _get_gliner_annotator(model_name) + ) if isinstance(annotator, _UnavailableAnnotator): raise EngineNotAvailable(annotator.message) - payload = annotator.annotate(text) - return _entities_from_dict(text, payload, engine="gliner", confidence=0.8) + return _annotator_entities(text, annotator, "gliner", 0.8) @lru_cache(maxsize=1) @@ -398,15 +489,15 @@ def _get_spacy_annotator(): ) -@lru_cache(maxsize=1) -def _get_gliner_annotator(): +@lru_cache(maxsize=4) +def _get_gliner_annotator(model_name: str = "urchade/gliner_multi_pii-v1"): try: from .processing.text_processing.gliner_annotator import GLiNERAnnotator except ImportError as exc: return _UnavailableAnnotator(str(exc)) try: - annotator = GLiNERAnnotator.create() + annotator = GLiNERAnnotator.create(model_name=model_name) except ImportError as exc: return _UnavailableAnnotator(str(exc)) except Exception as exc: @@ -438,44 +529,39 @@ def _filter_entity_types( return [entity for entity in entities if entity.type in allowed] -# Python's re module backtracks; a quantified group containing another -# quantifier (e.g. ``(a+)+``) can take exponential time on adversarial -# input, and entity text can be attacker-influenced (LLM messages, tool -# output). Reject that construct outright rather than matching under it. -_NESTED_QUANTIFIER = re.compile( - r"\((?:[^()\\]|\\.)*(? list["re.Pattern[str]"]: +def _compile_allowlist_patterns(allowlist_patterns: Optional[list[str]]) -> list: compiled = [] + if isinstance(allowlist_patterns, str): + raise ValueError("allowlist_patterns must be a list of patterns") for raw in allowlist_patterns or []: - if len(raw) > MAX_ALLOWLIST_PATTERN_LENGTH: + if not isinstance(raw, str) or len(raw) > MAX_ALLOWLIST_PATTERN_LENGTH: raise ValueError( - "allowlist_patterns entries must be at most " - f"{MAX_ALLOWLIST_PATTERN_LENGTH} characters" - ) - if _NESTED_QUANTIFIER.search(raw): - raise ValueError( - "allowlist_patterns contains a quantified group with a nested " - f"quantifier ({raw!r}), which risks catastrophic backtracking; " - "rewrite the pattern without nesting quantifiers" + "allowlist_patterns entries must be strings of at most 512 characters" ) try: - compiled.append(re.compile(raw)) - except re.error as exc: + compiled.append(bounded_re.compile(raw)) + except bounded_re.error as exc: raise ValueError( f"allowlist_patterns contains an invalid regex: {raw!r} ({exc})" ) from None return compiled +def _allowlist_matches(pattern: object, value: str) -> bool: + if len(value) > MAX_PATTERN_SUBJECT_LENGTH: + return False + try: + return pattern.fullmatch(value, timeout=ALLOWLIST_MATCH_TIMEOUT) is not None + except TimeoutError: + # A slow exemption must never suppress a sensitive finding. + return False + + def _apply_allowlist( entities: list[Entity], allowlist: Optional[list[str]], @@ -498,11 +584,7 @@ def _apply_allowlist( entity for entity in entities if entity.text not in exact - and not any( - pattern.fullmatch(entity.text) - for pattern in patterns - if len(entity.text) <= MAX_PATTERN_SUBJECT_LENGTH - ) + and not any(_allowlist_matches(pattern, entity.text) for pattern in patterns) ] @@ -532,8 +614,7 @@ def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: An explicit ``backend`` wins over ``DATAFOG_BACKEND``. ``auto`` uses the native engine whenever Core is importable and degrades to Python otherwise. - Non-``regex`` engines always take the Python path, because the native - backend serves ``regex`` and model findings only. + Every engine uses this backend for structured PII; NER adds spans. """ requested = backend or os.environ.get(BACKEND_ENV) or "auto" normalized = requested.strip().lower() @@ -543,8 +624,6 @@ def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: "backend argument)" ) if normalized == "auto": - if engine != "regex": - return "python" return "rust" if core_available() else "python" return normalized @@ -552,8 +631,35 @@ def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: def _validate_backend(backend: str, engine: str) -> None: if backend not in {"python", "rust"}: raise ValueError("backend must be one of: python, rust") - if backend == "rust" and engine != "regex": - raise ValueError("backend='rust' supports only engine='regex'") + + +def _validate_entity_types( + entity_types: Optional[list[str]], backend: str, engine: str +) -> Optional[list[str]]: + if entity_types is None: + return None + if not isinstance(entity_types, (list, tuple)) or any( + not isinstance(value, str) or not value.strip() for value in entity_types + ): + raise ValueError("entity_types must be a list of entity labels") + requested = [_canonical_type(value) for value in entity_types] + if backend == "rust": + from ._core_capabilities import _read_capabilities + + supported, _, _ = _read_capabilities(_require_core().capabilities) + supported = supported | MODEL_ENTITY_TYPES + else: + supported = {_canonical_type(label) for label in RegexAnnotator.LABELS} + if engine == "spacy": + supported |= SPACY_ENTITY_TYPES + elif engine in {"smart", "gliner"}: + supported |= NER_ENTITY_TYPES | {"DATE"} + unsupported = set(requested) - supported + if unsupported: + raise ValueError( + f"Unsupported entity_types for {backend}/{engine}: {', '.join(sorted(unsupported))}" + ) + return requested def scan( @@ -565,14 +671,16 @@ def scan( allowlist_patterns: Optional[list[str]] = None, *, backend: Optional[str] = None, + gliner_model: Optional[str] = None, ) -> ScanResult: """Scan text for PII entities. ``backend`` defaults to ``auto``: the native engine when ``datafog-core`` is importable, otherwise the pure-Python engine with a one-time warning. Set ``DATAFOG_BACKEND`` or pass ``backend="python"`` to force the fallback. The - native path serves ``engine="regex"``; non-regex engines always use Python. - Probe names and address blocks come from the embedded model on the native + selected backend supplies structured PII for every engine mode; optional + NER engines add entity spans. + Prose names and address blocks come from the embedded model on the native path, so ``PERSON`` and ``ADDRESS`` work without spaCy or a model download. Results retain legacy regex provenance and confidence; these are @@ -593,6 +701,7 @@ def scan( ) backend = resolve_backend(backend, engine) _validate_backend(backend, engine) + entity_types = _validate_entity_types(entity_types, backend, engine) if backend == "python" and requested_backend == "auto" and not core_available(): _warn_core_missing() @@ -634,7 +743,11 @@ def scan( if engine == "gliner" and _needs_ner(entity_types): try: - gliner_entities = _gliner_entities(text) + gliner_entities = ( + _gliner_entities(text) + if gliner_model is None + else _gliner_entities(text, gliner_model) + ) combined.extend(gliner_entities) engines_used.add("gliner") except EngineNotAvailable: @@ -649,13 +762,16 @@ def scan( if engine == "smart" and _needs_ner(entity_types): try: - gliner_entities = _gliner_entities(text) + gliner_entities = ( + _gliner_entities(text) + if gliner_model is None + else _gliner_entities(text, gliner_model) + ) combined.extend(gliner_entities) engines_used.add("gliner") - except EngineNotAvailable: + except EngineNotAvailable as exc: warnings.warn( - "GLiNER not available, smart scan falling back to spaCy. " - "Install with: pip install datafog[nlp-advanced]", + f"GLiNER unavailable; smart scan falling back to spaCy. {exc}", UserWarning, stacklevel=2, ) @@ -673,6 +789,8 @@ def scan( filtered = _filter_entity_types(combined, entity_types) filtered = _apply_allowlist(filtered, allowlist, allowlist_patterns) + # Optional NER adds candidates without suppressing structured findings. + # Transformation resolves overlaps after selection, preserving scan labels. deduped = _dedupe_entities(filtered) return ScanResult( entities=deduped, @@ -681,10 +799,62 @@ def scan( ) +def _validate_hash_parameters( + strategy: str, hash_type: str, hash_key: bytes | None +) -> None: + if hash_key is not None and (not isinstance(hash_key, bytes) or len(hash_key) < 32): + raise ValueError("hash_key must contain at least 32 bytes") + if strategy == "hash": + try: + hashlib.new(hash_type).hexdigest() + except (ValueError, TypeError): + raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None + + +def _hash_value( + text: str, + hash_type: str, + hash_key: bytes | None, + *, + plain_length: int = 12, +) -> str: + """Retain legacy plain-hash widths; keyed HMAC always uses its full digest. + + Unkeyed hashes are deterministic pseudonyms and remain vulnerable to + dictionary attacks regardless of digest width. Supply a private key to + prevent direct enumeration of low-entropy identifiers. + """ + _validate_hash_parameters("hash", hash_type, hash_key) + encoded = text.encode("utf-8", "surrogatepass") + if hash_key is not None: + return hmac.new(hash_key, encoded, hash_type).hexdigest() + return hashlib.new(hash_type, encoded).hexdigest()[:plain_length] + + +def _validated_redaction_entities(text: str, entities: list[Entity]) -> list[Entity]: + valid_entities = [] + for entity in entities: + if ( + not 0 <= entity.start < entity.end <= len(text) + or not entity.text + or text[entity.start : entity.end] != entity.text + ): + raise ValueError( + "entities must contain valid ranges whose text matches the source" + ) + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", entity.type): + raise ValueError("entity types must be uppercase labels") + valid_entities.append(entity) + return _suppress_overlapping_entities(valid_entities) + + def redact( text: str, entities: list[Entity], strategy: str = "token", + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Redact PII entities from text. @@ -694,6 +864,12 @@ def redact( type priority and then confidence. Suppressed spans are omitted from ``RedactResult.entities`` and ``mapping``. + Plain hashes retain the historical 12-character digest prefix and provide + deterministic pseudonymization. With ``hash_key`` (at least 32 bytes), HMAC + uses the complete digest. Unkeyed hashes do not resist dictionary attacks. + + Invalid ranges, mismatched source text, and malformed labels raise ValueError. + ``mapping`` for the ``mask`` strategy is keyed by per-type indexed keys (``[EMAIL_MASK_1]``) rather than the mask string, because distinct same-length values produce identical masks and would collide. @@ -703,22 +879,19 @@ def redact( if strategy not in {"token", "mask", "hash", "pseudonymize"}: raise ValueError("strategy must be one of: token, mask, hash, pseudonymize") - redacted_text = text + _validate_hash_parameters(strategy, hash_type, hash_key) + mapping: dict[str, str] = {} counters: dict[str, int] = {} pseudonym_by_value: dict[tuple[str, str], str] = {} - valid_entities = [ - entity - for entity in entities - if 0 <= entity.start < entity.end <= len(text) and entity.text - ] - # Returns non-overlapping spans sorted by document position. - valid_entities = _suppress_overlapping_entities(valid_entities) + valid_entities = _validated_redaction_entities(text, entities) + + occupied_tokens = set(re.findall(r"\[[A-Z][A-Za-z0-9_]*\]", text)) # Assign replacements in document order so the per-type counters used by - # the token/pseudonymize strategies number the first occurrence _1; spans - # are applied right-to-left below so earlier offsets stay valid. + # the token/pseudonymize strategies number the first occurrence _1. + # Assemble the result in one pass without repeated document copies. replacements: list[tuple[Entity, str]] = [] for entity in valid_entities: original = text[entity.start : entity.end] @@ -727,30 +900,57 @@ def redact( replacement = "*" * max(len(original), 1) # Distinct same-length values share a mask, so the mask string # cannot key the mapping without collisions. - counters[entity.type] = counters.get(entity.type, 0) + 1 - mapping_key = f"[{entity.type}_MASK_{counters[entity.type]}]" + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + mapping_key = f"[{entity.type}_MASK_{counters[entity.type]}]" + if mapping_key not in occupied_tokens: + break + occupied_tokens.add(mapping_key) elif strategy == "hash": - digest = hashlib.sha256(original.encode("utf-8")).hexdigest()[:12] - replacement = f"[{entity.type}_{digest}]" + digest = _hash_value(original, hash_type, hash_key) + key = (entity.type, original) + if key not in pseudonym_by_value: + candidate = f"[{entity.type}_{digest}]" + suffix = 1 + while candidate in occupied_tokens: + suffix += 1 + candidate = f"[{entity.type}_{digest}_{suffix}]" + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) + replacement = pseudonym_by_value[key] elif strategy == "pseudonymize": key = (entity.type, original) if key not in pseudonym_by_value: - counters[entity.type] = counters.get(entity.type, 0) + 1 - pseudonym_by_value[key] = ( - f"[{entity.type}_PSEUDO_{counters[entity.type]}]" - ) + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + candidate = f"[{entity.type}_PSEUDO_{counters[entity.type]}]" + if candidate not in occupied_tokens: + break + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) replacement = pseudonym_by_value[key] else: # token - counters[entity.type] = counters.get(entity.type, 0) + 1 - replacement = f"[{entity.type}_{counters[entity.type]}]" + key = (entity.type, original) + if key not in pseudonym_by_value: + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + candidate = f"[{entity.type}_{counters[entity.type]}]" + if candidate not in occupied_tokens: + break + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) + replacement = pseudonym_by_value[key] replacements.append((entity, replacement)) mapping[mapping_key if mapping_key is not None else replacement] = original - for entity, replacement in reversed(replacements): - redacted_text = ( - redacted_text[: entity.start] + replacement + redacted_text[entity.end :] - ) + parts = [] + offset = 0 + for entity, replacement in replacements: + parts.extend((text[offset : entity.start], replacement)) + offset = entity.end + parts.append(text[offset:]) + redacted_text = "".join(parts) return RedactResult( redacted_text=redacted_text, @@ -769,6 +969,8 @@ def scan_and_redact( allowlist_patterns: Optional[list[str]] = None, *, backend: Optional[str] = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Scan with the selected backend, then apply legacy Python redaction. @@ -783,4 +985,10 @@ def scan_and_redact( allowlist_patterns=allowlist_patterns, backend=backend, ) - return redact(text=text, entities=scan_result.entities, strategy=strategy) + return redact( + text=text, + entities=scan_result.entities, + strategy=strategy, + hash_type=hash_type, + hash_key=hash_key, + ) diff --git a/datafog/integrations/claude_code.py b/datafog/integrations/claude_code.py index 28c4e787..5eb72fb3 100644 --- a/datafog/integrations/claude_code.py +++ b/datafog/integrations/claude_code.py @@ -22,8 +22,9 @@ whose full text matches are exempt (note: a pattern containing a comma cannot be expressed here). -Failure policy: fail open. A hook bug must never brick a Claude Code -session, so any unexpected error exits non-blocking with no output. +Oversized inputs receive the configured ask/deny decision rather than proceeding +with an unscanned tail. Unexpected hook errors fail open with a non-blocking +error, so a hook bug does not prevent using the Claude Code session. """ import json @@ -50,6 +51,10 @@ _EXIT_FAIL_OPEN = 1 +class ScanLimitExceeded(ValueError): + """The payload cannot be fully inspected within the configured budget.""" + + def _entity_types(env: Mapping[str, str]) -> list[str]: raw = env.get("DATAFOG_HOOK_ENTITIES", "") parsed = [t.strip().upper() for t in raw.split(",") if t.strip()] @@ -82,6 +87,7 @@ def _iter_strings(value: Any) -> Iterator[str]: yield current elif isinstance(current, dict): stack.extend(current.values()) + stack.extend(current.keys()) elif isinstance(current, (list, tuple)): stack.extend(current) @@ -98,9 +104,9 @@ def _scan_findings( counts: dict[str, int] = {} total_budget = TOTAL_SCAN_CHARS for text in _iter_strings(value): - if total_budget <= 0: - break - chunk = text[: min(MAX_SCAN_CHARS, total_budget)] + if len(text) > MAX_SCAN_CHARS or len(text) > total_budget: + raise ScanLimitExceeded("Input exceeds the scan budget") + chunk = text total_budget -= len(chunk) result = datafog.scan( chunk, @@ -191,8 +197,20 @@ def run(payload: dict, env: Mapping[str, str]) -> tuple[int, str]: if handler is None: return _EXIT_OK, "" return _EXIT_OK, handler(payload, env) + except ScanLimitExceeded: + event = payload.get("hook_event_name") + reason = "DataFog PII firewall: input exceeds the scan budget and was not fully scanned. Split the input before proceeding." + if event == "PreToolUse": + return _EXIT_OK, _emit( + event, + { + "permissionDecision": _action(env), + "permissionDecisionReason": reason, + }, + ) + return _EXIT_OK, _emit(event, {"additionalContext": reason}) except Exception as exc: # noqa: BLE001 — fail open by design - print(f"datafog-hook error (fail-open): {exc}", file=sys.stderr) + print(f"datafog-hook error (fail-open): {type(exc).__name__}", file=sys.stderr) return _EXIT_FAIL_OPEN, "" @@ -206,7 +224,10 @@ def main() -> None: if not isinstance(payload, dict): payload = {} except Exception as exc: # noqa: BLE001 — fail open by design - print(f"datafog-hook: invalid hook payload (fail-open): {exc}", file=sys.stderr) + print( + f"datafog-hook: invalid hook payload (fail-open): {type(exc).__name__}", + file=sys.stderr, + ) sys.exit(_EXIT_FAIL_OPEN) code, stdout = run(payload, os.environ) diff --git a/datafog/integrations/litellm_guardrail.py b/datafog/integrations/litellm_guardrail.py index 7e33ed55..74a56169 100644 --- a/datafog/integrations/litellm_guardrail.py +++ b/datafog/integrations/litellm_guardrail.py @@ -30,7 +30,9 @@ ``datafog`` core; the LiteLLM proxy, where this runs, always ships fastapi). """ +import copy import logging +import re from typing import Any, Optional from fastapi import HTTPException @@ -42,15 +44,74 @@ VALID_ACTIONS = {"redact", "block"} VALID_FAIL_POLICIES = {"open", "closed"} +MAX_STREAM_CHARS = 8_000_000 +MAX_STREAM_CHUNKS = 100_000 logger = logging.getLogger(__name__) +class _RedactionSession: + """Token assignments belong to one request or response, never the adapter.""" + + def __init__(self, strings=()): + self.tokens: dict[tuple[str, str], str] = {} + self.counters: dict[str, int] = {} + self.reserved = { + token for text in strings for token in re.findall(r"\[[A-Z_]+_\d+\]", text) + } + + def token(self, entity): + key = (entity.type, entity.text) + if key not in self.tokens: + number = self.counters.get(entity.type, 0) + 1 + while f"[{entity.type}_{number}]" in self.reserved: + number += 1 + token = f"[{entity.type}_{number}]" + self.counters[entity.type] = number + self.tokens[key] = token + self.reserved.add(token) + return self.tokens[key] + + +def _strings(value): + stack = [value] + visited = set() + while stack: + item = stack.pop() + if isinstance(item, (dict, list)): + if id(item) in visited: + continue + visited.add(id(item)) + if isinstance(item, str): + yield item + elif isinstance(item, dict): + stack.extend(item.values()) + stack.extend(item.keys()) + elif isinstance(item, list): + stack.extend(item) + + +def _get(value, name, default=None): + return ( + value.get(name, default) + if isinstance(value, dict) + else getattr(value, name, default) + ) + + +def _set(value, name, item): + if isinstance(value, dict): + value[name] = item + else: + setattr(value, name, item) + + def _redact_text( text: str, entity_types: list[str], allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, + session: _RedactionSession | None = None, ) -> tuple[str, dict[str, int]]: """Redact ``text``; return (redacted_text, counts per entity type).""" import datafog @@ -65,7 +126,14 @@ def _redact_text( counts: dict[str, int] = {} for entity in result.entities: counts[entity.type] = counts.get(entity.type, 0) + 1 - return result.redacted_text, counts + session = session or _RedactionSession([text]) + parts = [] + cursor = 0 + for entity in sorted(result.entities, key=lambda entity: entity.start): + parts.extend((text[cursor : entity.start], session.token(entity))) + cursor = entity.end + parts.append(text[cursor:]) + return "".join(parts), counts def _summary(counts: dict[str, int]) -> str: @@ -97,39 +165,51 @@ def __init__( self.allowlist_patterns = allowlist_patterns super().__init__(**kwargs) - def _process_content(self, content: Any) -> tuple[Any, dict[str, int]]: - """Redact a message content value (str or list of content parts).""" + def _process_content( + self, content: Any, session=None + ) -> tuple[Any, dict[str, int]]: + """Walk textual input, including nested tool arguments and content parts.""" + session = session or _RedactionSession(_strings(content)) counts: dict[str, int] = {} if isinstance(content, str): redacted, counts = _redact_text( - content, self.entity_types, self.allowlist, self.allowlist_patterns + content, + self.entity_types, + self.allowlist, + self.allowlist_patterns, + session, ) return redacted, counts - if isinstance(content, list): - new_parts = [] - skipped_parts = 0 - for part in content: - if isinstance(part, dict) and isinstance(part.get("text"), str): - redacted, part_counts = _redact_text( - part["text"], - self.entity_types, - self.allowlist, - self.allowlist_patterns, - ) - new_parts.append({**part, "text": redacted}) - for etype, n in part_counts.items(): - counts[etype] = counts.get(etype, 0) + n + if isinstance(content, (list, dict)): + items = enumerate(content) if isinstance(content, list) else content.items() + output = [] if isinstance(content, list) else {} + for key, value in items: + # Media payloads are opaque; preserve them while scanning any + # textual sibling. name/id/url in ordinary user data are text. + media_type = content.get("type") if isinstance(content, dict) else None + if not isinstance(media_type, str): + media_type = None + opaque = { + "image_url": {"image_url"}, + "input_image": {"image_url", "file_id"}, + "image": {"source"}, + "input_audio": {"input_audio"}, + "audio": {"source", "data"}, + "video_url": {"video_url"}, + "file": {"file"}, + "input_file": {"file_id", "file_data"}, + } + if key in opaque.get(media_type, set()): + new_value, found = value, {} else: - # Images and other non-text parts are not scanned — - # count them so the blind spot is auditable. - new_parts.append(part) - skipped_parts += 1 - if skipped_parts: - logger.debug( - "DataFog guardrail: %d non-text content parts not scanned", - skipped_parts, - ) - return new_parts, counts + new_value, found = self._process_content(value, session) + if isinstance(output, list): + output.append(new_value) + else: + output[key] = new_value + for label, count in found.items(): + counts[label] = counts.get(label, 0) + count + return output, counts return content, counts def _handle_engine_error(self, exc: Exception) -> None: @@ -158,21 +238,20 @@ async def async_pre_call_hook( data: dict, call_type: str, ) -> dict: - messages = data.get("messages") - if not isinstance(messages, list): - return data - total_counts: dict[str, int] = {} - new_messages = [] + new_data = dict(data) + fields = ("messages", "prompt", "input") + session = _RedactionSession( + text for field in fields for text in _strings(data.get(field)) + ) try: - for message in messages: - if isinstance(message, dict) and "content" in message: - new_content, counts = self._process_content(message["content"]) - new_messages.append({**message, "content": new_content}) - for etype, n in counts.items(): - total_counts[etype] = total_counts.get(etype, 0) + n - else: - new_messages.append(message) + for field in fields: + if field in data: + new_data[field], counts = self._process_content( + data[field], session + ) + for label, count in counts.items(): + total_counts[label] = total_counts.get(label, 0) + count except Exception as exc: # noqa: BLE001 — fail policy decides self._handle_engine_error(exc) return data @@ -197,7 +276,6 @@ async def async_pre_call_hook( }, ) - new_data = {**data, "messages": new_messages} self._record_guardrail_logging(new_data, total_counts) return new_data @@ -215,42 +293,130 @@ def _record_guardrail_logging( except Exception: # noqa: BLE001 — observability must never break traffic logger.debug("could not record guardrail logging information") + @staticmethod + def _response_fields(response): + """Yield only generated text fields, preserving response metadata.""" + for choice in _get(response, "choices", []) or []: + if isinstance(_get(choice, "text"), str): + yield choice, "text" + message = _get(choice, "message") + if message is None: + continue + yield message, "content" + for call in _get(message, "tool_calls", []) or []: + function = _get(call, "function") + if function is not None: + yield function, "arguments" + function = _get(message, "function_call") + if function is not None: + yield function, "arguments" + async def async_post_call_success_hook( self, data: dict, user_api_key_dict: Any, response: Any, ) -> Any: - """Redact PII from model responses. + """Redact generated text in place after every field has been scanned.""" + try: + fields = list(self._response_fields(response)) + session = _RedactionSession( + text + for target, field in fields + for text in _strings(_get(target, field)) + ) + changes = [] + for target, field in fields: + value, _ = self._process_content(_get(target, field), session) + changes.append((target, field, value)) + for target, field, value in changes: + _set(target, field, value) + except Exception as exc: # noqa: BLE001 — fail policy decides + self._handle_engine_error(exc) + return response - Mutates ``response`` in place — deliberate: litellm post_call - guardrails share the response object rather than cloning it, and - an unredacted clone escaping through another callback would defeat - the purpose. + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict, response, request_data + ): + """Buffer to completion before emitting, protecting split entities. + + Native text and tool argument deltas are joined per choice/tool. The + first corresponding delta carries the redacted text and later deltas + are empty; metadata and finish events retain their original order. + Incomplete or oversized streams are rejected without emitting chunks. """ - choices = getattr(response, "choices", None) - if not choices: - return response + chunks = [] + groups = {} + size = 0 try: - skipped_parts = 0 - for choice in choices: - message = getattr(choice, "message", None) - if message is not None and isinstance(message.content, str): - redacted, counts = _redact_text( - message.content, - self.entity_types, - self.allowlist, - self.allowlist_patterns, - ) - if counts: - message.content = redacted - elif message is not None and message.content is not None: - skipped_parts += 1 - if skipped_parts: - logger.debug( - "DataFog guardrail: %d non-text response parts not scanned", - skipped_parts, + async for original in response: + # Iterators may reuse a mutable chunk object. Keep each emitted + # delta's value and metadata as they existed when received. + chunk = copy.deepcopy(original) + chunks.append(chunk) + for position, choice in enumerate(_get(chunk, "choices", []) or []): + index = _get(choice, "index", position) + fields = [((index, "text"), choice, "text")] + delta = _get(choice, "delta") + if delta is not None: + fields.append(((index, "content"), delta, "content")) + for call_position, call in enumerate( + _get(delta, "tool_calls", []) or [] + ): + function = _get(call, "function") + if function is not None: + fields.append( + ( + ( + index, + "tool", + _get(call, "index", call_position), + ), + function, + "arguments", + ) + ) + function = _get(delta, "function_call") + if function is not None: + fields.append( + ((index, "function_call"), function, "arguments") + ) + for key, target, field in fields: + value = _get(target, field) + if isinstance(value, str): + size += len(value) + groups.setdefault(key, []).append((target, field, value)) + if size > MAX_STREAM_CHARS or len(chunks) > MAX_STREAM_CHUNKS: + raise _StreamLimitExceeded() + except _StreamLimitExceeded: + raise RuntimeError( + "DataFog streaming response exceeded the scan budget; rejecting unscanned output" + ) from None + except Exception as exc: + # A failed upstream stream cannot be safely completed or replayed. + # Provider exceptions can contain private prompt/response text. + raise RuntimeError( + f"DataFog streaming response did not complete ({type(exc).__name__}); rejecting incomplete output" + ) from None + try: + texts = [ + "".join(value for _, _, value in group) for group in groups.values() + ] + session = _RedactionSession(texts) + changes = [] + for group, text in zip(groups.values(), texts, strict=True): + redacted, _ = self._process_content(text, session) + changes.extend( + (target, field, redacted if i == 0 else "") + for i, (target, field, _) in enumerate(group) ) - except Exception as exc: # noqa: BLE001 — fail policy decides + for target, field, value in changes: + _set(target, field, value) + except Exception as exc: self._handle_engine_error(exc) - return response + for chunk in chunks: + yield chunk + + +class _StreamLimitExceeded(Exception): + """Internal marker; stream-budget failures always reject output.""" diff --git a/datafog/main.py b/datafog/main.py index 0dc6f5d5..30fd1eed 100644 --- a/datafog/main.py +++ b/datafog/main.py @@ -15,12 +15,12 @@ from datafog._legacy_retirement import warn_legacy_surface from .config import OperationType +from .core import get_supported_entities from .engine import scan, scan_and_redact from .models.anonymizer import Anonymizer, AnonymizerType, HashType from .processing.text_processing.regex_annotator import RegexAnnotator logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) class DataFog: @@ -42,6 +42,7 @@ def __init__( hash_type: HashType = HashType.SHA256, anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, locales: List[str] | None = None, + hash_key: bytes | None = None, ): self.locales = locales self.regex_annotator = RegexAnnotator(locales=locales) @@ -59,6 +60,7 @@ def __init__( hash_type=hash_type, anonymizer_type=anonymizer_type ) self.hash_type = hash_type + self.hash_key = hash_key self.anonymizer_type = anonymizer_type self.logger = logging.getLogger(__name__) self.logger.info("Initializing lightweight DataFog class with regex engine") @@ -106,7 +108,15 @@ def run_text_pipeline_sync(self, str_list: List[str]) -> List: _start = _time.monotonic() try: self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: + if any( + op in self.operations + for op in ( + OperationType.SCAN, + OperationType.REDACT, + OperationType.REPLACE, + OperationType.HASH, + ) + ): annotated_text = [self.detect(text) for text in str_list] self.logger.info( @@ -190,8 +200,11 @@ def detect(self, text: str) -> dict: # Only pre-populate keys for labels active under the configured # locales so the default output shape matches v4.4.0 (no DE_* keys # unless German locale support is enabled). - result = {label: [] for label in RegexAnnotator.active_labels_for(self.locales)} legacy_map = {"DATE": "DOB", "ZIP_CODE": "ZIP"} + result = { + legacy_map.get(label, label): [] + for label in get_supported_entities(self.locales) + } for entity in scan_result.entities: label = legacy_map.get(entity.type, entity.type) result.setdefault(label, []).append(entity.text) @@ -239,6 +252,8 @@ def process( _start = _time.monotonic() + if method not in {"redact", "replace", "hash"}: + raise ValueError("method must be 'redact', 'replace', or 'hash'") annotations_dict = self.detect(text) result = {"original": text, "findings": annotations_dict} @@ -249,12 +264,18 @@ def process( "replace": "pseudonymize", "hash": "hash", } - strategy = strategy_map.get(method, "token") + strategy = strategy_map[method] redact_result = scan_and_redact( text=text, engine="regex", strategy=strategy, locales=self.locales, + hash_type=( + self.hash_type.value + if isinstance(self.hash_type, HashType) + else self.hash_type + ), + hash_key=self.hash_key, ) result["anonymized"] = redact_result.redacted_text @@ -298,6 +319,7 @@ class TextPIIAnnotator: """ def __init__(self, locales: List[str] | None = None): + self.locales = locales self.regex_annotator = RegexAnnotator(locales=locales) def run(self, text, output_path=None): @@ -312,7 +334,13 @@ def run(self, text, output_path=None): Dictionary mapping entity types to lists of found entities """ try: - annotated_text = self.regex_annotator.annotate(text) + result = scan(text=text, engine="regex", locales=self.locales) + legacy = {"DATE": "DOB", "ZIP_CODE": "ZIP"} + annotated_text = {} + for entity in result.entities: + annotated_text.setdefault( + legacy.get(entity.type, entity.type), [] + ).append(entity.text) # Optionally, output the results to a JSON file if output_path: diff --git a/datafog/main_lean.py b/datafog/main_lean.py deleted file mode 100644 index f67501e6..00000000 --- a/datafog/main_lean.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Historical shadow main module. - -This file is not the live DataFog 4.5 ``DataFog`` implementation. Use -``datafog/main.py`` for the current backward-compatible class and text/OCR -compatibility methods. This lean snapshot remains importable only as historical -reference until legacy cleanup can remove it safely. - -Lean main module for DataFog core functionality. - -This module contains the lightweight core classes for DataFog: -- DataFog: Main class for regex-based PII detection -- TextPIIAnnotator: Class for annotating PII in text using regex patterns - -These classes provide the core PII detection functionality without heavy dependencies. -""" - -import json -import logging -from typing import List - -from .config import OperationType -from .models.anonymizer import Anonymizer, AnonymizerType, HashType -from .processing.text_processing.regex_annotator import RegexAnnotator - -logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) - - -class DataFog: - """ - Lightweight main class for regex-based PII detection and anonymization. - - Handles text processing operations using fast regex patterns for PII detection. - For advanced features like OCR, spaCy, or Spark, install additional extras. - - Attributes: - regex_annotator: Core regex-based PII annotator. - operations: List of operations to perform. - anonymizer: Anonymizer for PII redaction, replacement, or hashing. - """ - - def __init__( - self, - operations: List[OperationType] = [OperationType.SCAN], - hash_type: HashType = HashType.SHA256, - anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, - ): - self.regex_annotator = RegexAnnotator() - self.operations: List[OperationType] = operations - self.anonymizer = Anonymizer( - hash_type=hash_type, anonymizer_type=anonymizer_type - ) - self.logger = logging.getLogger(__name__) - self.logger.info("Initializing lightweight DataFog class with regex engine") - self.logger.info(f"Operations: {operations}") - self.logger.info(f"Hash Type: {hash_type}") - self.logger.info(f"Anonymizer Type: {anonymizer_type}") - - def run_text_pipeline_sync(self, str_list: List[str]) -> List[str]: - """ - Run the text pipeline synchronously on a list of input text. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List[str]: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: - annotated_text = [] - - for text in str_list: - # Use regex annotator for core PII detection - annotations = self.regex_annotator.annotate(text) - annotated_text.append(annotations) - - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if any( - op in self.operations - for op in [ - OperationType.REDACT, - OperationType.REPLACE, - OperationType.HASH, - ] - ): - return [ - self.anonymizer.anonymize(text, annotations).anonymized_text - for text, annotations in zip( - str_list, annotated_text, strict=True - ) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return str_list - except Exception as e: - self.logger.error(f"Error in run_text_pipeline_sync: {str(e)}") - raise - - def detect(self, text: str) -> dict: - """ - Simple PII detection using regex patterns. - - Args: - text: Input text to scan for PII - - Returns: - Dictionary mapping entity types to lists of found entities - """ - return self.regex_annotator.annotate(text) - - def process( - self, text: str, anonymize: bool = False, method: str = "redact" - ) -> dict: - """ - Process text to detect and optionally anonymize PII. - - Args: - text: Input text to process - anonymize: Whether to anonymize detected PII - method: Anonymization method ('redact', 'replace', 'hash') - - Returns: - Dictionary with original text, anonymized text (if requested), and findings - """ - annotations = self.detect(text) - - result = {"original": text, "findings": annotations} - - if anonymize: - if method == "redact": - anonymizer_type = AnonymizerType.REDACT - elif method == "replace": - anonymizer_type = AnonymizerType.REPLACE - elif method == "hash": - anonymizer_type = AnonymizerType.HASH - else: - anonymizer_type = AnonymizerType.REDACT - - anonymized_result = self.anonymizer.anonymize( - text, annotations, anonymizer_type - ) - result["anonymized"] = anonymized_result.anonymized_text - - return result - - -class TextPIIAnnotator: - """ - Lightweight class for annotating PII in text using regex patterns. - - Provides functionality to detect and annotate Personally Identifiable Information (PII) - in text using fast regex patterns instead of heavy NLP models. - - Attributes: - regex_annotator: RegexAnnotator instance for text annotation. - """ - - def __init__(self): - self.regex_annotator = RegexAnnotator() - - def run(self, text, output_path=None): - """ - Run PII annotation on text using regex patterns. - - Args: - text: Input text to annotate - output_path: Optional path to save results as JSON - - Returns: - Dictionary mapping entity types to lists of found entities - """ - try: - annotated_text = self.regex_annotator.annotate(text) - - # Optionally, output the results to a JSON file - if output_path: - with open(output_path, "w") as f: - json.dump(annotated_text, f) - - return annotated_text - - except Exception as e: - logging.error(f"Error in TextPIIAnnotator.run: {str(e)}") - raise diff --git a/datafog/main_original.py b/datafog/main_original.py deleted file mode 100644 index 9594c824..00000000 --- a/datafog/main_original.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Historical shadow main module. - -This file is not the live DataFog 4.5 ``DataFog`` implementation. Use -``datafog/main.py`` for the current backward-compatible class and use the -optional OCR/Spark service modules directly when those surfaces are needed. -This original full-featured snapshot remains importable for legacy tests until -cleanup can remove it safely. - -Main module for DataFog. - -This module contains the core classes for DataFog: -- DataFog: Main class for running OCR and text processing pipelines. -- TextPIIAnnotator: Class for annotating PII in text. - -These classes provide high-level interfaces for image and text processing, -including OCR, PII detection, annotation, and anonymization. -""" - -import json -import logging -from typing import List - -from .config import OperationType -from .models.anonymizer import Anonymizer, AnonymizerType, HashType -from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator -from .services.image_service import ImageService -from .services.spark_service import SparkService -from .services.text_service import TextService - -logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) - - -class DataFog: - """ - Main class for running OCR and text processing pipelines. - - Handles image and text processing operations, including OCR, PII detection, and anonymization. - - Attributes: - image_service: Service for image processing and OCR. - text_service: Service for text processing and annotation. - spark_service: Optional Spark service for distributed processing. - operations: List of operations to perform. - anonymizer: Anonymizer for PII redaction, replacement, or hashing. - """ - - def __init__( - self, - image_service=None, - text_service=None, - spark_service=None, - operations: List[OperationType] = [OperationType.SCAN], - hash_type: HashType = HashType.SHA256, - anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, - ): - self.image_service = image_service or ImageService() - self.text_service = text_service or TextService() - self.spark_service: SparkService = spark_service - self.operations: List[OperationType] = operations - self.anonymizer = Anonymizer( - hash_type=hash_type, anonymizer_type=anonymizer_type - ) - self.logger = logging.getLogger(__name__) - self.logger.info( - "Initializing DataFog class with the following services and operations:" - ) - self.logger.info(f"Image Service: {type(self.image_service)}") - self.logger.info(f"Text Service: {type(self.text_service)}") - self.logger.info( - f"Spark Service: {type(self.spark_service) if self.spark_service else 'None'}" - ) - self.logger.info(f"Operations: {operations}") - self.logger.info(f"Hash Type: {hash_type}") - self.logger.info(f"Anonymizer Type: {anonymizer_type}") - - async def run_ocr_pipeline(self, image_urls: List[str]): - """ - Run the OCR pipeline asynchronously on a list of images provided via URL. - - This method performs optical character recognition (OCR) on the images specified by the URLs. - If PII annotation is enabled, it also annotates the extracted text for personally identifiable information. - If redaction, replacement, or hashing is enabled, it applies the corresponding anonymization. - - Args: - image_urls (List[str]): A list of URLs pointing to the images to be processed. - - Returns: - List: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the OCR or text processing. - """ - try: - extracted_text = await self.image_service.ocr_extract(image_urls) - self.logger.info(f"OCR extraction completed for {len(image_urls)} images.") - - return await self._process_text(extracted_text) - except Exception as e: - logging.error(f"Error in run_ocr_pipeline: {str(e)}") - return [f"Error: {str(e)}"] - - async def run_text_pipeline(self, str_list: List[str]): - """ - Run the text pipeline asynchronously on a list of input text. - - This method processes a list of text strings, potentially annotating them for personally - identifiable information (PII) and applying anonymization if enabled. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - return await self._process_text(str_list) - except Exception as e: - self.logger.error(f"Error in run_text_pipeline: {str(e)}") - raise - - async def _process_text(self, text_list: List[str]): - """ - Internal method to process text based on enabled operations. - """ - if OperationType.SCAN in self.operations: - annotated_text = await self.text_service.batch_annotate_text_async( - text_list - ) - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if OperationType.REDACT in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.REDACT - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - elif OperationType.REPLACE in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.REPLACE - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - elif OperationType.HASH in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.HASH - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return text_list - - def run_text_pipeline_sync(self, str_list: List[str]) -> List[str]: - """ - Run the text pipeline synchronously on a list of input text. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List[str]: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: - annotated_text = self.text_service.batch_annotate_text_sync(str_list) - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if any( - op in self.operations - for op in [ - OperationType.REDACT, - OperationType.REPLACE, - OperationType.HASH, - ] - ): - return [ - self.anonymizer.anonymize(text, annotations).anonymized_text - for text, annotations in zip( - str_list, annotated_text, strict=True - ) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return str_list - except Exception as e: - self.logger.error(f"Error in run_text_pipeline_sync: {str(e)}") - raise - - def _add_attributes(self, attributes: dict): - """ - Add multiple attributes to the DataFog instance. - - This private method allows for the dynamic addition of multiple attributes to the - DataFog instance. It iterates through the provided dictionary of attributes and - adds each key-value pair as an attribute. - - Args: - attributes (dict): A dictionary where keys are attribute names and values are - the corresponding attribute values to be added. - - Note: - This method is intended for internal use and may be used for extending the - functionality of the DataFog class dynamically. Care should be taken when - using this method to avoid overwriting existing attributes. - """ - for key, value in attributes.items(): - setattr(self, key, value) - - -class TextPIIAnnotator: - """ - Class for annotating PII in text. - - Provides functionality to detect and annotate Personally Identifiable Information (PII) in text. - - Attributes: - text_annotator: SpacyPIIAnnotator instance for text annotation. - spark_processor: Optional SparkService for distributed processing. - """ - - def __init__(self): - self.text_annotator = SpacyPIIAnnotator.create() - self.spark_processor: SparkService = None - - def run(self, text, output_path=None): - try: - annotated_text = self.text_annotator.annotate(text) - - # Optionally, output the results to a JSON file - if output_path: - with open(output_path, "w") as f: - json.dump(annotated_text, f) - - return annotated_text - - finally: - # Ensure Spark resources are released - if self.spark_processor: - self.spark_processor.stop() diff --git a/datafog/models/annotator.py b/datafog/models/annotator.py index 2f915ea4..489689b5 100644 --- a/datafog/models/annotator.py +++ b/datafog/models/annotator.py @@ -22,12 +22,12 @@ class AnnotatorRequest(BaseModel): text: str language: str - correlation_id: Optional[str] - score_threshold: Optional[float] - entities: Optional[List[EntityTypes]] - return_decision_process: Optional[bool] - ad_hoc_recognizers: Optional[List[PatternRecognizer]] - context: Optional[List[str]] + correlation_id: Optional[str] = None + score_threshold: Optional[float] = None + entities: Optional[List[EntityTypes]] = None + return_decision_process: Optional[bool] = None + ad_hoc_recognizers: Optional[List[PatternRecognizer]] = None + context: Optional[List[str]] = None class AnnotationResult(BaseModel): @@ -39,16 +39,19 @@ class AnnotationResult(BaseModel): start: int end: int - score: Optional[float] + score: Optional[float] = None entity_type: str - recognition_metadata: Optional[AnnotatorMetadata] + recognition_metadata: Optional[AnnotatorMetadata] = None - @field_validator("entity_type") + @field_validator("entity_type", mode="before") @classmethod def validate_entity_type(cls, v): - if v not in EntityTypes.__members__: - return "UNKNOWN" - return v + from datafog.engine import ALL_ENTITY_TYPES, CANONICAL_TYPE_MAP + + if isinstance(v, EntityTypes): + v = v.name + v = CANONICAL_TYPE_MAP.get(v, v) + return v if v in ALL_ENTITY_TYPES else "UNKNOWN" class AnalysisExplanation(BaseModel): @@ -60,14 +63,14 @@ class AnalysisExplanation(BaseModel): """ recognizer: str - pattern_name: Optional[str] - pattern: Optional[str] + pattern_name: Optional[str] = None + pattern: Optional[str] = None original_score: float score: float - textual_explanation: Optional[str] + textual_explanation: Optional[str] = None score_context_improvement: float supportive_context_word: str - validation_result: Optional[float] + validation_result: Optional[float] = None class AnnotationResultWithAnaysisExplanation(AnnotationResult): @@ -75,4 +78,4 @@ class AnnotationResultWithAnaysisExplanation(AnnotationResult): Extends AnnotationResult with detailed analysis explanation. """ - analysis_explanation: Optional[AnalysisExplanation] + analysis_explanation: Optional[AnalysisExplanation] = None diff --git a/datafog/models/anonymizer.py b/datafog/models/anonymizer.py index 6b81f5be..3ec36664 100644 --- a/datafog/models/anonymizer.py +++ b/datafog/models/anonymizer.py @@ -50,6 +50,33 @@ class Anonymizer(BaseModel): def __init__(self, **data): super().__init__(**data) + def _selected_annotations(self, text, annotations): + from datafog.engine import CANONICAL_TYPE_MAP + + selected = ( + None + if not self.entities + else { + CANONICAL_TYPE_MAP.get(entity.name, entity.name) + for entity in self.entities + } + ) + merged = [] + for annotation in sorted(annotations, key=lambda a: (a.start, -a.end)): + if not 0 <= annotation.start < annotation.end <= len(text): + raise ValueError( + "Annotation range must be nonempty and inside the text" + ) + if selected is not None and annotation.entity_type not in selected: + continue + if merged and annotation.start < merged[-1].end: + merged[-1] = merged[-1].model_copy( + update={"end": max(merged[-1].end, annotation.end)} + ) + else: + merged.append(annotation) + return reversed(merged) + def anonymize( self, text: str, annotations: List[AnnotationResult] ) -> AnonymizationResult: @@ -68,19 +95,18 @@ def replace_pii( ) -> AnonymizationResult: """Replace PII in text with anonymized values.""" replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if not self.entities or annotation.entity_type in self.entities: - replacement = self._generate_replacement( - text[annotation.start : annotation.end], annotation.entity_type - ) - replacements.append( - { - "original": text[annotation.start : annotation.end], - "replacement": replacement, - "entity_type": annotation.entity_type, - } - ) - text = text[: annotation.start] + replacement + text[annotation.end :] + for annotation in self._selected_annotations(text, annotations): + replacement = self._generate_replacement( + text[annotation.start : annotation.end], annotation.entity_type + ) + replacements.append( + { + "original": text[annotation.start : annotation.end], + "replacement": replacement, + "entity_type": annotation.entity_type, + } + ) + text = text[: annotation.start] + replacement + text[annotation.end :] return AnonymizationResult( anonymized_text=text, anonymized_entities=replacements @@ -88,13 +114,13 @@ def replace_pii( def _generate_replacement(self, original: str, entity_type: EntityTypes) -> str: """Generate a replacement for the given entity.""" - if entity_type == EntityTypes.PERSON: + if entity_type == "PERSON": return f"[PERSON_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.ORGANIZATION: + elif entity_type == "ORGANIZATION": return f"[ORGANIZATION_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.LOCATION: + elif entity_type == "LOCATION": return f"[LOCATION_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.DATE: + elif entity_type == "DATE": return "[REDACTED_DATE]" else: return f"[{entity_type}_{secrets.token_hex(4).upper()}]" @@ -104,10 +130,7 @@ def hash_pii( ) -> AnonymizationResult: """Hash PII in text.""" replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if self.entities and annotation.entity_type not in self.entities: - continue - + for annotation in self._selected_annotations(text, annotations): start, end = annotation.start, annotation.end original = text[start:end] replacement = self._hash_text(original) @@ -139,10 +162,7 @@ def redact_pii( self, text: str, annotations: List[AnnotationResult] ) -> AnonymizationResult: replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if self.entities and annotation.entity_type not in self.entities: - continue - + for annotation in self._selected_annotations(text, annotations): start, end = annotation.start, annotation.end original = text[start:end] replacement = "[REDACTED]" diff --git a/datafog/models/common.py b/datafog/models/common.py index f01d6651..1f72d4aa 100644 --- a/datafog/models/common.py +++ b/datafog/models/common.py @@ -40,8 +40,8 @@ class PatternRecognizer(BaseModel): name: str supported_language: str patterns: List[Pattern] - deny_list: Optional[List[str]] - context: Optional[List[str]] + deny_list: Optional[List[str]] = None + context: Optional[List[str]] = None supported_entity: str diff --git a/datafog/native.py b/datafog/native.py index cde64934..44b6fb29 100644 --- a/datafog/native.py +++ b/datafog/native.py @@ -40,7 +40,7 @@ def _core() -> Any: except ImportError as exc: # pragma: no cover - base install always has it raise ImportError( "datafog.native requires datafog-core. Install with: " - 'pip install "datafog==4.10.0"' + 'pip install "datafog[rust]"' ) from exc @@ -52,7 +52,13 @@ def __getattr__(name: str) -> Any: def version() -> str: """Installed Core version, for bug reports and reproducibility locks.""" - return importlib.metadata.version("datafog-core") + _core() + try: + return importlib.metadata.version("datafog-core") + except importlib.metadata.PackageNotFoundError as exc: + raise ImportError( + "datafog-core distribution metadata is missing; reinstall datafog[rust]" + ) from exc def contract_version() -> int: diff --git a/datafog/processing/spark_processing/pyspark_udfs.py b/datafog/processing/spark_processing/pyspark_udfs.py index aaa53ece..bb9b8c18 100644 --- a/datafog/processing/spark_processing/pyspark_udfs.py +++ b/datafog/processing/spark_processing/pyspark_udfs.py @@ -28,8 +28,9 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]: if text: if len(text) > MAXIMAL_STRING_SIZE: - # Cut the strings for required sizes - text = text[:MAXIMAL_STRING_SIZE] + raise ValueError( + "Spark PII input exceeds the maximum supported string length" + ) nlp = broadcasted_nlp.value doc = nlp(text) @@ -38,8 +39,15 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]: _label: [] for _label in PII_ANNOTATION_LABELS } for ent in doc.ents: - # Add entities from extracted values - classified_entities[ent.label_].append(ent.text) + label = { + "PERSON": "PER", + "GPE": "LOC", + "NORP": "NRP", + "DATE": "DATE_TIME", + "TIME": "DATE_TIME", + }.get(ent.label_, ent.label_) + if label in classified_entities: + classified_entities[label].append(ent.text) return [_ent for _ent in classified_entities.values()] else: diff --git a/datafog/processing/text_processing/gliner_annotator.py b/datafog/processing/text_processing/gliner_annotator.py index 5a195f89..b871cf63 100644 --- a/datafog/processing/text_processing/gliner_annotator.py +++ b/datafog/processing/text_processing/gliner_annotator.py @@ -10,6 +10,8 @@ from pydantic import BaseModel, ConfigDict +from .regex_annotator.regex_annotator import AnnotationResult, Span + # Default entity types for PII detection using GLiNER # These can be customized based on specific use cases DEFAULT_PII_ENTITIES = [ @@ -92,55 +94,37 @@ def create( f"datafog download-model {model_name} --engine gliner" ) from e - def annotate(self, text: str) -> Dict[str, List[str]]: - """ - Annotate text for PII entities using GLiNER. - - Args: - text: Text to analyze for PII entities - - Returns: - Dictionary mapping entity types to lists of detected entities - """ - try: - if not text: - return { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } - - if len(text) > MAXIMAL_STRING_SIZE: - text = text[:MAXIMAL_STRING_SIZE] - logging.warning(f"Text truncated to {MAXIMAL_STRING_SIZE} characters") - - # Predict entities using GLiNER - entities = self.model.predict_entities(text, self.entity_types) - - # Organize results by entity type - classified_entities: Dict[str, List[str]] = { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } - + def annotate_with_spans(self, text: str): + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text) > MAXIMAL_STRING_SIZE: + raise ValueError( + f"GLiNER input exceeds {MAXIMAL_STRING_SIZE} characters; split the document before scanning" + ) + classified_entities = { + label.upper().replace(" ", "_"): [] for label in self.entity_types + } + spans = [] + if text: + try: + entities = self.model.predict_entities(text, self.entity_types) + except Exception as exc: + raise RuntimeError("GLiNER annotation failed") from exc for entity in entities: - entity_label = entity["label"].upper().replace(" ", "_") - entity_text = entity["text"] + label = entity["label"].upper().replace(" ", "_") + classified_entities.setdefault(label, []).append(entity["text"]) + spans.append( + Span( + label=label, + start=entity["start"], + end=entity["end"], + text=entity["text"], + ) + ) + return classified_entities, AnnotationResult(text=text, spans=spans) - if entity_label in classified_entities: - classified_entities[entity_label].append(entity_text) - else: - # Handle cases where GLiNER returns entity types not in our list - classified_entities[entity_label] = [entity_text] - - return classified_entities - - except Exception as e: - logging.error(f"Error processing text with GLiNER: {str(e)}") - # Return empty annotations in case of error - return { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } + def annotate(self, text: str) -> Dict[str, List[str]]: + return self.annotate_with_spans(text)[0] def set_entity_types(self, entity_types: List[str]) -> None: """ diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py index 44dfa2e0..03641e82 100644 --- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py +++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py @@ -40,11 +40,14 @@ class RegexAnnotator: ] LABELS = BASE_LABELS + GERMAN_LABELS DEFAULT_LABELS = BASE_LABELS - SUPPORTED_LOCALES = {"de", "de-de", "de_de"} + SUPPORTED_LOCALES = {"de", "de-de", "de_de", "en", "en-us", "fr"} LOCALE_LABELS = { "de": GERMAN_LABELS, "de-de": GERMAN_LABELS, "de_de": GERMAN_LABELS, + "en": [], + "en-us": [], + "fr": [], } def __init__( @@ -63,14 +66,13 @@ def __init__( # Note: This is broader than the spec to catch more potential emails "EMAIL": re.compile( r""" - (? (? (? (? (? tuple[str, . locales = [locales] normalized = [] for locale in locales: + if not isinstance(locale, str) or not locale.strip(): + raise ValueError("locales must contain nonempty locale identifiers") value = locale.strip().lower() - if not value: - continue if value not in cls.SUPPORTED_LOCALES: allowed = ", ".join(sorted(cls.SUPPORTED_LOCALES)) raise ValueError(f"locale must be one of: {allowed}") @@ -360,15 +362,27 @@ def active_labels_for( active.update(label.strip().upper() for label in enabled_labels) return [label for label in cls.LABELS if label in active] - @staticmethod - def _match_text(match: Match[str]) -> str: - return match.groupdict().get("value") or match.group() + @classmethod + def _match_text(cls, match: Match[str]) -> str: + start, end = cls._match_span(match) + return match.string[start:end] @staticmethod def _match_span(match: Match[str]) -> tuple[int, int]: if "value" in match.groupdict() and match.group("value") is not None: return match.start("value"), match.end("value") - return match.start(), match.end() + start, end = match.start(), match.end() + value = match.group() + if "@" in value and any(char.isalnum() for char in value[: value.rfind("@")]): + local = value[: value.rfind("@")] + # Query/assignment separators and prose wrappers are context, + # not part of the mailbox. Consume the whole lexical run before + # trimming so failed candidates never retry every suffix. + boundary = max((local.rfind(char) for char in "&?=|/"), default=-1) + start += boundary + 1 + while start < end and match.string[start] in ".!?'*{}`": + start += 1 + return start, end @classmethod def create(cls, **kwargs) -> "RegexAnnotator": diff --git a/datafog/processing/text_processing/spacy_pii_annotator.py b/datafog/processing/text_processing/spacy_pii_annotator.py index d2d1d765..1cb2cbd2 100644 --- a/datafog/processing/text_processing/spacy_pii_annotator.py +++ b/datafog/processing/text_processing/spacy_pii_annotator.py @@ -1,8 +1,9 @@ -import logging from typing import Any, Dict, List from pydantic import BaseModel, ConfigDict +from .regex_annotator.regex_annotator import AnnotationResult, Span + PII_ANNOTATION_LABELS = [ "CARDINAL", "DATE", @@ -28,7 +29,7 @@ class SpacyPIIAnnotator(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) + model_config = ConfigDict(arbitrary_types_allowed=True, protected_namespaces=()) nlp: Any model_name: str = DEFAULT_SPACY_MODEL @@ -53,22 +54,32 @@ def create(cls, model_name: str = DEFAULT_SPACY_MODEL) -> "SpacyPIIAnnotator": return cls(nlp=nlp, model_name=model_name) - def annotate(self, text: str) -> Dict[str, List[str]]: - try: - if not text: - return {label: [] for label in PII_ANNOTATION_LABELS} - if len(text) > MAXIMAL_STRING_SIZE: - text = text[:MAXIMAL_STRING_SIZE] - doc = self.nlp(text) - classified_entities: Dict[str, List[str]] = { - label: [] for label in PII_ANNOTATION_LABELS - } + def annotate_with_spans(self, text: str): + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text) > MAXIMAL_STRING_SIZE: + raise ValueError( + f"spaCy input exceeds {MAXIMAL_STRING_SIZE} characters; split the document before scanning" + ) + classified_entities = {label: [] for label in PII_ANNOTATION_LABELS} + spans = [] + if text: + try: + doc = self.nlp(text) + except Exception as exc: + raise RuntimeError("spaCy annotation failed") from exc for ent in doc.ents: if ent.label_ in classified_entities: classified_entities[ent.label_].append(ent.text) - return classified_entities - except Exception as e: - logging.error(f"Error processing text for PII annotations: {str(e)}") - return { - label: [] for label in PII_ANNOTATION_LABELS - } # Return empty annotations in case of error + spans.append( + Span( + label=ent.label_, + start=ent.start_char, + end=ent.end_char, + text=ent.text, + ) + ) + return classified_entities, AnnotationResult(text=text, spans=spans) + + def annotate(self, text: str) -> Dict[str, List[str]]: + return self.annotate_with_spans(text)[0] diff --git a/datafog/services/text_service.py b/datafog/services/text_service.py index 6b80b26d..a7826a7e 100644 --- a/datafog/services/text_service.py +++ b/datafog/services/text_service.py @@ -1,8 +1,8 @@ """Lean text processing service for PII annotation. -Provides synchronous and asynchronous methods for annotating text with personally -identifiable information (PII) using regex patterns. Supports chunking long texts -and batch processing. SpaCy integration available as optional extra. +Provides synchronous, asynchronous and batch access to the shared PII engine. +Structured PII spans are scanned across complete documents; optional NLP engines +add spans with their original character coordinates. """ import asyncio @@ -31,8 +31,8 @@ class TextService: Lightweight service for annotating text with PII entities using regex patterns. This service provides methods to detect and annotate personally identifiable information (PII) - in text using fast regex patterns. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. + in text through the shared engine. Complete-document structured scanning preserves + entities across boundaries, and optional NLP engines add canonical entity spans. For advanced NLP-based detection using spaCy, install the 'nlp' extra: pip install datafog[nlp] @@ -49,7 +49,8 @@ def __init__( Initialize the TextService with specified chunk length and annotation engine. Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. + text_chunk_length: Positive compatibility setting. Structured detection scans the + complete document; the shared engine bounds native model chunks separately. engine: The annotation engine to use. Options are: - "regex": (Default) Use RegexAnnotator for fast pattern-based entity detection - "spacy": Use SpacyPIIAnnotator for NLP-based entity detection (requires nlp extra) @@ -61,10 +62,17 @@ def __init__( German structured identifiers. Raises: - AssertionError: If an invalid engine type is provided + ValueError: If the engine or text_chunk_length is invalid ImportError: If spacy/gliner engine is requested but corresponding extra is not installed """ - assert engine in {"regex", "spacy", "gliner", "auto", "smart"}, "Invalid engine" + if engine not in {"regex", "spacy", "gliner", "auto", "smart"}: + raise ValueError("engine must be one of: regex, spacy, gliner, auto, smart") + if ( + not isinstance(text_chunk_length, int) + or isinstance(text_chunk_length, bool) + or text_chunk_length <= 0 + ): + raise ValueError("text_chunk_length must be a positive integer") self.engine = engine self.text_chunk_length = text_chunk_length self.gliner_model = gliner_model @@ -379,17 +387,32 @@ def annotate_text_sync( _start = _time.monotonic() - if len(text) <= self.text_chunk_length: - # Single chunk processing - result = self._annotate_single_chunk(text, structured) + from datafog.engine import scan + + # Structured PII is scanned as a complete document. Fixed-size chunks + # lose entities at their boundaries; native model work chunks separately. + options = { + "engine": "smart" if self.engine == "auto" else self.engine, + "locales": self.locales, + } + if self.gliner_model != "urchade/gliner_multi_pii-v1": + options["gliner_model"] = self.gliner_model + entities = scan(text, **options).entities + if structured: + SpanClass = _get_span_class() + result = [ + SpanClass( + label=entity.type, + start=entity.start, + end=entity.end, + text=entity.text, + ) + for entity in entities + ] else: - # Multi-chunk processing - chunks = self._chunk_text(text) - - if structured: - result = self._annotate_multiple_chunks_structured(chunks) - else: - result = self._annotate_multiple_chunks_dict(chunks) + result = {} + for entity in entities: + result.setdefault(entity.type, []).append(entity.text) try: from datafog.telemetry import ( diff --git a/datafog/services/text_service_lean.py b/datafog/services/text_service_lean.py deleted file mode 100644 index 1262db3f..00000000 --- a/datafog/services/text_service_lean.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Historical shadow text service module. - -This file is not the live DataFog 4.5 text service. Use -``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER, -auto, and smart engine boundary. This lean snapshot remains importable only as -historical reference until legacy cleanup can remove it safely. - -Lean text processing service for PII annotation. - -Provides synchronous and asynchronous methods for annotating text with personally -identifiable information (PII) using regex patterns. Supports chunking long texts -and batch processing. SpaCy integration available as optional extra. -""" - -import asyncio -from typing import Dict, List, Union - -from datafog.processing.text_processing.regex_annotator.regex_annotator import ( - RegexAnnotator, - Span, -) - - -class TextService: - """ - Lightweight service for annotating text with PII entities using regex patterns. - - This service provides methods to detect and annotate personally identifiable information (PII) - in text using fast regex patterns. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. - - For advanced NLP-based detection using spaCy, install the 'nlp' extra: - pip install datafog[nlp] - """ - - def __init__(self, text_chunk_length: int = 1000, engine: str = "regex"): - """ - Initialize the TextService with specified chunk length and annotation engine. - - Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. - engine: The annotation engine to use. Options are: - - "regex": (Default) Use RegexAnnotator for fast pattern-based entity detection - - "spacy": Use SpacyPIIAnnotator for NLP-based entity detection (requires nlp extra) - - "auto": Try RegexAnnotator first and fall back to SpacyPIIAnnotator if no entities found - - Raises: - AssertionError: If an invalid engine type is provided - ImportError: If spacy engine is requested but nlp extra is not installed - """ - assert engine in {"regex", "spacy", "auto"}, "Invalid engine" - self.engine = engine - self.regex_annotator = RegexAnnotator() - self.text_chunk_length = text_chunk_length - - # Only initialize spacy if needed and available - self.spacy_annotator = None - if engine in {"spacy", "auto"}: - try: - from datafog.processing.text_processing.spacy_pii_annotator import ( - SpacyPIIAnnotator, - ) - - self.spacy_annotator = SpacyPIIAnnotator.create() - except ImportError: - if engine == "spacy": - raise ImportError( - "SpaCy engine requires additional dependencies. " - "Install with: pip install datafog[nlp]" - ) - # For auto mode, just continue with regex only - self.spacy_annotator = None - - def _chunk_text(self, text: str) -> List[str]: - """Split the text into chunks of specified length.""" - return [ - text[i : i + self.text_chunk_length] - for i in range(0, len(text), self.text_chunk_length) - ] - - def _combine_annotations( - self, chunk_annotations: List[Dict[str, List[str]]] - ) -> Dict[str, List[str]]: - """Combine annotations from multiple chunks.""" - combined = {} - for annotations in chunk_annotations: - for entity_type, entities in annotations.items(): - if entity_type not in combined: - combined[entity_type] = [] - combined[entity_type].extend(entities) - return combined - - def annotate_text_sync( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text synchronously for PII entities. - - Args: - text: The text to annotate - structured: If True, return structured Span objects. If False, return dict format. - - Returns: - Dictionary mapping entity types to lists of entities, or list of Span objects - """ - if len(text) <= self.text_chunk_length: - # Single chunk processing - if self.engine == "regex": - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return self.regex_annotator.annotate(text) - elif self.engine == "spacy": - if self.spacy_annotator is None: - raise ImportError( - "SpaCy engine not available. Install with: pip install datafog[nlp]" - ) - return self.spacy_annotator.annotate(text) - elif self.engine == "auto": - # Try regex first - regex_result = self.regex_annotator.annotate(text) - - # Check if regex found any entities - if any(entities for entities in regex_result.values()): - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return regex_result - - # Fall back to spacy if available - if self.spacy_annotator is not None: - return self.spacy_annotator.annotate(text) - - # Return regex result even if empty - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return regex_result - else: - # Multi-chunk processing - chunks = self._chunk_text(text) - chunk_annotations = [] - - for chunk in chunks: - chunk_result = self.annotate_text_sync(chunk, structured=False) - chunk_annotations.append(chunk_result) - - if structured: - # For structured output with chunking, we need to recalculate positions - # This is more complex, so for now return dict format - return self._combine_annotations(chunk_annotations) - - return self._combine_annotations(chunk_annotations) - - async def annotate_text_async( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text asynchronously for PII entities. - - Args: - text: The text to annotate - structured: If True, return structured Span objects. If False, return dict format. - - Returns: - Dictionary mapping entity types to lists of entities, or list of Span objects - """ - # For regex processing, we can just run synchronously since it's fast - return self.annotate_text_sync(text, structured) - - def batch_annotate_text_sync(self, texts: List[str]) -> List[Dict[str, List[str]]]: - """ - Annotate multiple texts synchronously. - - Args: - texts: List of texts to annotate - - Returns: - List of annotation dictionaries, one per input text - """ - return [self.annotate_text_sync(text) for text in texts] - - async def batch_annotate_text_async( - self, texts: List[str] - ) -> List[Dict[str, List[str]]]: - """ - Annotate multiple texts asynchronously. - - Args: - texts: List of texts to annotate - - Returns: - List of annotation dictionaries, one per input text - """ - # For better performance with many texts, we can process them concurrently - tasks = [self.annotate_text_async(text) for text in texts] - return await asyncio.gather(*tasks) diff --git a/datafog/services/text_service_original.py b/datafog/services/text_service_original.py deleted file mode 100644 index 8ad576a9..00000000 --- a/datafog/services/text_service_original.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Historical shadow text service module. - -This file is not the live DataFog 4.5 text service. Use -``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER, -auto, and smart engine boundary. This original snapshot remains importable for -legacy tests until cleanup can remove it safely. - -Text processing service for PII annotation. - -Provides synchronous and asynchronous methods for annotating text with personally identifiable information (PII) using SpaCy or regex patterns. Supports chunking long texts and batch processing. -""" - -import asyncio -from typing import Dict, List, Union - -from datafog.processing.text_processing.regex_annotator.regex_annotator import ( - RegexAnnotator, - Span, -) -from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator - - -class TextService: - """ - Service for annotating text with PII entities. - - This service provides methods to detect and annotate personally identifiable information (PII) - in text using different annotation engines. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. - """ - - def __init__(self, text_chunk_length: int = 1000, engine: str = "auto"): - """ - Initialize the TextService with specified chunk length and annotation engine. - - Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. - engine: The annotation engine to use. Options are: - - "regex": Use only the RegexAnnotator for pattern-based entity detection - - "spacy": Use only the SpacyPIIAnnotator for NLP-based entity detection - - "auto": (Default) Try RegexAnnotator first and fall back to SpacyPIIAnnotator if no entities are found - - Raises: - AssertionError: If an invalid engine type is provided - """ - assert engine in {"regex", "spacy", "auto"}, "Invalid engine" - self.engine = engine - self.spacy_annotator = SpacyPIIAnnotator.create() - self.regex_annotator = RegexAnnotator() - self.text_chunk_length = text_chunk_length - - def _chunk_text(self, text: str) -> List[str]: - """Split the text into chunks of specified length.""" - return [ - text[i : i + self.text_chunk_length] - for i in range(0, len(text), self.text_chunk_length) - ] - - def _combine_annotations( - self, annotations: List[Dict[str, List[str]]] - ) -> Dict[str, List[str]]: - """Combine annotations from multiple chunks.""" - combined: Dict[str, List[str]] = {} - for annotation in annotations: - for key, value in annotation.items(): - if key not in combined: - combined[key] = [] - combined[key].extend(value) - return combined - - def _annotate_with_engine( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text using the selected engine based on the engine parameter. - - This method implements the engine selection logic: - - For "regex" mode: Uses only the RegexAnnotator - - For "spacy" mode: Uses only the SpacyPIIAnnotator - - For "auto" mode: Tries RegexAnnotator first and falls back to SpacyPIIAnnotator if no entities are found - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary of annotations by entity type where keys are entity types (e.g., "EMAIL", "PERSON", "ORG") - and values are lists of detected entities of that type - If structured=True: List of Span objects with entity information - """ - if structured: - # Handle structured output mode - if self.engine == "regex": - _, annotation_result = self.regex_annotator.annotate_with_spans(text) - return annotation_result.spans - elif self.engine == "spacy": - # For spaCy, we need to convert the dictionary format to spans - spacy_dict = self.spacy_annotator.annotate(text) - spacy_spans: List[Span] = [] - for label, entities in spacy_dict.items(): - for entity in entities: - # Find the start and end positions of the entity in the text - start = text.find(entity) - if start >= 0: - end = start + len(entity) - span = Span(start=start, end=end, label=label, text=entity) - spacy_spans.append(span) - return spacy_spans - else: # "auto" mode - # Try regex first - _, annotation_result = self.regex_annotator.annotate_with_spans(text) - if annotation_result.spans: - return annotation_result.spans - - # If regex found nothing, fall back to spaCy - spacy_dict = self.spacy_annotator.annotate(text) - auto_spans: List[Span] = [] - for label, entities in spacy_dict.items(): - for entity in entities: - # Find the start and end positions of the entity in the text - start = text.find(entity) - if start >= 0: - end = start + len(entity) - span = Span(start=start, end=end, label=label, text=entity) - auto_spans.append(span) - return auto_spans - else: - # Handle legacy dictionary output mode - if self.engine == "regex": - return self.regex_annotator.annotate(text) - elif self.engine == "spacy": - return self.spacy_annotator.annotate(text) - else: # auto mode - # Try regex first - regex_dict = self.regex_annotator.annotate(text) - - # Check if any VALID entities were found (ignore empty strings) - has_entities = any( - any(entity.strip() for entity in entities) - for entities in regex_dict.values() - ) - - # If regex found entities, return those results - if has_entities: - return regex_dict - - # Otherwise, fall back to spaCy - return self.spacy_annotator.annotate(text) - - def annotate_text_sync( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Synchronously annotate a text string. - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary mapping entity types to lists of strings - If structured=True: List of Span objects with entity information - """ - if not text: - return [] if structured else {} - - chunks = self._chunk_text(text) - - if structured: - # Handle structured output mode - all_spans: List[Span] = [] - chunk_offset = 0 # Track the offset for each chunk in the original text - - for chunk in chunks: - # Process each chunk and get spans - chunk_spans = self._annotate_with_engine(chunk, structured=True) - if not isinstance(chunk_spans, list): - continue # Skip if not a list of spans - - # Adjust span positions based on chunk offset in the original text - for span in chunk_spans: - if not isinstance(span, Span): - continue # Skip if not a Span object - span.start += chunk_offset - span.end += chunk_offset - # Verify the span text matches the text at the adjusted position - if span.start < len(text) and span.end <= len(text): - span.text = text[span.start : span.end] - all_spans.append(span) - - # Update offset for the next chunk - chunk_offset += len(chunk) - - print(f"Done processing {text.split()[0]}") - return all_spans - else: - # Handle legacy dictionary output mode - annotations: List[Dict[str, List[str]]] = [] - for chunk in chunks: - res = self._annotate_with_engine(chunk) - if isinstance(res, dict): - annotations.append(res) - combined = self._combine_annotations(annotations) - print(f"Done processing {text.split()[0]}") - return combined - - def batch_annotate_text_sync( - self, texts: List[str], structured: bool = False - ) -> Dict[str, Union[Dict[str, List[str]], List[Span]]]: - """ - Synchronously annotate a list of text input. - - Args: - texts: List of text strings to annotate - structured: If True, return structured output (list of Span objects) for each text - - Returns: - Dictionary mapping each input text to its annotation result - """ - results = [ - self.annotate_text_sync(text, structured=structured) for text in texts - ] - return dict(zip(texts, results, strict=True)) - - async def annotate_text_async( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Asynchronously annotate a text string. - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary mapping entity types to lists of strings - If structured=True: List of Span objects with entity information - """ - if not text: - return [] if structured else {} - - chunks = self._chunk_text(text) - - if structured: - # Handle structured output mode asynchronously - all_spans: List[Span] = [] - chunk_offset = 0 # Track the offset for each chunk in the original text - - for chunk in chunks: - # We can't easily parallelize this due to the need to track offsets sequentially - # In a production environment, you might want a more sophisticated approach - chunk_spans = self._annotate_with_engine(chunk, structured=True) - if not isinstance(chunk_spans, list): - continue # Skip if not a list of spans - - # Adjust span positions based on chunk offset in the original text - for span in chunk_spans: - if not isinstance(span, Span): - continue # Skip if not a Span object - span.start += chunk_offset - span.end += chunk_offset - # Verify the span text matches the text at the adjusted position - if span.start < len(text) and span.end <= len(text): - span.text = text[span.start : span.end] - all_spans.append(span) - - # Update offset for the next chunk - chunk_offset += len(chunk) - - return all_spans - else: - # Handle legacy dictionary output mode asynchronously - tasks = [ - asyncio.to_thread(self._annotate_with_engine, chunk) for chunk in chunks - ] - results = await asyncio.gather(*tasks) - annotations: List[Dict[str, List[str]]] = [ - r for r in results if isinstance(r, dict) - ] - return self._combine_annotations(annotations) - - async def batch_annotate_text_async( - self, texts: List[str], structured: bool = False - ) -> Dict[str, Union[Dict[str, List[str]], List[Span]]]: - """ - Asynchronously annotate a list of text input. - - Args: - texts: List of text strings to annotate - structured: If True, return structured output (list of Span objects) for each text - - Returns: - Dictionary mapping each input text to its annotation result - """ - tasks = [ - self.annotate_text_async(text, structured=structured) for text in texts - ] - results = await asyncio.gather(*tasks) - return dict(zip(texts, results, strict=True)) diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md index 9a730c3b..6ea5db00 100644 --- a/docs/migration-4.10.md +++ b/docs/migration-4.10.md @@ -40,7 +40,7 @@ The native engine finds things the Python regex engine never did: `API_KEY`, stay opt-in. These differences go the other way and are recorded, not hidden, in -[tests/contracts/rust-1.3.0.json](../tests/contracts/rust-1.3.0.json): +[tests/contracts/rust-1.4.0.json](../tests/contracts/rust-1.4.0.json): | Input | Python | Native | Why | | --- | --- | --- | --- | @@ -105,5 +105,5 @@ a contract bump is a Core MAJOR and raises the floor here. ## What is not changing yet `detect()`, `process()`, OCR and Spark all still work and still warn. They are -removed in 5.0, not here. The last supported version is 4.10.0 — see the +removed in 5.0, not here. The last supported version in the current release train is 4.12.0 — see the [retirement schedule](retirement.rst). diff --git a/docs/retirement.rst b/docs/retirement.rst index 17c5c0af..01b2fc53 100644 --- a/docs/retirement.rst +++ b/docs/retirement.rst @@ -21,37 +21,37 @@ Removed or scheduled - Removed in - Deprecated in * - ``OCRMService`` - - ``4.10.0`` + - ``4.12.0`` - none - ``5.0.0`` - ``4.9.0`` * - ``DonutProcessor`` - - ``4.10.0`` + - ``4.12.0`` - none - ``5.0.0`` - ``4.9.0`` * - ``[ocr]`` extra (Donut, Tesseract, Pillow) - - ``4.10.0`` + - ``4.12.0`` - none - ``5.0.0`` - ``4.9.0`` * - ``[distributed]`` extra (PySpark, pandas) - - ``4.10.0`` + - ``4.12.0`` - :ref:`spark-recipe` - ``5.0.0`` - ``4.9.0`` * - ``detect()`` - - ``4.10.0`` + - ``4.12.0`` - ``scan()`` - ``5.0.0`` - ``4.9.0`` * - ``process()`` - - ``4.10.0`` + - ``4.12.0`` - ``redact()`` - ``5.0.0`` - ``4.9.0`` -Installing ``pip install "datafog==4.10.0"`` keeps every row above working. +After the 4.12.0 release is published, installing ``pip install "datafog==4.12.0"`` keeps every row above working. That version stays installable on PyPI permanently. OCR @@ -63,9 +63,9 @@ pin. .. code-block:: bash - pip install "datafog==4.10.0[ocr]" + pip install "datafog[ocr]==4.12.0" -If you need continued OCR support beyond that, stay on ``4.10.0``. Track the +If you need continued OCR support beyond that, stay on ``4.12.0``. Track the surfaces you depend on by name, not by staying on a major version range, so a lock file review shows the pin. diff --git a/docs/review-remediation.md b/docs/review-remediation.md new file mode 100644 index 00000000..bd7a67dd --- /dev/null +++ b/docs/review-remediation.md @@ -0,0 +1,134 @@ +# November 2026 review remediation + +DataFog 4.12.0 addresses the review of DataFog 4.11.0 and Core 1.4.0. The +review's suggested fixes were assessed against the local implementations; +regression tests cover the implemented behavior. + +## Observable behavior + +All engine modes and legacy scanning APIs share the structured scan boundary. +Optional NER adds spans with its own character offsets. TextService scans the +whole document; `text_chunk_length` remains validated for compatibility. +Unsupported labels, invalid ranges and invalid locales fail explicitly. Source +code-point offsets remain stable when lone surrogate characters are replaced +for native transport. `get_supported_entities()` reports supported labels, +including opt-in ones, rather than only currently active default labels. + +The `create_guardrail` decorator supports sync and async functions returning +strings. Non-string results raise TypeError; they cannot silently bypass the +filter. Apply a guardrail directly to the text fields of structured results. + +LiteLLM scans messages, nested tool/function arguments, prompt and input fields. +Image and binary parts are retained without OCR. Tokens are assigned per request +or response and repeated values reuse their token. Streamed response text and +tool arguments are buffered until completion, then scanned before any chunk is +emitted. This adds completion latency. Exceeding eight million text characters +or 100,000 chunks rejects the stream. Engine failures follow the explicit +`fail_policy`; deployments requiring protected egress should select `closed`. +The Claude Code hook requests review or denies oversized PreToolUse inputs; +nonblocking events receive an explicit incomplete-scan notice. Unexpected hook +failures retain the documented fail-open contract and log exception types only. + +`hash_type` selects md5, sha256 or sha3_256 on the CLI and legacy model APIs. +The engine also accepts fixed-length algorithms provided by hashlib. Existing +plain-hash marker formats remain compatible: engine, class facade and CLI +markers use 12 hex characters with SHA256 as their default; deprecated top-level +`process` uses eight hex characters and retains its MD5 default. Its explicit +`hash_type` option uses that same legacy width. The legacy Anonymizer model keeps +its existing full plain digests. Explicit keyed hashes use the full HMAC digest; +an omitted algorithm selects SHA256 for this new keyed mode. +Plain deterministic hashes are pseudonymization: SSNs, phones and +other guessable inputs can be enumerated. Use `hash_key` (at least 32 random +bytes) for keyed HMAC and keep the key outside logs and source control: + +```python +import secrets +import datafog +key = secrets.token_bytes(32) # store securely to reproduce pseudonyms +result = datafog.redact("mail jane@example.com", strategy="hash", hash_key=key) +``` + +## Release sequence and qualification + +Python 4.12.0 declares `datafog-core~=1.5.0` in its base +requirements and `rust` extra. Its reviewed native contract is +[`tests/contracts/rust-1.5.0.json`](../tests/contracts/rust-1.5.0.json). +Core 1.5.0 is published separately as a minor release with the detector and +transformation safety fixes from the review, including bounded mask reveal +and complete scanner coverage for automatic overlap handling. Public APIs +remain available and both packages continue to use capability contract 1. + +The actual PyPI Core 1.5.0 macOS ARM wheel was downloaded and verified against +its registry SHA-256, +`b3ebaca3c7751ec0288ffebec154ffbb6ba73e54b340f3c4e287e1946bbe5e87`. +All five runtime package files, including the native library, exactly match +the final qualified Core build from source `b183bede189ac35413120b995efcb9a29256b4b0`; +only the build SBOM and RECORD differ. Installing that published wheel into +the isolated Python 4.12 environment passes `pip check`, the installed native +bridge smoke test, and the complete non-slow suite: 1,143 passed, 21 skipped, +19 expected failures. The 726-pass real-model/native corpus qualification +continues to apply through the verified byte-identical runtime payload. +Hosted release gates independently qualify published Core on all required +Python and platform combinations before publishing Python 4.12. + +Scheduled Python jobs only qualify candidates. Manual `release.yml` runs default +to `dry_run: true`; explicitly selecting false is the publication boundary. +Candidate bases must be newer than every stable Git tag and the latest stable +PyPI release. Development qualification refuses a ref missing `main` history. +The workflow atomically pushes the tested version commit and immutable tag +before uploading to PyPI, then fast-forwards `dev` after a stable publication. +The user must authorize deployment before any publishing workflow is dispatched. + +If registry upload fails after the atomic ref push, keep the tag. Inspect PyPI +for partial artifacts, compare hashes to the retained workflow artifact and tag, +and resume only the missing upload from those exact validated artifacts. Do not +move the tag, reuse the version for different bytes, or rerun version generation +with an existing release tag. A changed branch must be qualified again. + +Core's individual registries have separate availability; never infer that a +Python version is present on npm or crates.io. Native wheel coverage and source +build requirements are documented in Core's platform policy. + +## Python finding disposition + +All 35 Python findings have a concrete resolution below. Core-owned fixes are +separately tracked in Core's `docs/reviews/2026-11-core-findings.md`; their combined +release includes the published-artifact qualification above. + +| Finding | Resolution and proof | +| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| PY-H1 | Replace lone surrogates only for native transport, preserving source coordinates. Engine and hook/LiteLLM block regressions cover mixed payloads. | +| PY-H2 | Interval queries replace pairwise overlap checks; redaction and legacy assembly are linear after sorting. Dense scan/redaction benchmarks and overlap tests cover the change. | +| PY-H3 | TextService scans complete documents through the shared engine. Boundary-crossing email/SSN tests pass. | +| PY-H4 | spaCy/GLiNER return actual character spans. Stub and real-spaCy tests distinguish Ann from Annual and repeated names. | +| PY-H5 | Every engine and legacy/CLI path uses the same structured layer before optional NER. Discover/IPv6/credential tests cover the paths. | +| PY-M1 | Coroutine decorators await results; non-string results raise a documented TypeError rather than silently escaping inspection. | +| PY-M2 | Redact, replace and hash pipeline operations imply scanning. All three are tested without an explicit SCAN operation. | +| PY-M3 | Fallback email consumes each lexical run once; German separator expressions avoid ambiguous repetition. Adversarial 80K-character measurements and positive matches pass. | +| PY-M4 | Native backend selection also supplies the structured stage of smart/spaCy/GLiNER. CLI commands work with DATAFOG_BACKEND=rust. | +| PY-M5 | Request-scoped tokens, nested tool/function arguments, prompt/input fields and buffered streaming protect all tested request/response shapes. Live proxy deployment is not claimed. | +| PY-M6 | Bare strings, invalid labels and labels unavailable to the selected backend/engine raise explicit errors. Supported aliases remain accepted. | +| PY-M7 | APIs and CLI honor hash_type, preserve existing plain-hash formats and support full-length keyed HMAC. Plain-hash guessing limitations are documented; algorithm/key and compatibility tests pass. | +| PY-M8 | Hook per-string/total budget exhaustion emits review/deny or an incomplete-scan notice. Tests put PII past the cap. | +| PY-M9 | Allowlist expressions run with match timeouts; timed-out expressions retain findings. Adversarial patterns and safe quantified groups are tested. | +| PY-M10 | Dependency resolution stays within the explicitly reviewed Core minor, replacing the broad less-than-2 range. Exact output locks should still pin patch versions. | +| PY-M11 | Reconcile stable history, reject stale/PyPI-occupied versions, gate 3.13, sync version metadata and push immutable refs before upload. Scheduled runs are nonpublishing; selector tests and actionlint pass. | +| PY-M12 | Spark maps ordinary PERSON/ORG/GPE/NORP/DATE labels into its fixed output schema. Stub tests cover real spaCy label spellings and oversized input errors. | +| PY-M13 | Optional Pydantic fields have defaults, labels use canonical names, enum filters work and overlaps merge before replacement. Model/anonymizer regressions pass. | +| PY-M14 | Preserve supported GLiNER labels, use exact offsets, exclude relative spaCy DATE mentions, propagate model failures and reject oversized annotator input explicitly. | +| PY-M15 | Embedded model calls use bounded overlapping chunks. Large PERSON/ADDRESS plus structured-PII tests succeed with published and candidate Core. Native model limits remain enforced. | +| PY-M16 | Core 1.5 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | +| PY-L1 | Console bootstrap prints the CLI-extra install hint on a base install. A clean installed-wheel check confirms no traceback. | +| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 1.5 suppresses the labelled-tax-ID PHONE false positive. | +| PY-L3 | Retirement notices derive the current supported release version, and the native install hint no longer names an older release. | +| PY-L4 | Supported-entity reporting follows the selected native/Python capability inventory, including opt-in labels. Tests separate supported from default-active labels. | +| PY-L5 | Document actual token/pseudonym outputs and reject invalid methods in both process APIs. | +| PY-L6 | Legacy APIs and TextPIIAnnotator route through the shared engine; service/engine integration covers native-only entities and locale behavior. Explicit Python fallback remains a narrower documented detector set. | +| PY-L7 | Tokens reuse values and avoid existing markers; invalid explicit ranges/text raise errors. Collision, repeated-value and validation tests pass. | +| PY-L8 | Remove unused implementation copies, declare an isolated build, include test packages/corpora/scripts in sdists, align formatter pins and extend tox through 3.14. Optional OCR imports skip cleanly when absent. | +| PY-L9 | CLI configuration hides API keys; import and commands preserve host logging levels. CLI/logger regressions pass. | +| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 1.5 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | +| PY-L11 | TextService validates positive chunk lengths and engine names with explicit exceptions, including optimized Python execution. | +| PY-L12 | Remove incompatible Torch upper bound and include spaCy in nlp-advanced. Resolving it with Core GLiNER2 succeeds with Torch 2.12.0. | +| PY-L13 | Declare Pydantic 2.1 minimum. A clean Python 3.10 base install with Pydantic 2.1 imports and scans successfully. | +| PY-L14 | Correct native-default documentation, normalize missing native metadata to ImportError, clarify missing-model hints, validate locales consistently and surface invalid CLI operations as usage errors. | diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..cb600497 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/scripts/check_rust_install.py b/scripts/check_rust_install.py index bf84c45d..57604d3f 100644 --- a/scripts/check_rust_install.py +++ b/scripts/check_rust_install.py @@ -3,6 +3,8 @@ import importlib.metadata import sys +from packaging.requirements import Requirement + def main(): assert sys.flags.isolated, "Run with python -I" @@ -14,7 +16,20 @@ def main(): from datafog.compat import v4 installed = importlib.metadata.version("datafog-core") - assert installed.split(".")[0] == "1", installed + requirements = [ + Requirement(value) for value in importlib.metadata.requires("datafog") or [] + ] + core_requirements = [ + requirement + for requirement in requirements + if requirement.name == "datafog-core" and requirement.marker is None + ] + assert ( + core_requirements + ), "Installed DataFog must declare its Core compatibility range" + assert all( + installed in requirement.specifier for requirement in core_requirements + ), installed assert datafog_core.capabilities()["contract_version"] == 1 assert v4.Entity is datafog.Entity assert v5.Finding is datafog_core.Finding diff --git a/scripts/release_version.py b/scripts/release_version.py new file mode 100644 index 00000000..f78afd28 --- /dev/null +++ b/scripts/release_version.py @@ -0,0 +1,75 @@ +"""Select a release version while refusing stale or already published bases.""" + +import argparse +import json +import re +import subprocess +import urllib.request +from pathlib import Path + +from packaging.version import Version + + +def select_version( + current, release_type, tags, latest_stable, override=None, published=() +): + if release_type not in {"alpha", "beta", "stable"}: + raise ValueError("Unknown release type") + source = Version(current) + occupied = set(tags) | {f"v{Version(version)}" for version in published} + base = override or source.base_version + if not re.fullmatch(r"\d+\.\d+\.\d+", base): + raise ValueError("Release base must be numeric major.minor.patch") + if Version(base) <= Version(latest_stable): + raise ValueError("Release base must be newer than the latest stable release") + if release_type == "stable": + candidate = base + else: + suffix = "a" if release_type == "alpha" else "b" + number = 1 + while f"v{base}{suffix}{number}" in occupied: + number += 1 + candidate = f"{base}{suffix}{number}" + if f"v{candidate}" in occupied: + raise ValueError( + "Release tag already exists; inspect publication state before retrying" + ) + return candidate + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--type", required=True, choices=["alpha", "beta", "stable"]) + parser.add_argument("--override", default="") + parser.add_argument( + "--latest-stable", help="Offline preflight only; workflow reads PyPI" + ) + args = parser.parse_args() + tags = subprocess.check_output(["git", "tag", "-l"], text=True).splitlines() + stable_tags = [tag[1:] for tag in tags if re.fullmatch(r"v\d+\.\d+\.\d+", tag)] + if args.latest_stable: + published = args.latest_stable + published_versions = [published] + else: + with urllib.request.urlopen( + "https://pypi.org/pypi/datafog/json", timeout=30 + ) as response: + payload = json.load(response) + releases = [ + Version(version) + for version in payload["releases"] + if not Version(version).is_prerelease + ] + published = str(max(releases)) + published_versions = list(payload["releases"]) + latest = str(max(Version(version) for version in [published, *stable_tags])) + about = Path("datafog/__about__.py") + current = re.search(r'^__version__ = "([^"]+)"', about.read_text(), re.M).group(1) + candidate = select_version( + current, args.type, tags, latest, args.override or None, published_versions + ) + print(candidate) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 7ce0366d..76049bfd 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,13 @@ # Core lightweight dependencies only core_deps = [ - # Native engine, the default detection path. Exact upper bound: detection - # output is contract-versioned and must stay reproducible. - "datafog-core>=1.4.0,<2", - "pydantic>=2.0,<3.0", + # Core minor releases may change findings; stay on this reviewed minor. + "datafog-core~=1.5.0", + "pydantic>=2.1,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", + # Match timeouts bound untrusted allowlist expressions. + "regex>=2024.11.6", ] # Optional heavy dependencies @@ -34,7 +35,7 @@ nlp_advanced_deps = [ "gliner>=0.2.5", - "torch>=2.1.0,<2.7", + "torch>=2.1.0,<3", "transformers>=4.20.0", "huggingface-hub>=0.16.0", "sentencepiece>=0.2.0", @@ -88,9 +89,9 @@ extras_require = { # Retained for explicitness; the base install already pulls Core. - "rust": ["datafog-core>=1.4.0,<2"], + "rust": ["datafog-core~=1.5.0"], "nlp": nlp_deps, - "nlp-advanced": nlp_advanced_deps, + "nlp-advanced": nlp_deps + nlp_advanced_deps, "ocr": ocr_deps, "distributed": distributed_deps, "web": web_deps, @@ -117,6 +118,8 @@ version=version, author="Sid Mohan", author_email="sid@datafog.ai", + license="MIT", + license_files=["LICENSE"], description="Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)", long_description=long_description, long_description_content_type="text/markdown", @@ -127,14 +130,13 @@ python_requires=">=3.10", entry_points={ "console_scripts": [ - "datafog=datafog.client:app [cli]", # Requires cli extra + "datafog=datafog.cli:main", "datafog-hook=datafog.integrations.claude_code:main", # Core only ], }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/setup_lean.py b/setup_lean.py deleted file mode 100644 index 57315788..00000000 --- a/setup_lean.py +++ /dev/null @@ -1,109 +0,0 @@ -from setuptools import find_packages, setup - -# Historical shadow packaging snapshot. The live DataFog packaging input is -# setup.py. Do not use this file for builds, releases, or dependency changes. - -# Read README for the long description -with open("README.md", "r") as f: - long_description = f.read() - -# Use a single source of truth for the version -__version__ = "4.1.1" - -project_urls = { - "Homepage": "https://datafog.ai", - "Documentation": "https://docs.datafog.ai", - "Discord": "https://discord.gg/bzDth394R4", - "Twitter": "https://twitter.com/datafoginc", - "GitHub": "https://github.com/datafog/datafog-python", -} - -# Core lightweight dependencies only -core_deps = [ - "pydantic>=2.0,<3.0", - "typing-extensions>=4.0", -] - -# Optional heavy dependencies -extras_require = { - "nlp": [ - "spacy>=3.7.0,<4.0", - ], - "ocr": [ - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - ], - "distributed": [ - "pandas>=2.0.0", - "numpy>=1.24.0", - ], - "web": [ - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - ], - "cli": [ - "typer>=0.12.0", - ], - "crypto": [ - "cryptography>=40.0.0", - ], - "dev": [ - "pytest>=7.0.0", - "pytest-asyncio>=0.21.0", - "pytest-cov>=4.0.0", - "sphinx>=7.0.0", - ], - # Convenience bundles - "all": [ - "spacy>=3.7.0,<4.0", - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - "pandas>=2.0.0", - "numpy>=1.24.0", - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - "typer>=0.12.0", - "cryptography>=40.0.0", - ], -} - -setup( - name="datafog", - version=__version__, - author="Sid Mohan", - author_email="sid@datafog.ai", - description="Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)", - long_description=long_description, - long_description_content_type="text/markdown", - packages=find_packages(), - install_requires=core_deps, - extras_require=extras_require, - python_requires=">=3.10", - entry_points={ - "console_scripts": [ - "datafog=datafog.client:app [cli]", # Requires cli extra - ], - }, - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Text Processing", - "Topic :: Security", - ], - project_urls=project_urls, - keywords="pii detection anonymization privacy regex performance", -) diff --git a/setup_original.py b/setup_original.py deleted file mode 100644 index 006a3608..00000000 --- a/setup_original.py +++ /dev/null @@ -1,92 +0,0 @@ -from setuptools import find_packages, setup - -# Historical shadow packaging snapshot. The live DataFog packaging input is -# setup.py. Do not use this file for builds, releases, or dependency changes. - -# Read README for the long description -with open("README.md", "r") as f: - long_description = f.read() - -# Use a single source of truth for the version -__version__ = "4.1.1" - -project_urls = { - "Homepage": "https://datafog.ai", - "Documentation": "https://docs.datafog.ai", - "Discord": "https://discord.gg/bzDth394R4", - "Twitter": "https://twitter.com/datafoginc", - "GitHub": "https://github.com/datafog/datafog-python", -} - -setup( - name="datafog", - version=__version__, - author="Sid Mohan", - author_email="sid@datafog.ai", - description="Scan, redact, and manage PII in your documents before they get uploaded to a Retrieval Augmented Generation (RAG) system.", - long_description=long_description, - long_description_content_type="text/markdown", - packages=find_packages(), - install_requires=[ - "pandas", - "requests==2.32.3", - "spacy==3.7.5", - "pydantic", - "Pillow", - "sentencepiece", - "protobuf", - "pytesseract", - "aiohttp", - "pytest-asyncio", - "numpy", - "fastapi", - "asyncio", - "setuptools", - "pydantic-settings==2.3.4", - "typer==0.12.3", - "sphinx", - "cryptography", - ], - python_requires=">=3.10", - entry_points={ - "console_scripts": [ - "datafog=datafog.client:app", - ], - }, - classifiers=[ - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Framework :: tox", - "Framework :: Pytest", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "Intended Audience :: Information Technology", - "Intended Audience :: System Administrators", - ], - keywords="pii, redaction, nlp, rag, retrieval augmented generation", - maintainer="DataFog", - maintainer_email="hi@datafog.ai", - url="https://datafog.ai", - project_urls=project_urls, - license="MIT", - extras_require={ - "dev": [ - "just", - "isort", - "black", - "blacken-docs", - "flake8", - "tox", - "pytest", - "pytest-codeblocks", - "pytest-cov", - "build", - "twine", - ], - }, -) diff --git a/tests/benchmark_text_service.py b/tests/benchmark_text_service.py index 3178c61e..e802c40e 100644 --- a/tests/benchmark_text_service.py +++ b/tests/benchmark_text_service.py @@ -107,7 +107,7 @@ def test_spacy_performance(benchmark, sample_text_10kb, spacy_service): # Verify spaCy found expected entities assert "PERSON" in result or "PER" in result - assert "ORG" in result + assert "ORGANIZATION" in result # Print some stats about the results entity_counts = {key: len(values) for key, values in result.items() if values} @@ -178,7 +178,7 @@ def test_auto_engine_fallback_performance(benchmark, spacy_only_text, auto_servi ) # Check if we have spaCy entities (depends on spaCy availability) - spacy_entities = ["PERSON", "ORG", "GPE", "CARDINAL", "DATE", "TIME"] + spacy_entities = ["PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"] has_spacy_entities = any(entity in result for entity in spacy_entities) # If no spaCy entities, check if spaCy is available diff --git a/tests/contracts/rust-1.5.0.json b/tests/contracts/rust-1.5.0.json new file mode 100644 index 00000000..13f39961 --- /dev/null +++ b/tests/contracts/rust-1.5.0.json @@ -0,0 +1,77 @@ +{ + "core_version": "1.5.0", + "cases": { + "scan-observation-invalid-card": { + "classification": "detector-difference", + "reason": "Core validates card checksums; Python 4.8.1 accepts this card-shaped value. Experimental precision difference, not a claim of full parity.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "4111 1111 1111 1112", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "scan-observation-boundary": { + "classification": "detector-difference", + "reason": "Core rejects the SSN embedded in an ASCII alphanumeric token; Python 4.8.1 detects the numeric substring.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "x123-45-6789y", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "locale-unknown": { + "classification": "validation-difference", + "reason": "The capability adapter rejects the unadvertised locale with a backend-specific diagnostic instead of the Python regex locale inventory. The exception remains ValueError.", + "expected": { + "error": { + "type": "ValueError", + "message": "Unsupported locale for the Rust backend: 'xx'" + }, + "warnings": [] + } + }, + "scan-observation-ambiguous-number": { + "classification": "detector-difference", + "reason": "Core's North American area-code check requires a leading digit of 2-9, so a bare 10-digit run starting with 1 stays undetected. This is the deliberate Unix-timestamp protection; Python 4.8.1 matches it.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Invoice 1234567890", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "locale-default-DE_TAX_ID": { + "classification": "detector-difference", + "reason": "CORE-17/PY-L2 suppress a German tax identifier from the generic PHONE detector even when German detectors are not activated. Explicit locale selection still detects DE_TAX_ID.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Steuer-ID 12345678901 liegt vor.", + "engine_used": "regex" + } + }, + "warnings": [] + } + } + }, + "review": "Core 1.5 candidate qualification retains reviewed historical differences and documents the contextual tax-ID correction. Wrapper 4.12 fixes are layered separately in wrapper-4.12.0.json." +} diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json new file mode 100644 index 00000000..cd78ae0d --- /dev/null +++ b/tests/contracts/wrapper-4.12.0.json @@ -0,0 +1,972 @@ +{ + "wrapper_version": "4.12.0", + "core_versions": ["1.4.0", "1.5.0"], + "source_oracle": "4.8.1.json (unchanged)", + "review": "2026-11-15 PDF remediation: explicit reviewed wrapper changes, not a regenerated historical oracle", + "cases": { + "selection-unknown": { + "classification": "validation-difference", + "reason": "PY-M6 rejects unknown entity labels instead of silently disabling detection.", + "expected": { + "error": { + "type": "ValueError", + "message": "Unsupported entity_types for rust/regex: NOT_A_TYPE" + }, + "warnings": [] + } + }, + "redact-token": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses one collision-safe token for repeated values.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_2] [EMAIL_1] [SSN_1]", + "mapping": { + "[EMAIL_1]": "a@example.com", + "[EMAIL_2]": "b@example.com", + "[SSN_1]": "123-45-6789" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "SSN", + "text": "123-45-6789", + "start": 42, + "end": 53, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "engine-redact-token": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses one collision-safe token for repeated values.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_2] [EMAIL_1]", + "mapping": { + "[EMAIL_1]": "a@example.com", + "[EMAIL_2]": "b@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "explicit-invalid-bounds": { + "classification": "validation-difference", + "reason": "PY-L7 rejects malformed explicit spans rather than silently skipping them.", + "expected": { + "error": { + "type": "ValueError", + "message": "entities must contain valid ranges whose text matches the source" + }, + "warnings": [] + } + }, + "explicit-mismatched-text": { + "classification": "validation-difference", + "reason": "PY-L7 rejects explicit span text that differs from the source.", + "expected": { + "error": { + "type": "ValueError", + "message": "entities must contain valid ranges whose text matches the source" + }, + "warnings": [] + } + }, + "error-invalid-pattern": { + "classification": "validation-difference", + "reason": "PY-M9 uses a timeout-capable matcher, whose syntax diagnostic counts positions differently.", + "expected": { + "error": { + "type": "ValueError", + "message": "allowlist_patterns contains an invalid regex: '[' (unterminated character set at position 1)" + }, + "warnings": [] + } + }, + "error-unsafe-pattern": { + "classification": "behavioral-fix", + "reason": "PY-M9 allows nested groups under a hard matching timeout instead of a bypassable syntax heuristic.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "error-oversized-pattern": { + "classification": "validation-difference", + "reason": "PY-M9 validates both pattern type and length with an explicit diagnostic.", + "expected": { + "error": { + "type": "ValueError", + "message": "allowlist_patterns entries must be strings of at most 512 characters" + }, + "warnings": [] + } + }, + "facade-sanitize": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": "[EMAIL_1] [EMAIL_1]", + "warnings": [] + } + }, + "facade-filter_output": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_1]", + "mapping": { + "[EMAIL_1]": "alice@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 0, + "end": 17, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 18, + "end": 35, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "facade-detect": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + }, + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 18, + "end": 35 + } + ], + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.detect() is deprecated and will be removed in 5.0. Use datafog.scan() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "facade-anonymize_text": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": "[EMAIL_1] [EMAIL_1]", + "warnings": [] + } + }, + "facade-process": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + }, + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 18, + "end": 35 + } + ] + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-redact": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_REDACTED]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-replace": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_XXXXX]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-hash": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release. Legacy default MD5 and its eight-character prefix remain compatible; explicit algorithms and keyed HMAC are additional options.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_c160f8cc]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-invalid": { + "classification": "validation-difference", + "reason": "PY-L5 rejects an invalid anonymization method instead of choosing an undocumented fallback; PY-L3 updates the notice.", + "expected": { + "error": { + "type": "ValueError", + "message": "method must be 'redact', 'replace', or 'hash'" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "service-default": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route TextService through shared native detection and return populated canonical labels.", + "expected": { + "value": { + "EMAIL": ["alice@example.com"] + }, + "warnings": [] + } + }, + "service-german": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route TextService through shared native detection with overlap resolution and populated canonical labels.", + "expected": { + "value": { + "DE_TAX_ID": ["12345678901"] + }, + "warnings": [] + } + }, + "service-batch": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route every TextService batch member through shared native detection, returning populated canonical labels.", + "expected": { + "value": [ + { + "EMAIL": ["alice@example.com"] + }, + {} + ], + "warnings": [] + } + }, + "public-signatures": { + "classification": "signature-addition", + "reason": "PY-M7 adds keyword-only hash_type/hash_key while preserving existing unkeyed output widths and omitted legacy hash defaults; TextService custom models use the engine gliner_model keyword; backend remains keyword-only.", + "expected": { + "value": { + "datafog:scan": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "preset", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:protect": { + "parameters": [ + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "on_detect", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + } + ] + }, + "datafog:detect": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:process": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "anonymize", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": false + }, + { + "name": "method", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:sanitize": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:scan_prompt": { + "parameters": [ + { + "name": "prompt", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:filter_output": { + "parameters": [ + { + "name": "output", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:create_guardrail": { + "parameters": [ + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "on_detect", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + } + ] + }, + "datafog.engine:Entity": { + "parameters": [ + { + "name": "type", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "start", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "end", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "confidence", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": ["type", "text", "start", "end", "confidence", "engine"] + }, + "datafog.engine:ScanResult": { + "parameters": [ + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine_used", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": ["entities", "text", "engine_used"] + }, + "datafog.engine:RedactResult": { + "parameters": [ + { + "name": "redacted_text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "mapping", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": ["redacted_text", "mapping", "entities"] + }, + "datafog.engine:scan": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "smart" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "gliner_model", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog.engine:redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog.engine:scan_and_redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "smart" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + } + }, + "warnings": [] + } + } + } +} diff --git a/tests/corpus/negative_cases.json b/tests/corpus/negative_cases.json index 44eb1fcf..c5b67b76 100644 --- a/tests/corpus/negative_cases.json +++ b/tests/corpus/negative_cases.json @@ -49,7 +49,16 @@ { "id": "markdown-link", "input": "See [RFC 1918](https://example.com/rfc1918)", - "expected_entities": [] + "expected_entities": [], + "permitted_gliner_entities": [ + { + "type": "URL", + "text": "https://example.com/rfc1918", + "start": 15, + "end": 42 + } + ], + "permitted_gliner_reason": "PY-M14 restores the existing GLiNER URL label. This exact public URL is valid URL detection; it must not become EMAIL, PHONE or IP_ADDRESS." }, { "id": "code-symbol", @@ -74,6 +83,15 @@ { "id": "url-with-at", "input": "https://example.com/@user/profile is a URL path", - "expected_entities": [] + "expected_entities": [], + "permitted_gliner_entities": [ + { + "type": "URL", + "text": "https://example.com/@user/profile", + "start": 0, + "end": 33 + } + ], + "permitted_gliner_reason": "PY-M14 restores the existing GLiNER URL label. This exact public URL is valid URL detection; it must not become EMAIL, PHONE or IP_ADDRESS." } ] diff --git a/tests/corpus/unstructured_pii.json b/tests/corpus/unstructured_pii.json index ad91c35b..1e5ddf79 100644 --- a/tests/corpus/unstructured_pii.json +++ b/tests/corpus/unstructured_pii.json @@ -168,8 +168,8 @@ { "type": "LOCATION", "text": "Jordan", - "start": 0, - "end": 6 + "start": 33, + "end": 39 } ] }, diff --git a/tests/rust_contract.py b/tests/rust_contract.py index 2b618007..9e673091 100644 --- a/tests/rust_contract.py +++ b/tests/rust_contract.py @@ -9,7 +9,11 @@ from tests.contract_481 import FIXTURE, observe -DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.4.0.json" +CORE_VERSION = importlib.metadata.version("datafog-core") +CORE_LINE = ".".join(CORE_VERSION.split(".")[:2]) +DEVIATIONS = Path(__file__).parent / "contracts" / f"rust-{CORE_LINE}.0.json" +WRAPPER_DEVIATIONS = Path(__file__).parent / "contracts" / "wrapper-4.12.0.json" + TARGETS = { "datafog:scan", "datafog:redact", @@ -20,6 +24,18 @@ } +def reviewed_deviation(case): + """Layer explicit wrapper fixes on the unchanged historical wheel oracle.""" + wrapper = json.loads(WRAPPER_DEVIATIONS.read_text(encoding="utf-8")) + native = json.loads(DEVIATIONS.read_text(encoding="utf-8")) + return native["cases"].get(case["id"]) or wrapper["cases"].get(case["id"]) + + +def expected_observation(case): + deviation = reviewed_deviation(case) + return copy.deepcopy(deviation["expected"] if deviation else case["expected"]) + + def applicable(case): return ( case.get("operation", "call") == "call" @@ -39,15 +55,14 @@ def main(): parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() contract = json.loads(FIXTURE.read_text(encoding="utf-8")) - deviations = json.loads(DEVIATIONS.read_text(encoding="utf-8")) results = [] for case in contract["cases"]: if not applicable(case): results.append({"id": case["id"], "status": "outside-backend-scope"}) continue actual = native_observation(case) - deviation = deviations["cases"].get(case["id"]) - expected = deviation["expected"] if deviation else case["expected"] + deviation = reviewed_deviation(case) + expected = expected_observation(case) status = "regression" if actual != expected else "match" if status == "match" and deviation: status = deviation["classification"] diff --git a/tests/simple_performance_test.py b/tests/simple_performance_test.py index 6da82d88..589ef59b 100644 --- a/tests/simple_performance_test.py +++ b/tests/simple_performance_test.py @@ -90,7 +90,7 @@ def test_simple_spacy_performance(): # Verify functionality assert "PERSON" in stats["result"] or "PER" in stats["result"] - assert "ORG" in stats["result"] + assert "ORGANIZATION" in stats["result"] return stats diff --git a/tests/test_allowlist.py b/tests/test_allowlist.py index f737a9f2..fc4b2865 100644 --- a/tests/test_allowlist.py +++ b/tests/test_allowlist.py @@ -93,13 +93,15 @@ def test_patterns_and_values_combine(self): class TestReDoSGuards: - def test_catastrophic_pattern_rejected(self): - with pytest.raises(ValueError, match="catastrophic backtracking"): - datafog.scan("text", engine="regex", allowlist_patterns=[r"(a+)+$"]) - - def test_nested_star_rejected(self): - with pytest.raises(ValueError, match="catastrophic backtracking"): - datafog.scan("text", engine="regex", allowlist_patterns=[r"(.*)*"]) + def test_nested_quantifier_uses_bounded_matching(self): + assert datafog.scan( + "a@b.com", engine="regex", allowlist_patterns=[r"(a+)+$"] + ).entities + + def test_safe_nested_star_is_supported(self): + assert not datafog.scan( + "a@b.com", engine="regex", allowlist_patterns=[r"(.*)*"] + ).entities def test_overlong_pattern_rejected(self): with pytest.raises(ValueError, match="at most"): diff --git a/tests/test_backend_resolution.py b/tests/test_backend_resolution.py index c6cd6d60..55abb06d 100644 --- a/tests/test_backend_resolution.py +++ b/tests/test_backend_resolution.py @@ -33,10 +33,10 @@ def test_auto_resolves_to_python_without_core(without_core): assert engine.resolve_backend(None, "regex") == "python" -def test_auto_keeps_non_regex_engines_on_python(): +def test_auto_uses_native_structured_layer_for_every_engine(): pytest.importorskip("datafog_core") for engine_name in ("smart", "spacy", "gliner"): - assert engine.resolve_backend(None, engine_name) == "python" + assert engine.resolve_backend(None, engine_name) == "rust" def test_python_fallback_warns_once_then_stays_quiet(without_core): diff --git a/tests/test_client.py b/tests/test_client.py index d5b042d1..24734be1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -165,7 +165,7 @@ def test_anonymizer_outputs(): ), AnnotationResult( start=33, - end=41, + end=len(text), score=1.0, entity_type=EntityTypes.LOCATION, recognition_metadata=AnnotatorMetadata(), diff --git a/tests/test_contract_481.py b/tests/test_contract_481.py index 43863dfd..b54ecf1e 100644 --- a/tests/test_contract_481.py +++ b/tests/test_contract_481.py @@ -1,64 +1,33 @@ """Frozen observations from the published 4.8.1 wheel; never regenerate in CI.""" -import copy import json import pytest from tests.contract_481 import FIXTURE, observe -from tests.rust_contract import DEVIATIONS +from tests.rust_contract import WRAPPER_DEVIATIONS, expected_observation CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) -# The default backend is native from 4.10.0, so the reviewed native -# differences are the published 4.10 observations for those cases. -REVIEWED = json.loads(DEVIATIONS.read_text(encoding="utf-8")) +REVIEWED_WRAPPER = json.loads(WRAPPER_DEVIATIONS.read_text(encoding="utf-8")) def expected_49(case): - """Allow only the added backend parameter, revised shim notices and the - reviewed native differences.""" - - deviation = REVIEWED["cases"].get(case["id"]) - expected = ( - copy.deepcopy(deviation["expected"]) - if deviation - else copy.deepcopy(case["expected"]) - ) - if case["id"] == "public-signatures": - for target in ( - "datafog:scan", - "datafog:redact", - "datafog.engine:scan", - "datafog.engine:scan_and_redact", - ): - expected["value"][target]["parameters"].append( - { - "name": "backend", - "kind": "KEYWORD_ONLY", - "required": False, - "default": None, - } - ) - if case.get("target") in {"datafog:detect", "datafog:process"}: - name = case["target"].split(":")[1] - replacement = ( - "datafog.scan()" - if name == "detect" - else "datafog.scan() or datafog.redact()" - ) - expected["warnings"] = [ - { - "type": "FutureWarning", - "message": ( - f"datafog.{name}() is deprecated and will be removed in 5.0. " - f"Use {replacement} instead. " - 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0". ' - "The earlier promise to retain this shim through 5.x has been revised. " - "See docs/retirement.rst." - ), - } - ] - return expected + """The immutable oracle plus individually explained native/wrapper fixes.""" + return expected_observation(case) + + +def test_reviewed_wrapper_deviations_are_explicit_and_in_scope(): + cases = {case["id"]: case for case in CONTRACT["cases"]} + assert set(REVIEWED_WRAPPER["cases"]) <= set(cases) + for case_id, deviation in REVIEWED_WRAPPER["cases"].items(): + assert deviation["reason"] + assert deviation["classification"] in { + "behavioral-fix", + "validation-difference", + "diagnostic-difference", + "signature-addition", + } + assert deviation["expected"] != cases[case_id]["expected"] @pytest.mark.parametrize("case", CONTRACT["cases"], ids=lambda case: case["id"]) diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index 8cd218a9..9f952bac 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -1,4 +1,4 @@ -"""Exercise actual Core 0.4 wheels through the Python compatibility adapter. +"""Exercise reviewed native wheels through the Python compatibility adapter. Examples originate in Core's fixtures/{german,jwt,private-key,npi, us-routing-number,uuid}.jsonl. Payloads are synthetic detector fixtures, @@ -6,6 +6,7 @@ """ import importlib.metadata +import json import re import pytest @@ -13,6 +14,7 @@ import datafog from datafog import engine, v5 from datafog.compat import v4 +from tests.rust_contract import DEVIATIONS core = pytest.importorskip("datafog_core") @@ -50,7 +52,8 @@ def test_supported_wheel_version_and_capability_contract(): version = importlib.metadata.version("datafog-core") - assert version.split(".")[:1] == ["1"], version + reviewed = json.loads(DEVIATIONS.read_text())["core_version"] + assert version.split(".")[:2] == reviewed.split(".")[:2], version capabilities = core.capabilities() assert capabilities["contract_version"] == 1 supported = capabilities["supported_entities"] @@ -210,9 +213,8 @@ def test_contextual_npi_survives_the_legacy_overlap_policy(): def test_structured_only_person_served_by_the_embedded_model(): """`PERSON` is structured-only in the rule inventory but the embedded CNN covers prose, so text selection succeeds instead of raising.""" - assert not datafog.scan( - "Jane Doe", entity_types=["PERSON"], backend="python" - ).entities + with pytest.raises(ValueError, match="Unsupported entity_types"): + datafog.scan("Jane Doe", entity_types=["PERSON"], backend="python") result = datafog.scan("Jane Doe\n14 Penny Lane", entity_types=["PERSON", "ADDRESS"]) assert [item.type for item in result.entities] == ["PERSON", "ADDRESS"] diff --git a/tests/test_core15_integration.py b/tests/test_core15_integration.py new file mode 100644 index 00000000..ed9bcdf4 --- /dev/null +++ b/tests/test_core15_integration.py @@ -0,0 +1,58 @@ +"""Core 1.5 candidate seam regressions; exercised before the dependency floor moves.""" + +import importlib.metadata + +import pytest + +import datafog + +pytestmark = pytest.mark.skipif( + importlib.metadata.version("datafog-core").split(".")[:2] != ["1", "5"], + reason="qualifies the separately built Core 1.5 candidate before publication", +) + + +@pytest.mark.parametrize("value", ["::ffff:192.168.1.1", "64:ff9b::192.0.2.33"]) +def test_ipv6_embedded_ipv4_tail_is_fully_redacted(value): + result = datafog.redact(value) + assert result.redacted_text == "[IP_ADDRESS_1]" + assert result.mapping == {"[IP_ADDRESS_1]": value} + + +def test_contextual_email_does_not_swallow_neighboring_phone(): + text = "phone=415-555-2671&email=bob@example.com" + result = datafog.scan(text) + assert [(item.type, item.text) for item in result.entities] == [ + ("PHONE", "415-555-2671"), + ("EMAIL", "bob@example.com"), + ] + assert datafog.redact(text).redacted_text == "phone=[PHONE_1]&email=[EMAIL_1]" + + +def test_phone_followed_by_zip_does_not_become_credit_card(): + text = "555-123-4567 94105" + assert [(item.type, item.text) for item in datafog.scan(text).entities] == [ + ("PHONE", "555-123-4567"), + ("ZIP_CODE", "94105"), + ] + assert datafog.scan(text, entity_types=["PHONE"]).entities[0].text == "555-123-4567" + + +def test_adding_model_selection_does_not_drop_large_document_ssns(): + text = "ssn 123-45-6789 " * 3000 + result = datafog.scan(text, entity_types=["SSN", "PERSON"]) + assert len([item for item in result.entities if item.type == "SSN"]) == 3000 + assert all(text[item.start : item.end] == item.text for item in result.entities) + + +def test_international_email_and_person_punctuation(): + assert datafog.scan("mail jörg@müller.de").entities[0].text == "jörg@müller.de" + result = datafog.scan("Jane Doe,\n14 Penny Lane", entity_types=["PERSON"]) + assert result.entities and all( + item.text == item.text.rstrip(",;:.!?") for item in result.entities + ) + + +def test_german_context_prevents_generic_false_positive_labels(): + assert not datafog.scan("Steuer-ID 12345678901 liegt vor.").entities + assert not datafog.scan("IBAN DE44 5001 0517 5407 3249 31 ist gueltig.").entities diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py index 920c35f3..d36cc6af 100644 --- a/tests/test_de_pii_regex.py +++ b/tests/test_de_pii_regex.py @@ -166,7 +166,10 @@ def test_german_vat_redaction_suppresses_inner_generic_ssn_match( def test_top_level_helpers_and_supported_entities_respect_locale() -> None: default_entities = get_supported_entities() - assert all(not entity.startswith("DE_") for entity in default_entities) + assert {"EMAIL", "DE_VAT_ID", "DE_IBAN", "JWT", "PERSON"} <= set(default_entities) + assert "JWT" not in get_supported_entities(backend="python") + # Supported inventory includes opt-in labels; scanning still requires locale activation. + assert "DE_TAX_ID" in default_entities german_entities = get_supported_entities(locales=["de"]) assert "DE_VAT_ID" in german_entities diff --git a/tests/test_detection_accuracy.py b/tests/test_detection_accuracy.py index 6e7d8dfe..8cff1e82 100644 --- a/tests/test_detection_accuracy.py +++ b/tests/test_detection_accuracy.py @@ -453,6 +453,33 @@ def _assert_expected_found( return actual, expected +def _unexpected_negative_entities( + case: dict[str, Any], actual: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Keep legacy negative PII checks while recognizing two reviewed URL spans. + + PY-M14 restores GLiNER's URL output. Only exact, source-aligned URL candidates + from that model are permitted; other labels and malformed offsets still fail. + These added label checks are outside the historical PII metric inventory. + """ + permitted = case.get("permitted_gliner_entities", []) + return [ + entity + for entity in actual + if not ( + entity.get("engine") == "gliner" + and any( + all( + entity.get(key) == span[key] + for key in ("type", "text", "start", "end") + ) + and case["input"][span["start"] : span["end"]] == span["text"] + for span in permitted + ) + ) + ] + + def _compute_metrics( engines: list[str], corpora: list[tuple[str, list[dict[str, Any]]]] ) -> dict[str, Any]: @@ -471,6 +498,8 @@ def _compute_metrics( locales=case.get("locales"), backend=_corpus_backend(engine), ) + if corpus_kind == "negative": + actual = _unexpected_negative_entities(case, actual) expected = _required_expected( case["expected_entities"], engine, corpus_kind ) @@ -569,6 +598,7 @@ def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None: for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) } actual = [e for e in actual if (e["type"], e["text"]) not in added] + actual = _unexpected_negative_entities(case, actual) assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -591,6 +621,7 @@ def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None: for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) } actual = [e for e in actual if (e["type"], e["text"]) not in added] + actual = _unexpected_negative_entities(case, actual) assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -666,3 +697,36 @@ def test_accuracy_metrics_snapshot() -> None: output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(json.dumps(metrics, indent=2), encoding="utf-8") assert "overall" in metrics and metrics["overall"] + + +def test_reviewed_url_candidates_require_exact_label_origin_and_offsets(): + case = next( + case + for case in load_corpus("negative_cases.json") + if case["id"] == "markdown-link" + ) + permitted = dict(case["permitted_gliner_entities"][0], engine="gliner") + assert _unexpected_negative_entities(case, [permitted]) == [] + for altered in [ + dict(permitted, type="EMAIL"), + dict(permitted, type="PHONE"), + dict(permitted, type="IP_ADDRESS"), + dict(permitted, start=14), + dict(permitted, end=41), + dict(permitted, engine="spacy"), + dict(permitted, text="example.com"), + ]: + assert _unexpected_negative_entities(case, [altered]) == [altered] + + +def test_reviewed_url_permission_is_confined_to_the_two_named_cases(): + permitted_cases = [ + case + for case in load_corpus("negative_cases.json") + if case.get("permitted_gliner_entities") + ] + assert {case["id"] for case in permitted_cases} == {"markdown-link", "url-with-at"} + for case in permitted_cases: + assert case["permitted_gliner_reason"].startswith("PY-M14") + assert len(case["permitted_gliner_entities"]) == 1 + assert case["permitted_gliner_entities"][0]["type"] == "URL" diff --git a/tests/test_donut_lazy_import.py b/tests/test_donut_lazy_import.py index 9b2a28f1..aab7bcfc 100644 --- a/tests/test_donut_lazy_import.py +++ b/tests/test_donut_lazy_import.py @@ -20,8 +20,7 @@ def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]: def test_no_torch_import_when_donut_disabled(): """Test that torch is not imported when use_donut is False""" - _run_isolated_python( - """ + _run_isolated_python(""" import sys from datafog.services.image_service import ImageService @@ -29,8 +28,7 @@ def test_no_torch_import_when_donut_disabled(): assert "torch" not in sys.modules assert "transformers" not in sys.modules -""" - ) +""") def test_lazy_import_mechanism(): @@ -39,8 +37,7 @@ def test_lazy_import_mechanism(): # to use lazy imports. We don't need to actually test the imports themselves, # just that the structure is correct. - _run_isolated_python( - """ + _run_isolated_python(""" import sys from datafog.processing.image_processing.donut_processor import DonutProcessor @@ -50,5 +47,4 @@ def test_lazy_import_mechanism(): assert "transformers" not in sys.modules assert hasattr(processor, "extract_text_from_image") assert not hasattr(processor, "ensure_installed") -""" - ) +""") diff --git a/tests/test_engine_api.py b/tests/test_engine_api.py index 2a43e147..35aebe2a 100644 --- a/tests/test_engine_api.py +++ b/tests/test_engine_api.py @@ -198,7 +198,7 @@ def test_redact_invalid_strategy_raises_value_error() -> None: redact("test", entities=[], strategy="invalid") -def test_redact_ignores_invalid_spans() -> None: +def test_redact_rejects_invalid_spans() -> None: text = "hello" entities = [ Entity( @@ -219,9 +219,8 @@ def test_redact_ignores_invalid_spans() -> None: ), ] - result = redact(text=text, entities=entities, strategy="token") - assert result.redacted_text == text - assert result.mapping == {} + with pytest.raises(ValueError, match="valid ranges"): + redact(text=text, entities=entities, strategy="token") def test_scan_and_redact_combines_operations() -> None: diff --git a/tests/test_gliner_annotator.py b/tests/test_gliner_annotator.py index fbf0bbdf..b9a82ad5 100644 --- a/tests/test_gliner_annotator.py +++ b/tests/test_gliner_annotator.py @@ -11,8 +11,8 @@ import pytest -@pytest.fixture(scope="session", autouse=True) -def mock_gliner_module(): +@pytest.fixture(autouse=True) +def mock_gliner_module(monkeypatch): """Mock the gliner module for all tests.""" # Create a mock gliner module mock_gliner = MagicMock() @@ -20,22 +20,42 @@ def mock_gliner_module(): # Configure the mock model behavior mock_model = MagicMock() - mock_model.predict_entities.return_value = [ - {"label": "person", "text": "John Doe", "start": 0, "end": 8}, - {"label": "email", "text": "john@example.com", "start": 20, "end": 36}, - {"label": "phone number", "text": "555-123-4567", "start": 50, "end": 62}, - ] + + def predict(text, labels): + findings = [] + for label, value in [ + ("person", "John Doe"), + ("email", "john@example.com"), + ("phone number", "555-123-4567"), + ]: + start = text.find(value) + if start >= 0 and label in labels: + findings.append( + { + "label": label, + "text": value, + "start": start, + "end": start + len(value), + } + ) + return findings + + mock_model.predict_entities.side_effect = predict mock_gliner_class.from_pretrained.return_value = mock_model mock_gliner.GLiNER = mock_gliner_class - # Add to sys.modules to make import work - sys.modules["gliner"] = mock_gliner + from datafog import engine + gliner_factory, spacy_factory = ( + engine._get_gliner_annotator, + engine._get_spacy_annotator, + ) + gliner_factory.cache_clear() + spacy_factory.cache_clear() + monkeypatch.setitem(sys.modules, "gliner", mock_gliner) yield mock_gliner_class, mock_model - - # Cleanup - if "gliner" in sys.modules: - del sys.modules["gliner"] + gliner_factory.cache_clear() + spacy_factory.cache_clear() class TestGLiNERAnnotatorWithDependencies: @@ -119,15 +139,9 @@ def test_gliner_annotate_long_text(self, mock_gliner_module): # Create text longer than MAXIMAL_STRING_SIZE long_text = "A" * 1000001 # Exceeds 1M character limit - with patch( - "datafog.processing.text_processing.gliner_annotator.logging" - ) as mock_logging: - result = annotator.annotate(long_text) - - # Should log a warning about truncation - mock_logging.warning.assert_called_once() - # Should still return a valid result - assert isinstance(result, dict) + with pytest.raises(ValueError, match="exceeds 1000000"): + annotator.annotate(long_text) + mock_model.predict_entities.assert_not_called() def test_gliner_download_model(self, mock_gliner_module): """Test GLiNER model download functionality.""" @@ -326,21 +340,19 @@ def test_text_service_gliner_engine_without_dependencies(self): ): TextService(engine="gliner") - def test_text_service_smart_engine_without_dependencies(self): - """Test smart engine degrades gracefully when GLiNER dependencies are missing.""" + def test_text_service_smart_engine_without_dependencies(self, monkeypatch): + """Missing optional engines preserve the structured native layer.""" + from datafog import engine from datafog.services.text_service import TextService - with patch.object(TextService, "_create_gliner_annotator", return_value=None): - with patch.object( - TextService, "_create_spacy_annotator", return_value=None - ): - service = TextService(engine="smart") - with pytest.warns(UserWarning, match="GLiNER not available"): - result = service.annotate_text_sync( - "John Doe from Acme Corporation needs follow up." - ) - assert "EMAIL" in result - assert result["EMAIL"] == [] + unavailable = engine._UnavailableAnnotator("optional model unavailable") + monkeypatch.setattr(engine, "_get_gliner_annotator", lambda: unavailable) + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: unavailable) + with pytest.warns(UserWarning, match="GLiNER unavailable"): + result = TextService(engine="smart").annotate_text_sync( + "mail john@example.com" + ) + assert result == {"EMAIL": ["john@example.com"]} def test_text_service_valid_engines(self): """Test that all valid engines are accepted.""" @@ -382,10 +394,10 @@ def test_text_service_valid_engines(self): assert service.engine == engine def test_text_service_invalid_engine(self): - """Test that invalid engines raise AssertionError.""" + """Test that invalid engines raise a configuration error.""" from datafog.services.text_service import TextService - with pytest.raises(AssertionError, match="Invalid engine"): + with pytest.raises(ValueError, match="engine must be one of"): TextService(engine="invalid_engine") @pytest.mark.parametrize( @@ -414,30 +426,14 @@ def test_cascade_should_stop_logic(self, engine, expected_count): result_above_threshold = {"TYPE1": ["entity1"] * (expected_count + 1)} assert service._cascade_should_stop(engine, result_above_threshold) - def test_smart_cascade_flow(self, mock_gliner_annotator): - """Test the smart cascading flow.""" + def test_smart_engine_combines_gliner_with_structured_detection(self): + """GLiNER adds exact NER spans without removing structured findings.""" from datafog.services.text_service import TextService - # Inject annotators directly to keep this cascade test deterministic - # across Python versions and import ordering. - mock_regex = Mock() - mock_regex.annotate.return_value = {"EMAIL": []} - - mock_spacy = Mock() - mock_spacy.annotate.return_value = {"PERSON": ["John Doe"]} - - service = TextService(engine="smart") - service._regex_annotator = mock_regex - service._gliner_annotator = mock_gliner_annotator - service._gliner_import_attempted = True - service._spacy_annotator = mock_spacy - service._spacy_import_attempted = True - - service.annotate_text_sync("John Doe works at john@example.com") - - # Should have tried regex first, then GLiNER. - mock_regex.annotate.assert_called_once() - mock_gliner_annotator.annotate.assert_called_once() + text = "John Doe owns 6011111111111117 and john@example.com" + result = TextService(engine="smart").annotate_text_sync(text, structured=True) + assert {item.label for item in result} == {"PERSON", "CREDIT_CARD", "EMAIL"} + assert all(text[item.start : item.end] == item.text for item in result) # Test CLI updates as well diff --git a/tests/test_image_service.py b/tests/test_image_service.py index da8e6deb..03d472b1 100644 --- a/tests/test_image_service.py +++ b/tests/test_image_service.py @@ -3,10 +3,11 @@ import asyncio import pytest -from PIL import Image from datafog.services.image_service import ImageService +Image = pytest.importorskip("PIL.Image", reason="Image tests require datafog[ocr]") + urls = [ "https://www.pdffiller.com/preview/101/35/101035394.png", "https://www.pdffiller.com/preview/435/972/435972694.png", diff --git a/tests/test_legacy_retirement.py b/tests/test_legacy_retirement.py index f8be98f0..ed5910a0 100644 --- a/tests/test_legacy_retirement.py +++ b/tests/test_legacy_retirement.py @@ -31,7 +31,7 @@ def unexpected_import(*args): LegacySurfaceWarning, match=( f"{surface} support is deprecated in 4.9 and will be removed in 5.0. " - 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0".' + 'Last supported version is 4.12.0. Pin it with: pip install "datafog==4.12.0".' ), ): (ImageService if surface == "OCR" else SparkService)() @@ -43,7 +43,7 @@ def unavailable(name): monkeypatch.setattr(importlib, "import_module", unavailable) with ( - pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"), + pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.12.0"), pytest.raises(ImportError, match=r"datafog\[distributed\]"), ): SparkService() @@ -85,7 +85,7 @@ def test_donut_mock_output_is_preserved(monkeypatch): monkeypatch.setattr(donut_processor, "DONUT_TESTING_ENABLED", False) with pytest.warns(LegacySurfaceWarning): processor = DonutProcessor() - with pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"): + with pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.12.0"): assert asyncio.run(processor.extract_text_from_image(object())) == ( '{"text": "Mock OCR text for testing"}' ) @@ -245,7 +245,7 @@ def test_image_cli_notice_visible_under_default_python_filters(image_argument): args.append("image.png") result = CliRunner().invoke(client.app, args) assert "deprecated in 4.9 and will be removed in 5.0" in result.stderr, result.output -assert 'Last supported version is 4.10.0' in result.stderr, result.output +assert 'Last supported version is 4.12.0' in result.stderr, result.output if sys.argv[1] == "True": assert result.exit_code == 0, result.output assert "OCR Pipeline Results: ['OCR text']" in result.stdout diff --git a/tests/test_litellm_guardrail.py b/tests/test_litellm_guardrail.py index 1d7656a4..c4c1d697 100644 --- a/tests/test_litellm_guardrail.py +++ b/tests/test_litellm_guardrail.py @@ -152,13 +152,14 @@ async def test_post_call_fail_open_returns_unredacted_response(self, monkeypatch @pytest.mark.asyncio class TestEdgeShapes: - async def test_data_without_messages_passes_through(self): + async def test_embedding_input_is_redacted(self): guardrail = DataFogGuardrail(guardrail_name="datafog-pii") data = {"input": f"embed {EMAIL}"} result = await guardrail.async_pre_call_hook( user_api_key_dict=None, cache=None, data=data, call_type="aembedding" ) - assert result == data + assert EMAIL not in result["input"] + assert result["input"].startswith("embed [EMAIL_") async def test_message_without_content_key_passes_through(self): guardrail = DataFogGuardrail(guardrail_name="datafog-pii") diff --git a/tests/test_no_network_core.py b/tests/test_no_network_core.py index f06e8360..cbdf8c08 100644 --- a/tests/test_no_network_core.py +++ b/tests/test_no_network_core.py @@ -20,8 +20,7 @@ def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]: def test_import_scan_and_redact_do_not_open_network() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import socket import urllib.request @@ -39,8 +38,7 @@ def blocked(*_args, **_kwargs): redact_result = datafog.redact("Email jane@example.com or call 415-555-1212") assert "jane@example.com" not in redact_result.redacted_text assert "415-555-1212" not in redact_result.redacted_text -""" - ) +""") def test_core_defaults_do_not_initialize_optional_engines(monkeypatch) -> None: @@ -78,8 +76,7 @@ def fail_optional_engine_probe(): def test_import_probes_do_not_load_optional_models() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import sys import types @@ -105,13 +102,11 @@ def from_pretrained(*_args, **_kwargs): import datafog assert datafog.scan("Email jane@example.com").entities -""" - ) +""") def test_core_path_does_not_import_optional_dependency_modules() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import importlib.abc import sys @@ -144,5 +139,4 @@ def find_spec(self, fullname, path=None, target=None): assert datafog.scan_prompt("Email jane@example.com").entities assert datafog.filter_output("Email jane@example.com").redacted_text == "Email [EMAIL_1]" assert datafog.create_guardrail().filter("Email jane@example.com").redacted_text == "Email [EMAIL_1]" -""" - ) +""") diff --git a/tests/test_ocr_integration.py b/tests/test_ocr_integration.py index 9ceef4a2..c7333dbd 100644 --- a/tests/test_ocr_integration.py +++ b/tests/test_ocr_integration.py @@ -9,11 +9,12 @@ from unittest.mock import patch import pytest -from PIL import Image from datafog.processing.image_processing.donut_processor import DonutProcessor from datafog.services.image_service import ImageService +Image = pytest.importorskip("PIL.Image", reason="OCR tests require datafog[ocr]") + # Mark all tests in this file as integration tests pytestmark = pytest.mark.integration diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py new file mode 100644 index 00000000..5e44c461 --- /dev/null +++ b/tests/test_review_engine.py @@ -0,0 +1,360 @@ +"""Regression contracts for the 4.11 review's engine and annotation findings.""" + +import hashlib +import hmac +import random +from types import SimpleNamespace + +import pytest + +from datafog import engine +from datafog.processing.text_processing.gliner_annotator import GLiNERAnnotator +from datafog.processing.text_processing.regex_annotator.regex_annotator import ( + RegexAnnotator, +) +from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator +from datafog.services.text_service import TextService + + +def test_native_surrogate_normalization_preserves_source_and_offsets(): + text = "\ud800😀 ssn 123-45-6789 \udfff" + result = engine.scan(text, engine="regex", backend="rust") + assert result.text == text + assert any(entity.type == "SSN" for entity in result.entities) + assert all(text[item.start : item.end] == item.text for item in result.entities) + assert "123-45-6789" not in engine.redact(text, result.entities).redacted_text + + +def test_interval_selection_preserves_longest_and_tiebreak_contract(): + rng = random.Random(113) + for _ in range(30): + entities = [ + engine.Entity( + rng.choice(["PHONE", "SSN"]), + "", + start, + start + rng.randint(1, 30), + rng.random(), + "regex", + ) + for start in [rng.randint(0, 200) for _ in range(100)] + ] + expected = [] + for entity in sorted( + entities, + key=lambda item: ( + -engine._entity_length(item), + -engine.ENTITY_TYPE_PRIORITY.get(item.type, 0), + -item.confidence, + item.start, + item.end, + item.type, + ), + ): + if not any(engine._entities_overlap(entity, kept) for kept in expected): + expected.append(entity) + assert engine._suppress_overlapping_entities(entities) == sorted( + expected, key=lambda item: (item.start, item.end, item.type) + ) + + +@pytest.mark.parametrize("structured", [False, True]) +@pytest.mark.parametrize("value", ["john.doe@example.com", "123-45-6789"]) +def test_text_service_retains_findings_at_chunk_boundaries(value, structured): + text = "x" * 996 + " " + value + " tail" + result = TextService(text_chunk_length=1000).annotate_text_sync(text, structured) + if structured: + assert any( + item.text == value and text[item.start : item.end] == value + for item in result + ) + else: + assert any(value in items for items in result.values()) + + +def test_spacy_uses_original_offsets_and_does_not_turn_relative_dates_into_pii( + monkeypatch, +): + text = "Annual report prepared by Ann yesterday." + spans = [ + SimpleNamespace(label_="PERSON", text="Ann", start_char=26, end_char=29), + SimpleNamespace(label_="DATE", text="yesterday", start_char=30, end_char=39), + ] + annotator = SpacyPIIAnnotator(nlp=lambda _: SimpleNamespace(ents=spans)) + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: annotator) + result = engine.scan_and_redact(text, engine="spacy") + assert result.redacted_text == "Annual report prepared by [PERSON_1] yesterday." + + +def test_legacy_string_annotator_does_not_match_names_inside_words(): + result = engine._entities_from_dict( + "Annual report by Ann.", {"PERSON": ["Ann"]}, "spacy", 0.7 + ) + assert [(item.start, item.end) for item in result] == [(17, 20)] + + +@pytest.mark.parametrize("mode", ["smart", "spacy", "gliner"]) +def test_every_engine_keeps_native_structured_detection(monkeypatch, mode): + monkeypatch.setattr(engine, "_gliner_entities", lambda _: []) + monkeypatch.setattr(engine, "_spacy_entities", lambda _: []) + text = "card 6011111111111117 ip 2001:db8::1" + result = engine.scan_and_redact(text, engine=mode, backend="rust") + assert {item.type for item in result.entities} == {"CREDIT_CARD", "IP_ADDRESS"} + assert "6011111111111117" not in result.redacted_text + assert "2001:db8::1" not in result.redacted_text + + +@pytest.mark.parametrize("labels", ["EMAIL", ["BOGUS"], [1], [""]]) +def test_entity_type_mistakes_fail_explicitly(labels): + with pytest.raises(ValueError, match="entity_types"): + engine.scan("john@example.com", entity_types=labels) + + +@pytest.mark.parametrize("label", ["JWT", "PERSON", "ORGANIZATION", "UUID"]) +def test_unavailable_fallback_labels_raise(label): + with pytest.raises(ValueError, match="Unsupported entity_types"): + engine.scan( + "john@example.com", engine="regex", backend="python", entity_types=[label] + ) + + +@pytest.mark.parametrize("pattern", [r"(a|a)+$", r"((a+))+$"]) +def test_pathological_allowlist_cannot_suppress_a_finding(pattern): + value = "a" * 300 + "@b.com" + entity = engine.Entity("EMAIL", value, 0, len(value), 1.0, "regex") + assert engine._apply_allowlist([entity], None, [pattern]) == [entity] + + +@pytest.mark.parametrize( + "pattern,value", + [(r"(\d{3}){2}", "123123"), (r"(?:\d{3}-){2}\d{4}", "415-555-2671")], +) +def test_safe_quantified_allowlist_patterns_are_supported(pattern, value): + entity = engine.Entity("PHONE", value, 0, len(value), 1.0, "regex") + assert engine._apply_allowlist([entity], None, [pattern]) == [] + + +@pytest.mark.parametrize( + "annotator", + [ + SpacyPIIAnnotator( + nlp=lambda _: (_ for _ in ()).throw(RuntimeError("raw secret")) + ), + GLiNERAnnotator( + model=SimpleNamespace( + predict_entities=lambda *_: (_ for _ in ()).throw( + RuntimeError("raw secret") + ) + ), + entity_types=["person"], + model_name="test", + ), + ], +) +def test_ner_failures_and_oversize_inputs_are_explicit(annotator): + with pytest.raises(RuntimeError, match="annotation failed"): + annotator.annotate("secret") + with pytest.raises(ValueError, match="exceeds"): + annotator.annotate("x" * 1_000_001) + + +@pytest.mark.parametrize( + "label", ["passport number", "medical record number", "license number", "url"] +) +def test_gliner_extra_labels_and_original_offsets_are_preserved(monkeypatch, label): + text = "prefix SECRET suffix" + annotator = GLiNERAnnotator( + model=SimpleNamespace( + predict_entities=lambda *_: [ + {"label": label, "text": "SECRET", "start": 7, "end": 13} + ] + ), + entity_types=[label], + model_name="test", + ) + monkeypatch.setattr(engine, "_get_gliner_annotator", lambda: annotator) + result = engine.scan( + text, engine="gliner", entity_types=[label.upper().replace(" ", "_")] + ) + assert len(result.entities) == 1 + assert result.entities[0].text == "SECRET" + assert result.entities[0].start == 7 + + +def test_model_chunks_are_bounded_and_keep_document_offsets(): + text = "😀" * 9000 + " Jane Doe" + calls = [] + + def scan_model(chunk): + calls.append(chunk) + start = chunk.find("Jane Doe") + findings = ( + [] + if start < 0 + else [ + SimpleNamespace( + entity_type="PERSON", + codepoint_range=SimpleNamespace(start=start, end=start + 8), + matched_text="Jane Doe", + ) + ] + ) + return SimpleNamespace(findings=findings) + + findings = engine._cnn_findings( + SimpleNamespace(scan_model=scan_model), {"PERSON"}, text + ) + assert len(calls) >= 3 and max(len(chunk.encode()) for chunk in calls) <= 16384 + assert any( + offset + item.codepoint_range.start == text.index("Jane Doe") + for offset, item in findings + ) + + +def test_token_collision_and_repeated_value_mapping(): + text = "[EMAIL_1] a@b.com and a@b.com" + result = engine.scan_and_redact(text, engine="regex") + assert result.redacted_text == "[EMAIL_1] [EMAIL_2] and [EMAIL_2]" + assert result.mapping == {"[EMAIL_2]": "a@b.com"} + + +@pytest.mark.parametrize( + "entity", + [ + engine.Entity("EMAIL", "", 0, 1, 1.0, "regex"), + engine.Entity("EMAIL", "a", 0, 100, 1.0, "regex"), + engine.Entity("EMAIL", "wrong", 0, 1, 1.0, "regex"), + ], +) +def test_invalid_explicit_spans_are_rejected(entity): + with pytest.raises(ValueError, match="valid ranges"): + engine.redact("a", [entity]) + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256", "sha3_256"]) +def test_hash_algorithm_and_hmac_are_honored(algorithm): + text = "a@b.com" + entities = engine.scan(text, engine="regex").entities + plain = engine.redact(text, entities, strategy="hash", hash_type=algorithm) + assert hashlib.new(algorithm, text.encode()).hexdigest()[:12] in plain.redacted_text + key = bytes(range(32)) + keyed = engine.redact( + text, entities, strategy="hash", hash_type=algorithm, hash_key=key + ) + assert hmac.new(key, text.encode(), algorithm).hexdigest() in keyed.redacted_text + assert plain != keyed + + +def test_invalid_hash_parameters_fail(): + with pytest.raises(ValueError, match="hash_key"): + engine.redact("text", [], strategy="hash", hash_key=b"short") + with pytest.raises(ValueError, match="hash_type"): + engine.redact("text", [], strategy="hash", hash_type="bogus") + + +@pytest.mark.parametrize("locales", [[""], [1]]) +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_invalid_locales_are_consistently_rejected(locales, backend): + with pytest.raises(ValueError, match="locale"): + engine.scan("safe", engine="regex", locales=locales, backend=backend) + + +@pytest.mark.parametrize("locale", ["fr", "en-US"]) +def test_fallback_accepts_locale_tags_without_extra_detectors(locale): + assert engine.scan( + "a@b.com", engine="regex", locales=[locale], backend="python" + ).entities + + +def test_unicode_email_in_fallback(): + result = engine.scan("mail jörg@müller.de", engine="regex", backend="python") + assert result.entities[0].text == "jörg@müller.de" + + +@pytest.mark.parametrize( + "options", + [ + {"engine": "bogus"}, + {"text_chunk_length": 0}, + {"text_chunk_length": -1}, + {"text_chunk_length": True}, + ], +) +def test_text_service_argument_validation(options): + with pytest.raises(ValueError): + TextService(**options) + + +def test_fallback_adversarial_patterns_complete(): + annotator = RegexAnnotator(locales=["de"]) + for value in [ + "!" * 80000, + "#a" * 40000, + "Steuer-ID" + " " * 80000 + "x", + "Passport" + " " * 80000, + ]: + assert not any(annotator.annotate(value).values()) + + +def test_hash_marker_cannot_collide_with_literal_input(): + value = "a@b.com" + digest = hashlib.sha256(value.encode()).hexdigest()[:12] + marker = f"[EMAIL_{digest}]" + text = f"{marker} {value} {value}" + result = engine.scan_and_redact(text, engine="regex", strategy="hash") + assert result.mapping == {f"[EMAIL_{digest}_2]": value} + assert result.redacted_text == f"{marker} [EMAIL_{digest}_2] [EMAIL_{digest}_2]" + + +@pytest.mark.parametrize( + "text", + [ + "?" * 11 + "xiaoming@example.cn", + '{"note":"email=alice@example.com"}', + "email = 'dev@example.com'", + "phone=415-555-2671&email=bob@example.com", + ], +) +def test_fallback_email_context_is_trimmed_without_losing_mailboxes(text): + result = engine.scan(text, engine="regex", backend="python", entity_types=["EMAIL"]) + assert len(result.entities) == 1 + assert result.entities[0].text in { + "xiaoming@example.cn", + "alice@example.com", + "dev@example.com", + "bob@example.com", + } + + +def test_spacy_selection_rejects_gliner_only_labels(): + with pytest.raises(ValueError, match="PASSPORT_NUMBER"): + engine.scan("passport SECRET", engine="spacy", entity_types=["PASSPORT_NUMBER"]) + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256", "sha3_256"]) +def test_shared_hash_helper_retains_legacy_widths_and_full_hmac(algorithm): + value = "a@b.com" + digest = hashlib.new(algorithm, value.encode()).hexdigest() + assert engine._hash_value(value, algorithm, None) == digest[:12] + assert engine._hash_value(value, algorithm, None, plain_length=8) == digest[:8] + key = bytes(range(32)) + assert ( + engine._hash_value(value, algorithm, key, plain_length=8) + == hmac.new(key, value.encode(), algorithm).hexdigest() + ) + + +@pytest.mark.parametrize("mode", ["smart", "spacy", "gliner"]) +def test_optional_ner_cannot_remove_overlapping_structured_scan_findings( + monkeypatch, mode +): + text = "Ping 127.0.0.1" + candidates = [engine.Entity("PERSON", text, 0, len(text), 0.8, mode)] + monkeypatch.setattr(engine, "_spacy_entities", lambda _: candidates) + monkeypatch.setattr(engine, "_gliner_entities", lambda _: candidates) + result = engine.scan(text, engine=mode) + assert any( + item.type == "IP_ADDRESS" and item.text == "127.0.0.1" + for item in result.entities + ) + assert any(item.type == "PERSON" for item in result.entities) diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py new file mode 100644 index 00000000..519c9cf6 --- /dev/null +++ b/tests/test_review_entrypoints.py @@ -0,0 +1,270 @@ +"""Regression contracts for the November 2026 entrypoint review.""" + +import asyncio +import hashlib +import importlib.util +import logging +from pathlib import Path +from types import SimpleNamespace + +import pytest +from typer.testing import CliRunner + +import datafog +from datafog.agent import create_guardrail +from datafog.client import app +from datafog.config import OperationType, get_config +from datafog.main import DataFog, TextPIIAnnotator +from datafog.models.annotator import AnnotationResult, AnnotatorRequest +from datafog.models.anonymizer import Anonymizer, AnonymizerType, HashType +from datafog.models.common import EntityTypes + +EMAIL = "jane@" "example.com" + + +@pytest.mark.asyncio +async def test_decorator_awaits_and_filters(): + @create_guardrail() + async def answer(): + await asyncio.sleep(0) + return EMAIL + + assert EMAIL not in await answer() + + +@pytest.mark.parametrize("output", [[EMAIL], {"text": EMAIL}, None, 1]) +def test_guarded_non_strings_rejected(output): + @create_guardrail() + def answer(): + return output + + with pytest.raises(TypeError, match="must return a string"): + answer() + + +@pytest.mark.parametrize( + "operation", [OperationType.REDACT, OperationType.REPLACE, OperationType.HASH] +) +def test_anonymizing_operations_imply_scan(operation): + service = DataFog(operations=[operation]) + assert EMAIL not in service.run_text_pipeline_sync([EMAIL])[0] + assert EMAIL not in service.process_text(EMAIL) + + +@pytest.mark.parametrize("algorithm", list(HashType)) +def test_hash_algorithm_honored_across_api_and_cli(algorithm): + digest = hashlib.new(algorithm.value, EMAIL.encode()).hexdigest() + assert ( + f"[EMAIL_{digest[:12]}]" + == DataFog(hash_type=algorithm).process(EMAIL, anonymize=True, method="hash")[ + "anonymized" + ] + ) + assert ( + f"[EMAIL_{digest[:8]}]" + == datafog.process( + EMAIL, anonymize=True, method="hash", hash_type=algorithm.value + )["anonymized"] + ) + result = CliRunner().invoke( + app, ["hash-text", EMAIL, "--hash-type", algorithm.value] + ) + assert result.exit_code == 0, result.output + assert result.stdout.strip() == f"[EMAIL_{digest[:12]}]" + + +def test_existing_plain_hash_defaults_remain_compatible(): + assert ( + datafog.redact(EMAIL, strategy="hash").redacted_text + == f"[EMAIL_{hashlib.sha256(EMAIL.encode()).hexdigest()[:12]}]" + ) + assert ( + datafog.process(EMAIL, anonymize=True, method="hash")["anonymized"] + == f"[EMAIL_{hashlib.md5(EMAIL.encode()).hexdigest()[:8]}]" + ) + + +def test_keyed_hash_public_facades(): + import hmac + + key = b"k" * 32 + digest = hmac.new(key, EMAIL.encode(), "sha256").hexdigest() + assert digest in datafog.redact(EMAIL, strategy="hash", hash_key=key).redacted_text + assert digest in datafog.anonymize_text(EMAIL, method="hash", hash_key=key) + assert ( + digest + in datafog.process(EMAIL, anonymize=True, method="hash", hash_key=key)[ + "anonymized" + ] + ) + + +@pytest.mark.parametrize( + "factory", [lambda: datafog.process, lambda: DataFog().process] +) +def test_bad_method_rejected(factory): + with pytest.raises(ValueError, match="method"): + factory()(EMAIL, anonymize=True, method="bogus") + + +def test_legacy_detection_uses_core(): + text = "card 6011111111111117 ip 2001:db8::1" + assert {item["type"] for item in datafog.detect(text)} >= { + "CREDIT_CARD", + "IP_ADDRESS", + } + assert "6011111111111117" not in datafog.process(text, anonymize=True)["anonymized"] + assert "IP_ADDRESS" in TextPIIAnnotator().run(text) + assert "JWT" in datafog.get_supported_entities() + + +def test_annotation_defaults_and_labels(): + assert AnnotatorRequest(text="x", language="en").score_threshold is None + for label in ["SSN", "ORG", "GPE", "IP_ADDRESS"]: + assert ( + AnnotationResult(start=0, end=1, entity_type=label).entity_type != "UNKNOWN" + ) + + +def test_enum_filter_and_date_replacement(): + annotation = AnnotationResult( + start=0, end=len(EMAIL), entity_type=EntityTypes.EMAIL + ) + assert ( + Anonymizer(anonymizer_type="redact", entities=[EntityTypes.EMAIL]) + .anonymize(EMAIL, [annotation]) + .anonymized_text + == "[REDACTED]" + ) + date = AnnotationResult(start=0, end=10, entity_type="DATE") + assert ( + Anonymizer().anonymize("2024-01-02", [date]).anonymized_text + == "[REDACTED_DATE]" + ) + + +@pytest.mark.parametrize("strategy", list(AnonymizerType)) +def test_legacy_empty_entity_selection_keeps_protecting_all(strategy): + annotation = AnnotationResult(start=0, end=len(EMAIL), entity_type="EMAIL") + result = Anonymizer(anonymizer_type=strategy, entities=[]).anonymize( + EMAIL, [annotation] + ) + assert EMAIL not in result.anonymized_text + assert len(result.anonymized_entities) == 1 + + +@pytest.mark.parametrize("strategy", list(AnonymizerType)) +def test_legacy_overlaps_never_leave_partial_value(strategy): + text = "abcdefghij" + spans = [ + AnnotationResult(start=0, end=6, entity_type="EMAIL"), + AnnotationResult(start=4, end=10, entity_type="SSN"), + ] + result = Anonymizer(anonymizer_type=strategy).anonymize(text, spans) + assert "ghij" not in result.anonymized_text + assert len(result.anonymized_entities) == 1 + assert result.anonymized_entities[0]["original"] == text + + +def test_cli_hides_key_and_handles_bad_operation(): + config = get_config() + old_key = config.api_key + try: + config.api_key = "private-test-key" + result = CliRunner().invoke(app, ["show-config"]) + assert result.exit_code == 0 + assert "private-test-key" not in result.stdout + assert "REDACTED" in result.stdout + finally: + config.api_key = old_key + result = CliRunner().invoke(app, ["scan-text", EMAIL, "--operations", "bogus"]) + assert result.exit_code == 2 + assert "operations must use" in result.output + assert "Traceback" not in result.output + + +def test_import_and_cli_preserve_logger_level(): + logger = logging.getLogger("datafog_logger") + old = logger.level + try: + logger.setLevel(logging.ERROR) + import importlib + + import datafog.main + + importlib.reload(datafog.main) + assert logger.level == logging.ERROR + finally: + logger.setLevel(old) + + +def test_spark_real_spacy_labels_and_oversize(monkeypatch): + from datafog.processing.spark_processing import pyspark_udfs as spark + + monkeypatch.setattr(spark, "ensure_installed", lambda _: None) + + def nlp(text): + return SimpleNamespace( + ents=[ + SimpleNamespace(label_="PERSON", text="Ann"), + SimpleNamespace(label_="ORG", text="Acme"), + ] + ) + + with pytest.warns(FutureWarning): + result = spark.pii_annotator("Ann at Acme", SimpleNamespace(value=nlp)) + assert result[spark.PII_ANNOTATION_LABELS.index("PER")] == ["Ann"] + assert result[spark.PII_ANNOTATION_LABELS.index("ORG")] == ["Acme"] + with pytest.warns(FutureWarning), pytest.raises(ValueError, match="maximum"): + spark.pii_annotator( + "x" * (spark.MAXIMAL_STRING_SIZE + 1), SimpleNamespace(value=nlp) + ) + + +def _version_selector(): + path = Path(__file__).parents[1] / "scripts/release_version.py" + spec = importlib.util.spec_from_file_location("release_version", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.select_version + + +@pytest.mark.parametrize( + "current, kind, override", + [ + ("4.9.0a2", "alpha", None), + ("4.11.0", "beta", None), + ("4.12.0", "stable", "4.10.0"), + ], +) +def test_release_rejects_stale_bases(current, kind, override): + with pytest.raises(ValueError, match="newer than"): + _version_selector()(current, kind, [], "4.11.0", override) + + +def test_release_version_positive_and_duplicate_paths(): + select = _version_selector() + assert select("4.12.0", "alpha", ["v4.12.0a1"], "4.11.0") == "4.12.0a2" + assert select("4.12.0", "stable", [], "4.11.0") == "4.12.0" + with pytest.raises(ValueError, match="already exists"): + select("4.12.0", "stable", ["v4.12.0"], "4.11.0") + + +def test_release_skips_published_prereleases_without_tags(): + select = _version_selector() + assert select("4.12.0", "alpha", [], "4.11.0", published=["4.12.0a1"]) == "4.12.0a2" + + +def test_native_version_uses_import_error_when_metadata_is_missing(monkeypatch): + import importlib.metadata + + import datafog.native + + assert datafog.native.version() + + def missing(_): + raise importlib.metadata.PackageNotFoundError("datafog-core") + + monkeypatch.setattr(importlib.metadata, "version", missing) + with pytest.raises(ImportError, match="metadata is missing"): + datafog.native.version() diff --git a/tests/test_review_guardrails.py b/tests/test_review_guardrails.py new file mode 100644 index 00000000..ea6c4cbe --- /dev/null +++ b/tests/test_review_guardrails.py @@ -0,0 +1,434 @@ +"""Review regressions for request/response coverage and safe stream failures.""" + +# ruff: noqa: E402 +# Optional proxy dependencies must be present before importing the adapter. +import copy +import json + +import pytest + +pytest.importorskip("litellm") +pytest.importorskip("fastapi") +from fastapi import HTTPException # noqa: E402 +from litellm import ModelResponseStream # noqa: E402 +from litellm.types.utils import Delta, StreamingChoices # noqa: E402 + +import datafog # noqa: E402 +from datafog.integrations import claude_code as hook # noqa: E402 +from datafog.integrations import litellm_guardrail as gateway # noqa: E402 + +EMAIL_A = "alice@" "example.com" +EMAIL_B = "bob@" "example.com" +SSN = "123-45-" "6789" + + +def guard(**kwargs): + return gateway.DataFogGuardrail(guardrail_name="review", **kwargs) + + +async def pre(adapter, data): + return await adapter.async_pre_call_hook(None, None, data, "completion") + + +async def source(chunks): + for chunk in chunks: + yield chunk + + +async def streamed(adapter, chunks): + return [ + chunk + async for chunk in adapter.async_post_call_streaming_iterator_hook( + None, source(chunks), {} + ) + ] + + +@pytest.mark.asyncio +async def test_request_shares_tokens_without_existing_token_collisions(): + original = { + "messages": [ + {"role": "user", "content": f"[EMAIL_1] {EMAIL_A}"}, + {"role": "user", "content": f"other {EMAIL_B}; same {EMAIL_A}"}, + ], + "prompt": EMAIL_B, + "input": [{"content": EMAIL_A}], + } + before = copy.deepcopy(original) + result = await pre(guard(), original) + assert result["messages"][0]["content"] == "[EMAIL_1] [EMAIL_2]" + assert result["messages"][1]["content"] == "other [EMAIL_3]; same [EMAIL_2]" + assert ( + result["prompt"] == "[EMAIL_3]" and result["input"][0]["content"] == "[EMAIL_2]" + ) + assert original == before + another = await pre(guard(), {"prompt": EMAIL_A}) + assert another["prompt"] == "[EMAIL_1]" + + +@pytest.mark.asyncio +async def test_request_scans_tool_arguments_and_structured_input_values(): + data = { + "messages": [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "save", + "arguments": json.dumps({"email": EMAIL_A, "ssn": SSN}), + }, + } + ], + } + ], + "input": {"name": EMAIL_B, "id": SSN, "url": "send " + EMAIL_A}, + } + result = await pre(guard(), data) + assert ( + EMAIL_A not in result["messages"][0]["tool_calls"][0]["function"]["arguments"] + ) + assert SSN not in result["messages"][0]["tool_calls"][0]["function"]["arguments"] + assert result["input"] == { + "name": "[EMAIL_2]", + "id": "[SSN_1]", + "url": "send [EMAIL_1]", + } + assert result["messages"][0]["tool_calls"][0]["id"] == "call_1" + + +@pytest.mark.asyncio +async def test_multimodal_media_remains_intact_and_text_is_scanned(): + image = { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,AA4242424242424242", + "detail": "high", + }, + } + audio = { + "type": "input_audio", + "input_audio": {"data": "4242424242424242", "format": "wav"}, + } + result = await pre( + guard(), + { + "messages": [ + { + "role": "user", + "content": [image, audio, {"type": "text", "text": EMAIL_A}], + } + ] + }, + ) + assert result["messages"][0]["content"][:2] == [image, audio] + assert result["messages"][0]["content"][2]["text"] == "[EMAIL_1]" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["messages", "prompt", "input"]) +async def test_surrogate_cannot_disable_blocking(field): + value = f"ssn {SSN} \ud83d" + data = ( + {field: [{"role": "user", "content": value}]} + if field == "messages" + else {field: value} + ) + with pytest.raises(HTTPException) as error: + await pre(guard(action="block"), data) + assert error.value.status_code == 400 and SSN not in str(error.value.detail) + + +@pytest.mark.asyncio +async def test_response_reserves_tokens_in_tools_and_covers_legacy_and_text(): + response = { + "choices": [ + { + "index": 0, + "message": { + "content": EMAIL_A, + "tool_calls": [ + { + "function": { + "name": "save", + "arguments": f"[EMAIL_1] {EMAIL_B}", + } + } + ], + }, + }, + { + "index": 1, + "message": { + "content": None, + "function_call": {"name": "old", "arguments": EMAIL_A}, + }, + }, + {"index": 2, "text": EMAIL_B}, + ] + } + result = await guard().async_post_call_success_hook({}, None, response) + assert result is response + assert result["choices"][0]["message"]["content"] == "[EMAIL_2]" + assert ( + result["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] + == "[EMAIL_1] [EMAIL_3]" + ) + assert result["choices"][1]["message"]["function_call"]["arguments"] == "[EMAIL_2]" + assert result["choices"][2]["text"] == "[EMAIL_3]" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_response_scanning_is_atomic_and_respects_fail_policy( + monkeypatch, policy +): + actual = gateway._redact_text + + def failing(text, *args, **kwargs): + if text == "second": + raise RuntimeError(EMAIL_A) + return actual(text, *args, **kwargs) + + monkeypatch.setattr(gateway, "_redact_text", failing) + response = { + "choices": [ + {"message": {"content": EMAIL_A}}, + {"message": {"content": "second"}}, + ] + } + before = copy.deepcopy(response) + if policy == "closed": + with pytest.raises(RuntimeError) as error: + await guard(fail_policy=policy).async_post_call_success_hook( + {}, None, response + ) + assert EMAIL_A not in str(error.value) + else: + assert ( + await guard(fail_policy=policy).async_post_call_success_hook( + {}, None, response + ) + is response + ) + assert response == before + + +@pytest.mark.asyncio +async def test_stream_scans_all_choice_and_tool_argument_fragments(): + chunks = [ + { + "id": "first", + "choices": [ + { + "index": 0, + "delta": { + "content": "[EMAIL_1] alice@", + "tool_calls": [ + { + "index": 0, + "id": "call_a", + "function": { + "name": "save", + "arguments": '{"ssn":"123-', + }, + }, + { + "index": 1, + "id": "call_b", + "function": {"name": "save", "arguments": "bob@"}, + }, + ], + }, + }, + { + "index": 1, + "delta": { + "content": "bob@", + "function_call": {"name": "old", "arguments": "alice@"}, + }, + }, + {"index": 2, "text": "bob@"}, + ], + }, + { + "id": "last", + "choices": [ + { + "index": 0, + "delta": { + "content": "example.com", + "tool_calls": [ + {"index": 0, "function": {"arguments": '45-6789"}'}}, + {"index": 1, "function": {"arguments": "example.com"}}, + ], + }, + "finish_reason": "tool_calls", + }, + { + "index": 1, + "delta": { + "content": "example.com", + "function_call": {"arguments": "example.com"}, + }, + "finish_reason": "stop", + }, + {"index": 2, "text": "example.com", "finish_reason": "stop"}, + ], + }, + ] + original = copy.deepcopy(chunks) + output = await streamed(guard(), chunks) + first, last = output + assert first["choices"][0]["delta"]["content"] == "[EMAIL_1] [EMAIL_2]" + calls = first["choices"][0]["delta"]["tool_calls"] + assert calls[0]["function"]["arguments"] == '{"ssn":"[SSN_1]"}' + assert calls[1]["function"]["arguments"] == "[EMAIL_3]" + assert first["choices"][1]["delta"]["content"] == "[EMAIL_3]" + assert first["choices"][1]["delta"]["function_call"]["arguments"] == "[EMAIL_2]" + assert first["choices"][2]["text"] == "[EMAIL_3]" + assert last["choices"][0]["delta"]["content"] == "" + assert last["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] == "" + assert last["choices"][0]["finish_reason"] == "tool_calls" + assert chunks == original + assert first["id"] == "first" and last["id"] == "last" + + +@pytest.mark.asyncio +async def test_stream_handles_real_litellm_native_chunks(): + chunks = [ + ModelResponseStream( + id="same", choices=[StreamingChoices(index=0, delta=Delta(content=part))] + ) + for part in ("alice@", "example.com") + ] + output = await streamed(guard(), chunks) + assert output[0].choices[0].delta.content == "[EMAIL_1]" + assert output[1].choices[0].delta.content == "" + assert chunks[0].choices[0].delta.content == "alice@" + + +@pytest.mark.asyncio +async def test_stream_snapshots_reused_mutable_chunks(): + async def reused(): + chunk = {"choices": [{"index": 0, "delta": {"content": "alice@"}}]} + yield chunk + chunk["choices"][0]["delta"]["content"] = "example.com" + yield chunk + + output = [ + chunk + async for chunk in guard().async_post_call_streaming_iterator_hook( + None, reused(), {} + ) + ] + assert output[0]["choices"][0]["delta"]["content"] == "[EMAIL_1]" + assert output[1]["choices"][0]["delta"]["content"] == "" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_stream_source_errors_emit_nothing_and_hide_provider_text(policy): + async def broken(): + yield {"choices": [{"delta": {"content": EMAIL_A}}]} + raise RuntimeError("provider failed on " + EMAIL_A) + + emitted = [] + with pytest.raises(RuntimeError) as error: + async for chunk in guard( + fail_policy=policy + ).async_post_call_streaming_iterator_hook(None, broken(), {}): + emitted.append(chunk) + assert not emitted and EMAIL_A not in str(error.value) + assert error.value.__cause__ is None and error.value.__suppress_context__ + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit", ["MAX_STREAM_CHARS", "MAX_STREAM_CHUNKS"]) +async def test_stream_limits_reject_without_emission(monkeypatch, limit): + monkeypatch.setattr(gateway, limit, 1) + chunks = [{"choices": [{"delta": {"content": "ab"}}]}] * 2 + emitted = [] + with pytest.raises(RuntimeError, match="scan budget"): + async for chunk in guard().async_post_call_streaming_iterator_hook( + None, source(chunks), {} + ): + emitted.append(chunk) + assert not emitted + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_stream_engine_failures_follow_documented_policy(monkeypatch, policy): + def failing(*args, **kwargs): + raise RuntimeError(EMAIL_A) + + monkeypatch.setattr(gateway, "_redact_text", failing) + chunks = [{"choices": [{"delta": {"content": EMAIL_A}}]}] + if policy == "open": + assert await streamed(guard(fail_policy=policy), chunks) == chunks + else: + with pytest.raises(RuntimeError, match="fail_policy is 'closed'") as error: + await streamed(guard(fail_policy=policy), chunks) + assert EMAIL_A not in str(error.value) + + +@pytest.mark.parametrize("action", ["ask", "deny"]) +def test_hook_oversize_never_silently_allows_unscanned_tail(action): + payload = { + "hook_event_name": "PreToolUse", + "tool_name": "Write", + "tool_input": {"content": "x" * (hook.MAX_SCAN_CHARS + 1) + " " + SSN}, + } + code, output = hook.run(payload, {"DATAFOG_HOOK_ACTION": action}) + decision = json.loads(output)["hookSpecificOutput"] + assert code == 0 and decision["permissionDecision"] == action + assert ( + "not fully scanned" in decision["permissionDecisionReason"] + and SSN not in output + ) + + +def test_hook_surrogate_and_total_budget_cannot_disable_pii_gate(monkeypatch): + payload = { + "hook_event_name": "PreToolUse", + "tool_input": {"command": "echo " + SSN, "description": "\udc00"}, + } + code, output = hook.run(payload, {}) + assert code == 0 and "SSN x1" in output + monkeypatch.setattr(hook, "TOTAL_SCAN_CHARS", 20) + code, output = hook.run( + {"hook_event_name": "PreToolUse", "tool_input": ["a" * 15, "b" * 15]}, {} + ) + assert ( + code == 0 + and json.loads(output)["hookSpecificOutput"]["permissionDecision"] == "ask" + ) + + +@pytest.mark.asyncio +async def test_decorator_awaits_and_rejects_non_string_returns(): + adapter = datafog.create_guardrail() + + @adapter + async def generated(): + return EMAIL_A + + assert await generated() == "[EMAIL_1]" + + @adapter + async def structured(): + return [EMAIL_A] + + with pytest.raises(TypeError, match="must return a string"): + await structured() + + @adapter + def sync_structured(): + return {"text": EMAIL_A} + + with pytest.raises(TypeError, match="must return a string"): + sync_structured() diff --git a/tests/test_runtime_dependency_safety.py b/tests/test_runtime_dependency_safety.py index d34ceb8a..b311fff4 100644 --- a/tests/test_runtime_dependency_safety.py +++ b/tests/test_runtime_dependency_safety.py @@ -42,8 +42,7 @@ def test_runtime_code_does_not_install_packages() -> None: def test_ocr_and_spark_public_services_do_not_require_optional_imports() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import importlib.abc import sys @@ -74,8 +73,7 @@ def find_spec(self, fullname, path=None, target=None): assert TextService is not None assert datafog.ImageService is ImageService assert datafog.SparkService is SparkService -""" - ) +""") def test_spacy_pii_missing_model_requires_explicit_download( diff --git a/tests/test_rust_backend.py b/tests/test_rust_backend.py index 34da6a21..3cbdd607 100644 --- a/tests/test_rust_backend.py +++ b/tests/test_rust_backend.py @@ -66,15 +66,11 @@ def test_selection_alias_and_empty_selection(native, selection): assert len(result.entities) == 1 -def test_unknown_selection_preserves_legacy_empty_result(native): +def test_unknown_selection_is_rejected(native): findings, _ = native findings.append(finding("EMAIL", "a@example.com", 0, 13)) - assert ( - engine.scan( - "a@example.com", "regex", entity_types=["UNKNOWN"], backend="rust" - ).entities - == [] - ) + with pytest.raises(ValueError, match="Unsupported entity_types"): + engine.scan("a@example.com", "regex", entity_types=["UNKNOWN"], backend="rust") def test_unknown_native_label_preserved_without_silently_dropping_pii(native): @@ -84,7 +80,15 @@ def test_unknown_native_label_preserved_without_silently_dropping_pii(native): assert [item.type for item in result.entities] == ["FUTURE_LABEL"] -def test_python_overlap_priority_precedes_selection(native): +def test_python_overlap_priority_precedes_selection(native, monkeypatch): + metadata = capability_fixture() + metadata["supported_entities"].extend(["PHONE", "CREDIT_CARD"]) + for label in ("PHONE", "CREDIT_CARD"): + metadata["entities"][label] = { + "scopes": ["text"], + "activation": {"kind": "default"}, + } + monkeypatch.setattr(sys.modules["datafog_core"], "capabilities", lambda: metadata) findings, _ = native text = "1234567890123456" findings.extend( @@ -116,7 +120,7 @@ def test_python_allowlist_semantics(native, kwargs, count): ) -@pytest.mark.parametrize("pattern", ["[", "(a+)+", "x" * 513]) +@pytest.mark.parametrize("pattern", ["[", "x" * 513]) def test_allowlist_validation_before_native(native, pattern): _, calls = native with pytest.raises(ValueError): @@ -146,11 +150,12 @@ def test_unknown_locale_validation_preserved(native): @pytest.mark.parametrize("name", ["smart", "spacy", "gliner"]) -def test_nonregex_engine_rejected(native, name): +def test_nonregex_engines_use_native_structured_stage(native, name, monkeypatch): _, calls = native - with pytest.raises(ValueError, match="only engine='regex'"): - engine.scan("", name, backend="rust") - assert not calls + monkeypatch.setattr(engine, "_spacy_entities", lambda _: []) + monkeypatch.setattr(engine, "_gliner_entities", lambda _: []) + assert not engine.scan("", name, backend="rust").entities + assert calls == [""] def test_bad_backend_rejected(monkeypatch): diff --git a/tests/test_rust_contract.py b/tests/test_rust_contract.py index c3f16bc0..73909bf4 100644 --- a/tests/test_rust_contract.py +++ b/tests/test_rust_contract.py @@ -6,7 +6,12 @@ import pytest from tests.contract_481 import FIXTURE -from tests.rust_contract import DEVIATIONS, applicable, native_observation +from tests.rust_contract import ( + DEVIATIONS, + applicable, + expected_observation, + native_observation, +) pytest.importorskip("datafog_core") CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) @@ -30,6 +35,4 @@ def test_native_version_and_review_inventory(): @pytest.mark.parametrize("case", CASES, ids=lambda case: case["id"]) def test_native_contract(case): - deviation = REVIEWED["cases"].get(case["id"]) - expected = deviation["expected"] if deviation else case["expected"] - assert native_observation(case) == expected, case["id"] + assert native_observation(case) == expected_observation(case), case["id"] diff --git a/tests/test_spacy_span_contract.py b/tests/test_spacy_span_contract.py new file mode 100644 index 00000000..7029b391 --- /dev/null +++ b/tests/test_spacy_span_contract.py @@ -0,0 +1,56 @@ +"""Optional real-spaCy offset integration without downloading model weights.""" + +import pytest + +from datafog import engine +from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator +from datafog.services.text_service import TextService + +spacy = pytest.importorskip("spacy") + + +@pytest.fixture +def real_annotator(monkeypatch): + nlp = spacy.blank("en") + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns( + [ + {"label": "PERSON", "pattern": "Ann"}, + {"label": "ORG", "pattern": "Acme Corp"}, + {"label": "DATE", "pattern": "yesterday"}, + ] + ) + annotator = SpacyPIIAnnotator(nlp=nlp, model_name="local-test-ruler") + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: annotator) + return annotator + + +def test_spacy_ranges_redact_actual_name_without_hitting_prefix(real_annotator): + text = "Annual report by Ann yesterday: 6011111111111117" + result = engine.scan_and_redact(text, engine="spacy") + assert ( + result.redacted_text == "Annual report by [PERSON_1] yesterday: [CREDIT_CARD_1]" + ) + assert all(text[item.start : item.end] == item.text for item in result.entities) + + +def test_spacy_service_combines_ner_and_native_detection(real_annotator): + text = "Ann at Acme Corp: 6011111111111117 ip 2001:db8::1" + service = TextService(engine="spacy") + result = service.annotate_text_sync(text) + assert result == { + "PERSON": ["Ann"], + "ORGANIZATION": ["Acme Corp"], + "CREDIT_CARD": ["6011111111111117"], + "IP_ADDRESS": ["2001:db8::1"], + } + assert all( + text[item.start : item.end] == item.text + for item in service.annotate_text_sync(text, structured=True) + ) + + +def test_spacy_repeated_names_keep_each_original_span(real_annotator): + text = "Ann wrote an Annual report for Ann." + result = engine.scan(text, engine="spacy") + assert [(item.start, item.end) for item in result.entities] == [(0, 3), (31, 34)] diff --git a/tests/test_text_service.py b/tests/test_text_service.py index 74156082..3d5973ad 100644 --- a/tests/test_text_service.py +++ b/tests/test_text_service.py @@ -1,366 +1,124 @@ -from unittest.mock import Mock, patch - -import pytest - -# Legacy coverage for the historical shadow TextService. New behavior belongs -# in datafog.services.text_service. -from datafog.services.text_service_original import TextService +"""Public TextService behavior using the shipped service and shared engine.""" +import asyncio +import threading +from types import SimpleNamespace -@pytest.fixture -def mock_annotator(): - mock = Mock() - mock.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - return mock - +import pytest -@pytest.fixture -def mock_regex_annotator(): - mock = Mock() - mock.annotate.return_value = { - "EMAIL": ["john@example.com"], - "PHONE": ["555-555-5555"], - } +from datafog import engine +from datafog.services.text_service import TextService - # Add mock for annotate_with_spans method - from datafog.processing.text_processing.regex_annotator import ( - AnnotationResult, - Span, - ) - spans = [ - Span(label="EMAIL", start=0, end=15, text="john@example.com"), - Span(label="PHONE", start=20, end=32, text="555-555-5555"), - ] - mock.annotate_with_spans.return_value = ( - {"EMAIL": ["john@example.com"], "PHONE": ["555-555-5555"]}, - AnnotationResult(text="test", spans=spans), - ) - return mock +@pytest.mark.parametrize( + "options", + [{"engine": "invalid"}, {"text_chunk_length": 0}, {"text_chunk_length": -1}], +) +def test_invalid_configuration_is_explicit(options): + with pytest.raises(ValueError): + TextService(**options) -@pytest.fixture -def text_service(mock_annotator, mock_regex_annotator): - # Configure regex annotator to return empty results so auto mode falls back to spaCy - # This ensures backward compatibility with existing tests while using 'auto' mode - mock_regex_annotator.annotate.return_value = { - key: [] - for key in ["EMAIL", "PHONE", "SSN", "CREDIT_CARD", "IP_ADDRESS", "DOB", "ZIP"] +def test_default_service_detects_native_only_entities(): + result = TextService().annotate_text_sync("card 6011111111111117 ip 2001:db8::1") + assert result == { + "CREDIT_CARD": ["6011111111111117"], + "IP_ADDRESS": ["2001:db8::1"], } - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - return_value=mock_annotator, - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - return_value=mock_regex_annotator, - ): - # Use 'auto' engine to match production default, but regex will find nothing - # so it will always fall back to spaCy, maintaining test compatibility - return TextService(text_chunk_length=10, engine="auto") - - -@pytest.fixture -def text_service_with_engine(mock_annotator, mock_regex_annotator): - def _create_service(engine="auto"): - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - return_value=mock_annotator, - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - return_value=mock_regex_annotator, - ): - return TextService(text_chunk_length=10, engine=engine) - return _create_service +def test_long_document_seams_preserve_complete_entities(): + value = "john@example.com" + text = "x" * 995 + " " + value + " " + "y" * 1010 + " ssn 123-45-6789" + result = TextService(text_chunk_length=1000).annotate_text_sync(text) + assert result == {"EMAIL": [value], "SSN": ["123-45-6789"]} -def test_init(text_service): - assert text_service.text_chunk_length == 10 - - -def test_init_with_default_engine(text_service): - assert text_service.text_chunk_length == 10 - # We're using 'auto' in our fixture to match production default - assert text_service.engine == "auto" - - -def test_init_with_custom_engine(text_service_with_engine): - service = text_service_with_engine(engine="regex") - assert service.engine == "regex" - - service = text_service_with_engine(engine="spacy") - assert service.engine == "spacy" - - service = text_service_with_engine(engine="auto") - assert service.engine == "auto" - - -def test_init_with_invalid_engine(): - with pytest.raises(AssertionError, match="Invalid engine"): - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - ): - TextService(engine="invalid") - - -def test_chunk_text(text_service): - text = "This is a test sentence for chunking." - chunks = text_service._chunk_text(text) - assert len(chunks) == 4 - assert chunks == ["This is a ", "test sente", "nce for ch", "unking."] +def test_structured_output_preserves_unicode_source_ranges(): + text = "😀 café john@example.com and john@example.com" + result = TextService(text_chunk_length=10).annotate_text_sync(text, structured=True) + assert len(result) == 2 + assert all( + item.label == "EMAIL" and text[item.start : item.end] == item.text + for item in result + ) + assert result[0].start < result[1].start -def test_combine_annotations(text_service): - annotations = [ - {"PER": ["John"], "ORG": ["Acme"]}, - {"PER": ["Doe"], "LOC": ["New York"]}, +def test_batch_results_preserve_input_order_and_duplicate_documents(): + texts = ["a@example.com", "no pii", "a@example.com"] + assert TextService().batch_annotate_text_sync(texts) == [ + {"EMAIL": ["a@example.com"]}, + {}, + {"EMAIL": ["a@example.com"]}, ] - combined = text_service._combine_annotations(annotations) - assert combined == {"PER": ["John", "Doe"], "ORG": ["Acme"], "LOC": ["New York"]} - - -def test_annotate_text_sync(text_service): - result = text_service.annotate_text_sync("John Doe works at Acme Inc") - assert result == { - "PER": ["John Doe", "John Doe", "John Doe"], - "ORG": ["Acme Inc", "Acme Inc", "Acme Inc"], - } - - -def test_batch_annotate_text_sync(text_service): - texts = ["John Doe", "Acme Inc"] - result = text_service.batch_annotate_text_sync(texts) - assert result == { - "John Doe": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - "Acme Inc": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - } @pytest.mark.asyncio -async def test_annotate_text_async(text_service): - result = await text_service.annotate_text_async("John Doe works at Acme Inc") - assert result == { - "PER": ["John Doe", "John Doe", "John Doe"], - "ORG": ["Acme Inc", "Acme Inc", "Acme Inc"], - } +async def test_async_results_match_synchronous_service(): + service = TextService() + texts = ["a@example.com", "ssn 123-45-6789", "no pii"] + assert await service.batch_annotate_text_async( + texts + ) == service.batch_annotate_text_sync(texts) + assert await service.annotate_text_async( + texts[0], structured=True + ) == service.annotate_text_sync(texts[0], structured=True) @pytest.mark.asyncio -async def test_batch_annotate_text_async(text_service): - texts = ["John Doe", "Acme Inc"] - result = await text_service.batch_annotate_text_async(texts) - assert result == { - "John Doe": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - "Acme Inc": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - } - - -def test_long_text_chunking(text_service): - long_text = "John Doe works at Acme Inc. Jane Smith is from New York City." - result = text_service.annotate_text_sync(long_text) - expected_count = len(text_service._chunk_text(long_text)) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -@pytest.mark.asyncio -async def test_long_text_chunking_async(text_service): - long_text = "John Doe works at Acme Inc. Jane Smith is from New York City." - result = await text_service.annotate_text_async(long_text) - expected_count = len(text_service._chunk_text(long_text)) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -def test_empty_string(text_service): - result = text_service.annotate_text_sync("") - assert result == {} - - -def test_short_string(text_service): - result = text_service.annotate_text_sync("Short") - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_special_characters(text_service): - result = text_service.annotate_text_sync("John@Doe.com works at Acme-Inc!!!") - expected_count = len(text_service._chunk_text("John@Doe.com works at Acme-Inc!!!")) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -def test_regex_engine(text_service_with_engine, mock_regex_annotator): - service = text_service_with_engine(engine="regex") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("john@example.com") - - # Should only call the regex annotator - assert mock_regex_annotator.annotate.called - assert not service.spacy_annotator.annotate.called - assert result == {"EMAIL": ["john@example.com"], "PHONE": ["555-555-5555"]} - - -def test_spacy_engine(text_service_with_engine, mock_annotator): - service = text_service_with_engine(engine="spacy") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("John Doe works at Acme Inc") - - # Should only call the spaCy annotator - assert mock_annotator.annotate.called - assert not service.regex_annotator.annotate.called - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_auto_engine_with_regex_results( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - # Configure regex annotator to return results - mock_regex_annotator.annotate.return_value = {"EMAIL": ["john@example.com"]} - - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("john@example.com") - - # Should call regex annotator but not spaCy - assert mock_regex_annotator.annotate.called - assert not mock_annotator.annotate.called - - assert result == {"EMAIL": ["john@example.com"]} - - -def test_auto_engine_with_fallback( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - # Configure regex annotator to return empty results - mock_regex_annotator.annotate.return_value = {"EMAIL": [], "PHONE": []} - - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("John Doe works at Acme Inc") - - # Should call both annotators - assert mock_regex_annotator.annotate.called - assert mock_annotator.annotate.called - - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_structured_output_regex_engine(text_service_with_engine, mock_regex_annotator): - """Test structured output mode with regex engine.""" - # Set up the mock to return spans that match the input text - from datafog.processing.text_processing.regex_annotator import ( - AnnotationResult, - Span, - ) - - # Create spans that will be returned by the mock - test_text = "john@example.com" - spans = [ - # Make sure the end position matches the actual length of the text - Span(label="EMAIL", start=0, end=len(test_text), text=test_text), - ] - - # Update the mock to return spans that match the input text - mock_regex_annotator.annotate_with_spans.return_value = ( - {"EMAIL": [test_text]}, - AnnotationResult(text=test_text, spans=spans), +async def test_async_detection_runs_outside_event_loop_thread(monkeypatch): + loop_thread = threading.get_ident() + worker_threads = [] + + def detect(text, **_): + worker_threads.append(threading.get_ident()) + return SimpleNamespace(entities=[]) + + monkeypatch.setattr(engine, "scan", detect) + await asyncio.wait_for(TextService().annotate_text_async("text"), timeout=2) + assert worker_threads and all(thread != loop_thread for thread in worker_threads) + + +@pytest.mark.parametrize("mode", ["auto", "smart"]) +def test_optional_ner_adds_to_structured_findings(monkeypatch, mode): + text = "Ann owns 6011111111111117" + monkeypatch.setattr( + engine, + "_gliner_entities", + lambda _: [engine.Entity("PERSON", "Ann", 0, 3, 0.8, "gliner")], ) + result = TextService(engine=mode).annotate_text_sync(text) + assert result == {"PERSON": ["Ann"], "CREDIT_CARD": ["6011111111111117"]} - service = text_service_with_engine(engine="regex") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync(test_text, structured=True) - - # Should call regex annotator's annotate_with_spans method - assert mock_regex_annotator.annotate_with_spans.called - - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 1 # Only one span should be returned (EMAIL) - - # Verify the span has the correct properties - assert result[0].label == "EMAIL" - assert result[0].text == test_text - assert result[0].start == 0 - assert result[0].end == len(test_text) - - -def test_structured_output_spacy_engine(text_service_with_engine, mock_annotator): - """Test structured output mode with spaCy engine.""" - service = text_service_with_engine(engine="spacy") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - # Set up mock to return entities that can be found in the test text - test_text = "John Doe works at Acme Inc" - mock_annotator.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} +def test_custom_gliner_model_is_forwarded(monkeypatch): + models = [] - result = service.annotate_text_sync(test_text, structured=True) + def detect(_, model_name): + models.append(model_name) + return [] - # Should call spaCy annotator - assert mock_annotator.annotate.called - - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 2 - - # Check that spans were created correctly - per_spans = [span for span in result if span.label == "PER"] - org_spans = [span for span in result if span.label == "ORG"] - - assert len(per_spans) == 1 - assert per_spans[0].text == "John Doe" - assert per_spans[0].start == test_text.find("John Doe") - assert per_spans[0].end == test_text.find("John Doe") + len("John Doe") - - assert len(org_spans) == 1 - assert org_spans[0].text == "Acme Inc" - assert org_spans[0].start == test_text.find("Acme Inc") - assert org_spans[0].end == test_text.find("Acme Inc") + len("Acme Inc") - - -def test_structured_output_auto_engine( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - """Test structured output mode with auto engine.""" - # Configure regex annotator to return empty spans - from datafog.processing.text_processing.regex_annotator import AnnotationResult - - mock_regex_annotator.annotate_with_spans.return_value = ( - {"EMAIL": [], "PHONE": []}, - AnnotationResult(text="test", spans=[]), + monkeypatch.setattr(engine, "_gliner_entities", detect) + assert ( + TextService(engine="smart", gliner_model="local/custom").annotate_text_sync( + "no pii" + ) + == {} ) + assert models == ["local/custom"] - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - - # Set up mock to return entities that can be found in the test text - test_text = "John Doe works at Acme Inc" - mock_annotator.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - result = service.annotate_text_sync(test_text, structured=True) +def test_locale_activates_additional_shared_detectors(): + text = "Aufenthaltstitel AT1234567 gueltig." + assert TextService(locales=["de"]).annotate_text_sync(text) == { + "DE_RESIDENCE_PERMIT_NUMBER": ["AT1234567"] + } - # Should call both annotators - assert mock_regex_annotator.annotate_with_spans.called - assert mock_annotator.annotate.called - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 2 +def test_empty_and_invalid_text(): + service = TextService() + assert service.annotate_text_sync("") == {} + assert service.annotate_text_sync("", structured=True) == [] + with pytest.raises(TypeError, match="text must be a string"): + service.annotate_text_sync(None) diff --git a/tests/test_text_service_integration.py b/tests/test_text_service_integration.py index d3363f5f..538f63c8 100644 --- a/tests/test_text_service_integration.py +++ b/tests/test_text_service_integration.py @@ -5,6 +5,15 @@ from datafog.services.text_service import TextService +@pytest.fixture +def require_spacy_model(): + spacy = pytest.importorskip("spacy") + try: + spacy.load("en_core_web_lg") + except OSError: + pytest.skip("en_core_web_lg must be provisioned for trained-model integration") + + @pytest.fixture def real_text_service(): """Create a real TextService instance for integration testing.""" @@ -32,7 +41,7 @@ def test_engine_regex_detects_simple_entities(): @pytest.mark.integration -def test_engine_auto_fallbacks_to_spacy(): +def test_engine_auto_fallbacks_to_spacy(require_spacy_model): """Test that auto mode works correctly with entity detection.""" # We need to test the auto mode in a more controlled way # Create a text that contains only named entities (no emails, phones, etc.) @@ -45,7 +54,7 @@ def test_engine_auto_fallbacks_to_spacy(): # Verify spaCy finds named entities assert "PERSON" in spacy_result and spacy_result["PERSON"] - assert "ORG" in spacy_result and spacy_result["ORG"] + assert "ORGANIZATION" in spacy_result and spacy_result["ORGANIZATION"] # Now create a special text that contains both regex-detectable and spaCy-detectable entities mixed_text = "John Smith's email is john.smith@example.com" @@ -61,7 +70,7 @@ def test_engine_auto_fallbacks_to_spacy(): @pytest.mark.integration -def test_engine_spacy_only(): +def test_engine_spacy_adds_to_structured_pii(require_spacy_model): """Test that spaCy engine is always used regardless of regex potential hits.""" # Sample text with both regex-detectable and spaCy-detectable entities text = """John Smith's email is john.smith@example.com. @@ -79,11 +88,10 @@ def test_engine_spacy_only(): # Verify spaCy detected named entities assert "PERSON" in spacy_result and spacy_result["PERSON"] - assert "ORG" in spacy_result and spacy_result["ORG"] + assert "ORGANIZATION" in spacy_result and spacy_result["ORGANIZATION"] - # Verify spaCy did NOT detect the email (which confirms it's using spaCy only) - # This is because spaCy doesn't have a built-in EMAIL entity type - assert "EMAIL" not in spacy_result or not spacy_result["EMAIL"] + # NER augments the shared structured detector. + assert "john.smith@example.com" in spacy_result["EMAIL"] @pytest.mark.integration @@ -130,7 +138,7 @@ def test_structured_annotation_output(): @pytest.mark.integration -def test_debug_entity_types(): +def test_spacy_entity_labels_are_canonical(require_spacy_model): """Debug test to print the actual entity types returned by spaCy.""" # Sample text with named entities text = """John Smith works at Microsoft Corporation in Seattle. @@ -146,8 +154,8 @@ def test_debug_entity_types(): if values: # Only print non-empty lists print(f" {entity_type}: {values}") - # No assertion needed, this is just for debugging - assert True + assert "ORGANIZATION" in spacy_result + assert "ORG" not in spacy_result @pytest.mark.skip(reason="Performance benchmarking requires more setup") diff --git a/tox.ini b/tox.ini index 83d3f793..a7a1565c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py310,py311,py312 +envlist = py310,py311,py312,py313,py314 isolated_build = True [testenv:integration]