diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e72c9e2a..46abe29a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.9.0 +current_version = 4.12.0 commit = True tag = True tag_name = v{new_version} diff --git a/.flake8 b/.flake8 index e2ed01d9..db0f341c 100644 --- a/.flake8 +++ b/.flake8 @@ -3,4 +3,7 @@ ignore = E203, E266, E501, W503, B006, B007, B008, F401, C416, B950, B904 max-line-length = 88 max-complexity = 18 select = B,C,E,F,W,T4,B9 -exclude = venv, .venv, tests/.datafog_env, examples/venv \ No newline at end of file +exclude = venv, .venv, tests/.datafog_env, examples/venv# datafog.native.py re-exports the installed datafog-core surface through a +# dynamic __getattr__, so its __all__ names resolve at runtime. +per-file-ignores = + datafog/native.py: F822 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index bcc51e1b..944cd756 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,6 +27,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[nlp]" + python -m spacy download en_core_web_lg pip install -r requirements-benchmark.txt # Verify pytest-benchmark is installed and working python -c "import pytest_benchmark; print('pytest-benchmark version:', pytest_benchmark.__version__)" @@ -54,13 +55,7 @@ jobs: # Run benchmarks with optimal performance settings (no memory debugging) echo "Running benchmarks with performance-optimized settings..." - # Try pytest-benchmark first - if python -m pytest tests/benchmark_text_service.py -v --benchmark-autosave --benchmark-json=benchmark-results.json --tb=short -p no:cacheprovider; then - echo "✅ pytest-benchmark tests completed successfully" - else - echo "⚠️ pytest-benchmark failed, running simple performance test as fallback" - python tests/simple_performance_test.py - fi + python -m pytest tests/benchmark_text_service.py -v --benchmark-autosave --benchmark-json=benchmark-results.json --tb=short -p no:cacheprovider - name: Check for performance regression run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be420465..ba447626 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,6 +119,16 @@ jobs: --cov-report=xml \ --cov-report=term-missing + - name: Run detection accuracy corpus on the native engine + if: matrix.python-version == '3.11' && matrix.install-profile == 'nlp-advanced' + env: + DATAFOG_ACCURACY_BACKEND: rust + run: | + pytest tests/test_detection_accuracy.py \ + --tb=short \ + --ignore=tests/test_spark_integration.py \ + --ignore=tests/test_text_service_integration.py + - name: Run tests (nlp) if: matrix.install-profile == 'nlp' run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77e7aedd..0e2b8c63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,9 +2,9 @@ name: Release on: schedule: - # Monday-Wednesday at 2 AM UTC: alpha builds from dev + # Scheduled runs validate candidates without publishing. - cron: "0 2 * * 1-3" - # Thursday at 2 AM UTC: beta builds from dev + # Thursday beta qualification also stops before publication. - cron: "0 2 * * 4" workflow_dispatch: inputs: @@ -19,7 +19,7 @@ on: dry_run: description: "Dry run (skip PyPI publish)" required: false - default: false + default: true type: boolean dry_run_ref: description: "Candidate ref to validate (allowed only with dry_run)" @@ -86,6 +86,15 @@ jobs: fetch-depth: 0 ref: ${{ steps.resolve.outputs.checkout_ref }} + - name: Reject a development branch missing stable changes + if: steps.resolve.outputs.release_type != 'stable' + run: | + git fetch origin main + git merge-base --is-ancestor origin/main HEAD || { + echo "Development source is missing stable changes; merge main into dev first" + exit 1 + } + - name: Freeze source commit id: source run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" @@ -125,7 +134,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v6 with: @@ -167,6 +176,11 @@ jobs: - name: Run detection accuracy corpus run: python -m pytest tests/test_detection_accuracy.py -v --tb=short + - name: Run detection accuracy corpus on the native engine + env: + DATAFOG_ACCURACY_BACKEND: rust + run: python -m pytest tests/test_detection_accuracy.py --tb=short + - name: Run performance validation run: | OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 python tests/simple_performance_test.py @@ -267,7 +281,7 @@ jobs: pip install build twine bump2version - name: Configure git - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true run: | git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]" @@ -277,57 +291,20 @@ jobs: env: VERSION_OVERRIDE: ${{ inputs.version_override }} run: | - set -e + set -euo pipefail git fetch --tags - TYPE="${{ needs.determine-release.outputs.release_type }}" - CURRENT=$(sed -n 's/^__version__ = "\([^"]*\)"/\1/p' datafog/__about__.py) - if [ -z "$CURRENT" ]; then - echo "Failed to parse current version from datafog/__about__.py" - exit 1 - fi - echo "Current version: $CURRENT" - - # Strip any pre-release suffix to get base version - BASE=$(echo "$CURRENT" | sed -E 's/(a|b)[0-9]+([.][0-9A-Za-z]+)?$//') - if [ -n "$VERSION_OVERRIDE" ]; then - BASE="$VERSION_OVERRIDE" - if echo "$BASE" | grep -Eq '(a|b)[0-9]+([.][0-9A-Za-z]+)?$'; then - echo "version_override must be a stable base version like 4.4.0, not a prerelease" - exit 1 - fi - fi - if ! echo "$BASE" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then - echo "Release base must have numeric major.minor.patch format" - exit 1 - fi - echo "Base version: $BASE" - - if [ "$TYPE" = "alpha" ]; then - ALPHA_NUM=1 - while git tag -l "v${BASE}a${ALPHA_NUM}" | grep -q .; do - ALPHA_NUM=$((ALPHA_NUM + 1)) - done - VERSION="${BASE}a${ALPHA_NUM}" - - elif [ "$TYPE" = "beta" ]; then - BETA_NUM=1 - while git tag -l "v${BASE}b${BETA_NUM}" | grep -q .; do - BETA_NUM=$((BETA_NUM + 1)) - done - VERSION="${BASE}b${BETA_NUM}" - - else - VERSION="$BASE" - fi - + VERSION=$(python scripts/release_version.py --type "$TYPE" --override "$VERSION_OVERRIDE") echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "Publishing version: $VERSION" - - sed -i "s/__version__ = \".*\"/__version__ = \"$VERSION\"/" datafog/__about__.py - if grep -q 'version="' setup.py 2>/dev/null; then - sed -i "s/version=\".*\"/version=\"$VERSION\"/" setup.py - fi + RELEASE_VERSION="$VERSION" python - <<'PYCODE' + import os, re + from pathlib import Path + version = os.environ["RELEASE_VERSION"] + about = Path("datafog/__about__.py") + about.write_text(re.sub(r'^__version__ = ".*"', f'__version__ = "{version}"', about.read_text(), flags=re.M)) + config = Path(".bumpversion.cfg") + config.write_text(re.sub(r'^current_version = .*', f'current_version = {version}', config.read_text(), flags=re.M)) + PYCODE - name: Generate changelog env: @@ -366,7 +343,7 @@ jobs: if-no-files-found: error - name: Verify release branch still matches tested source - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: RELEASE_BRANCH: ${{ needs.determine-release.outputs.target_branch }} TESTED_SHA: ${{ needs.determine-release.outputs.source_sha }} @@ -377,30 +354,36 @@ jobs: exit 1 fi + - name: Commit and tag the validated release before publication + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true + env: + VERSION: ${{ steps.version.outputs.version }} + BRANCH: ${{ needs.determine-release.outputs.target_branch }} + run: | + set -euo pipefail + git add datafog/__about__.py .bumpversion.cfg + if ! git diff --cached --quiet; then + git commit -m "chore: bump version to $VERSION [skip ci]" + fi + git tag -a "v$VERSION" -m "Release $VERSION" + git push --atomic origin "HEAD:$BRANCH" "v$VERSION" + - name: Publish to PyPI - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | python -m twine upload dist/* --verbose - - name: Commit version bump & create release - if: inputs.dry_run != true + - name: Create GitHub release + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: GITHUB_TOKEN: ${{ secrets.GH_PAT }} + VERSION: ${{ steps.version.outputs.version }} + TYPE: ${{ needs.determine-release.outputs.release_type }} + BRANCH: ${{ needs.determine-release.outputs.target_branch }} run: | - VERSION="${{ steps.version.outputs.version }}" - TYPE="${{ needs.determine-release.outputs.release_type }}" - BRANCH="${{ needs.determine-release.outputs.target_branch }}" - - git add datafog/__about__.py setup.py - git commit -m "chore: bump version to $VERSION [skip ci]" || echo "No version changes to commit" - git push origin "HEAD:$BRANCH" - - git tag -a "v$VERSION" -m "Release $VERSION" - git push origin "v$VERSION" - PRERELEASE_FLAG="" TITLE="" if [ "$TYPE" = "alpha" ]; then @@ -421,7 +404,7 @@ jobs: dist/* - name: Dry run summary - if: inputs.dry_run == true + if: github.event_name != 'workflow_dispatch' || inputs.dry_run == true run: | echo "DRY RUN COMPLETE" echo "Would have published: ${{ steps.version.outputs.version }}" @@ -429,9 +412,25 @@ jobs: ls -la dist/ # ── 4. Cleanup old pre-releases ─────────────────────────────────────── + sync-development: + needs: [determine-release, publish] + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.determine-release.outputs.release_type == 'stable' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: dev + fetch-depth: 0 + token: ${{ secrets.GH_PAT }} + - name: Bring stable release metadata back to development + run: | + git fetch origin main + git merge --ff-only origin/main + git push origin HEAD:dev + cleanup: needs: [determine-release, publish] - if: needs.determine-release.outputs.release_type != 'stable' && inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.determine-release.outputs.release_type != 'stable' runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/.gitignore b/.gitignore index 46214511..a594afa1 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ docs/* !docs/migration-4.8.1-contract.md !docs/migration-4.9.md !docs/core-0.4-integration.md +!docs/review-remediation.md !docs/agents/ !docs/agents/** !docs/audit/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4063257e..f6b76791 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort args: [--profile=black] exclude: .venv - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 26.3.1 hooks: - id: black language_version: python3 diff --git a/AGENTS.md b/AGENTS.md index 13439692..921d2d55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,9 +13,9 @@ ## Current Project Status -**Stable version: 4.9.0** +**Stable version: 4.11.0** -**Development version: 4.9.0** +**Development candidate: 4.12.0** **Next major target: 5.0.0** @@ -239,17 +239,17 @@ except ImportError: ### Workflow Architecture (3 workflows) -| Workflow | Purpose | Trigger | -| --------------- | ----------------------------------- | -------------------------- | -| `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | -| `release.yml` | Alpha/Beta/Stable publishing | Schedule + manual dispatch | -| `benchmark.yml` | Performance benchmarks | Push/PR/weekly | +| Workflow | Purpose | Trigger | +| --------------- | -------------------------------------- | --------------------------------------------- | +| `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | +| `release.yml` | Candidate qualification and publishing | Schedule validates; manual dispatch publishes | +| `benchmark.yml` | Performance benchmarks | Push/PR/weekly | ### Release Cadence -- **Alpha** (Mon-Wed 2AM UTC): Automatic from `dev`, date+commit versioning -- **Beta** (Thursday 2AM UTC): Automatic from `dev`, incremental beta numbers -- **Stable** (manual dispatch): From `main`, base version or override +- **Alpha** (Mon-Wed 2AM UTC): Candidate qualification from `dev`, without publication +- **Beta** (Thursday 2AM UTC): Candidate qualification from `dev`, without publication +- **Publication** (manual dispatch): Dry run by default; explicit non-dry runs publish alpha/beta from `dev` or stable from `main` ### Release Pipeline @@ -320,7 +320,7 @@ Before beginning any task please checkout a branch from `dev` and create a pull ### Release Preparation -- Alpha/beta releases are automated via `release.yml` schedule +- Alpha/beta candidate qualification is automated via `release.yml`; publication requires an explicit manual dispatch - Stable releases: merge `dev` → `main`, then trigger `release.yml` with `stable` type - Use `dry_run: true` to validate before actual publish - Performance validation on realistic data sets diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 576597a3..ab2e41b6 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,5 +1,79 @@ # ChangeLog +## [Unreleased] + +#### Security + +- Preserve native structured PII detection in smart, spaCy, GLiNER, legacy and + CLI paths. Normalize lone surrogates without moving source spans; retain + structured findings on bounded model inputs. Require review when hook payloads + exceed scan budgets, rather than silently allowing unscanned tails. +- Bound user allowlist matching with timeouts and remove quadratic overlap and + Python fallback regex behavior. Redact exact NER offsets and whole TextService + inputs so repeated names and chunk boundaries cannot expose PII. +- Cover LiteLLM tool arguments, prompt/input requests and complete streamed + responses, with request-scoped tokens. Hide configured API keys in CLI output. + +#### Changed + +- Hash APIs honor the selected algorithm while preserving existing plain-hash + marker widths and defaults. Optional `hash_key` enables full-length HMAC with + a key of at least 32 bytes. Plain hashes provide deterministic pseudonyms and + do not resist guessing of low-entropy PII. +- Reuse tokens for repeated values, avoid existing token collisions, reject + invalid explicit ranges and unsupported entity selections, and report backend + entity capabilities. Async decorators await results before filtering. +- Keep Core within the reviewed minor series, raise the Pydantic floor to 2.1, + include spaCy in `nlp-advanced`, and allow Torch 2.x versions compatible with + `datafog-core[gliner2]`. Add an isolated build declaration and runnable sdist + tests; remove unused duplicate implementation modules. +- Repair Pydantic optional defaults, legacy anonymizer enum filters and overlaps, + Spark label aliases, implicit pipeline scanning and invalid CLI operations. +- Scheduled release jobs qualify candidates without publication. Manual runs + default to dry runs, reject stale version bases and development history, gate + Python 3.13, record release refs before upload, and synchronize stable metadata + back to development after publication. + +## [4.11.0] + +#### Changed + +- Requires `datafog-core>=1.4.0`. Core's text-mode email scanning trims label + syntax by default, so `email=alice@example.com` reports `alice@example.com`, + quoted values drop the wrapping quote, and `SUPPORT_EMAIL=` labels stay out + of the span. Two recorded native-corpus divergences (`json-escaped`, + `markdown-code-block`) resolve and are removed from the reviewed deltas. The + `unicode-chinese-name` delta stays: its corpus fixture contains literal `?` + characters, which are valid local-part characters, so the reviewed + expectation remains overlap-only. The Rust contract review file moves to + `rust-1.4.0.json` with no new deviations: every recorded case still matches. + +## [4.10.0] + +#### Added + +- Native engine is the default: `backend` resolves through `auto`, which uses + `datafog-core` when it imports and the pure-Python engine otherwise. One + warning on fallback, `DATAFOG_BACKEND` and `backend="python"` to opt out. +- Prose `PERSON` and `ADDRESS` from the model embedded in `datafog-core`; no + spaCy, GLiNER, download or interpreter. +- `API_KEY`, `BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, + `ACCOUNT_NUMBER`, `US_ROUTING_NUMBER`, `NPI` and `UUID` entity types, plus the + German set behind the `de` locale. +- `datafog.native` native schema access and `VERSIONING.md`. + +#### Changed + +- `datafog-core>=1.3.0,<2` is a normal dependency; `[rust]` remains as an + explicit spelling. +- `Development Status: 5 - Production/Stable`. +- Retirement warnings name `4.10.0` as the last supported version with an exact + pin, replacing "the final 4.x release". + +#### Fixed + +- `redact(..., entities=[...])` validates `backend` instead of ignoring it. + ## [4.9.0] #### Added diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..75a7cea8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include LICENSE README.md pyproject.toml tox.ini requirements-test.txt +recursive-include tests *.py *.json *.txt +recursive-include scripts *.py +prune tests/.datafog_env +global-exclude __pycache__ *.py[cod] diff --git a/README.md b/README.md index 4fd354a3..73749bed 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DataFog Python +# DataFog Python DataFog is a Python library for detecting and redacting personally identifiable information (PII). @@ -19,8 +19,9 @@ values never echoed into logs or transcripts: - **Claude Code hook** (`datafog-hook`): gates agent tool calls (shell commands, web requests, file writes, MCP tools) and warns the model when - prompts or tool results carry PII. ~70–90ms per invocation including - process startup. Easiest install is the + prompts or tool results carry PII. ~105–110ms per invocation including + process startup and the native engine load (`DATAFOG_BACKEND=python` trims + ~20ms at the cost of detection coverage). Easiest install is the [Claude Code plugin](https://github.com/DataFog/datafog-claude-plugin): ``` @@ -32,8 +33,8 @@ values never echoed into logs or transcripts: - **LiteLLM guardrail** (`DataFogGuardrail`): redacts or blocks PII in requests and responses at the gateway, for any LiteLLM-proxied provider. - In-process (~40µs per message scanned; a request clears the guardrail in - well under a millisecond), no sidecar service. Setup: + In-process (tens of microseconds per message scanned on the native engine), + no sidecar service. Setup: [examples/litellm_guardrail/](examples/litellm_guardrail/). Both default to the high-precision entity set (`EMAIL`, `PHONE`, @@ -51,7 +52,7 @@ NER live in [benchmarks/](benchmarks/). ## Installation ```bash -# Core install (regex engine) +# Core install: native engine, no extra packages needed pip install datafog # Add spaCy support @@ -70,9 +71,34 @@ pip install datafog[distributed] pip install datafog[all] ``` +`pip install datafog` pulls `datafog-core`, which is the native engine. Wheels +exist for Linux (glibc and musl, x86_64/aarch64/s390x/ppc64le), macOS and +Windows (amd64/arm64); anywhere else the source build needs a Rust toolchain. +If Core cannot be imported, detection falls back to the pure-Python engine and +warns once — `DATAFOG_BACKEND=python` selects that path deliberately. + The `nlp-advanced` extra includes SentencePiece and protobuf for GLiNER's multilingual tokenizer; installing the OCR extra is not required for GLiNER. +## Backends + +`backend` defaults to `auto`: the native engine when `datafog-core` is +importable, the pure-Python regex engine otherwise. `DATAFOG_BACKEND` sets the +default and `backend="python"` / `backend="rust"` override it per call. +Every engine uses this backend for structured PII. `engine="spacy"`, +`"gliner"` and `"smart"` add model spans to the same structured findings; +selecting an ML engine does not disable native card or credential detection. + +The native engine adds detectors the Python engine never had — `API_KEY`, +`BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, +`US_ROUTING_NUMBER`, `NPI`, `UUID` and a German set — and serves prose `PERSON` +and `ADDRESS` from an embedded model, so name and address detection needs no +spaCy and no model download. Findings keep `engine="regex"` provenance and +`confidence=1.0` as compatibility values, not native confidence estimates. +Reviewed differences from the Python engine are listed in +[tests/contracts/rust-1.4.0.json](tests/contracts/rust-1.4.0.json). +`datafog.native` exposes the native schema directly. + Python 3.13 support is certified for the core SDK, CLI, `nlp`, `nlp-advanced`, and `ocr` install profiles. Donut OCR still requires a model that is available locally before runtime use. `distributed` and `all` remain @@ -126,10 +152,8 @@ import datafog text = "Steuer-ID 12345678901 liegt vor." -print(datafog.scan(text, engine="regex").entities) -# [] - -print(datafog.scan(text, engine="regex", locales=["de"]).entities) +print(datafog.scan(text, engine="regex", locales=["de"], + entity_types=["DE_TAX_ID"]).entities) # [Entity(type='DE_TAX_ID', text='12345678901', ...)] ``` @@ -188,10 +212,13 @@ scan/redact helpers, or guardrail helpers. model. - A Java runtime is required by PySpark. -DataFog 4.9.0 deprecates OCR and Spark with visible use-time -warnings; their APIs and extras will be removed in 5.0. They remain functional -in 4.9. Users who need these features can stay on the final 4.x release. See -the [4.9 migration guide](docs/migration-4.9.md) for the transition plan. +DataFog deprecates OCR and Spark with visible use-time +warnings; their APIs and extras will be removed in 5.0. The last supported +version in this release train is 4.12.0 — after publication, pin it with +`pip install "datafog==4.12.0"` — and every +retired surface is tracked in the [retirement schedule](docs/retirement.rst). +See the [4.9 migration guide](docs/migration-4.9.md) for the transition plan, +and [VERSIONING.md](VERSIONING.md) for what a version number promises. ## Backward-Compatible APIs @@ -218,6 +245,8 @@ print(result["PHONE"]) ## CLI +Install the command dependencies with `pip install "datafog[cli]"`. + ```bash # Scan text datafog scan-text "john@example.com" @@ -257,22 +286,18 @@ Telemetry does not include input text or detected PII values. ## Development -The [4.9 migration guide](docs/migration-4.9.md) explains opt-in Rust detection, -the native `datafog.v5` preview, and the revised 5.0 retirement schedule for -`detect`/`process`, OCR, and Spark. The Python detector remains the default. - -The experimental Rust adapter in 4.9.0 requires Core `>=0.4.0,<0.5` and capability -contract 1. Upgrade with `python -m pip install --upgrade "datafog[rust]==4.9.0"` -to evaluate it. Base-only users can install `datafog==4.9.0` without Core; -installing the Rust extra does not change the default backend. See the -[4.9.0 release notes](RELEASE_NOTES_4.9.0.md). -Entity labels, locales, and activation settings come from the installed -Core, allowing compatible releases to add detectors without a Python update. -German detection is opt-in through locale or entity selection; UUID is opt-in -through `entity_types=["UUID"]`. Core's structured-only `PERSON` is unavailable -for explicit Rust text selection. The legacy overlap policy can suppress NPI in -favor of PHONE; the migration guide explains native alternatives. Lock the Core -version if detection output must remain reproducible. +The native engine is the default, with a warned Python fallback. The 4.12 +review fixes preserve structured detection across every engine and legacy API. +The dependency stays within the reviewed Core 1.4 minor series (`~=1.4.0`); +Core 1.5 is a separate release candidate that must be published and qualified +before this SDK raises its floor. See [release preparation](docs/review-remediation.md). + +Entity labels, locales and activation settings come from the installed Core. +`get_supported_entities()` reports that backend's supported inventory, including +opt-in labels; it does not mean every label is enabled by default. German +detection is opt-in through locale or entity selection. `UUID`, `PERSON` and +`ADDRESS` can be selected explicitly. Unsupported labels and invalid arguments +raise errors instead of silently disabling detection. The [4.8.1 compatibility contract](docs/migration-4.8.1-contract.md) records published Python behavior for the Rust migration, with frozen fixtures and diff --git a/RELEASE_NOTES_4.10.0.md b/RELEASE_NOTES_4.10.0.md new file mode 100644 index 00000000..7608b6e0 --- /dev/null +++ b/RELEASE_NOTES_4.10.0.md @@ -0,0 +1,71 @@ +# DataFog Python 4.10.0 + +4.10.0 turns on the native engine. `pip install datafog` installs `datafog-core` +and detection uses it. Nothing is removed in this release and no call changes +shape; `backend="python"` restores the previous behaviour exactly. + +Read the [4.10 migration guide](docs/migration-4.10.md) before upgrading a +pipeline whose output is under test. + +## Upgrade + +```bash +python -m pip install --upgrade "datafog==4.10.0" +``` + +Wheels cover Linux (glibc and musl, x86_64/aarch64/s390x/ppc64le), macOS and +Windows (amd64/arm64). Elsewhere the source build needs Rust 1.88+. If Core +cannot be imported, detection falls back to the pure-Python engine and warns +once. + +## What changes + +- `backend` defaults to `auto`: native when `datafog-core` imports, Python + otherwise. `DATAFOG_BACKEND` sets the default; `backend="python"` forces the + fallback. Non-regex engines stay on the Python path. +- `PERSON` and `ADDRESS` for prose now come from an embedded model compiled into + `datafog-core` — no spaCy, no GLiNER, no model download, no interpreter. +- The native engine adds `API_KEY`, `BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, + `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, `US_ROUTING_NUMBER`, `NPI`, `UUID` and the + German identifiers. Credential detectors are on by default; German and `UUID` + stay opt-in. +- `datafog.native` exposes the native schema, `capabilities()`, the embedded + detector and version reporting. Experimental until 5.0. +- `datafog-core` is pinned `>=1.3.0,<2`. Detection output is contract-versioned. + +## Detection differences + +Four reviewed differences from the Python engine are recorded in +[tests/contracts/rust-1.3.0.json](tests/contracts/rust-1.3.0.json) and asserted +by the suite: Core validates card checksums, requires non-alphanumeric SSN +boundaries, requires a North American area code starting 2-9 (which is what +keeps Unix timestamps out of `PHONE`), and rejects unadvertised locales instead +of ignoring them. + +`(555) 123-4567` and similar form data now match: Core keeps the area-code check +and accepts exchange codes starting with 1. + +## Accuracy corpus + +The frozen corpus runs twice in CI now: once against the Python engine, whose +expectations it encodes, and once against the native engine with its twelve +recorded divergences in +[tests/contracts/native-corpus.json](tests/contracts/native-corpus.json). The +native divergences are strictness differences (Core validates more) plus email +boundary handling that Core currently exposes through `format: "env"` / +`"sql"` settings — flagged for a default decision in Core 1.4. + +## Deprecations unchanged + +`detect()`, `process()`, OCR and Spark still work and still warn; all are removed +in 5.0. Their messages now name `4.10.0` as the last supported version and give +the exact pin. See the [retirement schedule](docs/retirement.rst). + +## Notes + +- `VERSIONING.md` is new and defines what a version number promises here, with a + deprecation window, a detection-output rule and a support window. +- Hook invocations cost ~105–110ms including process startup, of which ~20ms is + the native engine load. `DATAFOG_BACKEND=python` trades that back for coverage. +- `datafog[rust]` remains as an explicit spelling of a dependency the base + install already has. diff --git a/VERSIONING.md b/VERSIONING.md new file mode 100644 index 00000000..802af079 --- /dev/null +++ b/VERSIONING.md @@ -0,0 +1,123 @@ +# Versioning and compatibility policy + +DataFog Python follows [Semantic Versioning 2.0.0](https://semver.org/spec/v2.0.0.html). +This document defines what that means here, so a version number is a promise +rather than a guess. + +DataFog versions independently of `datafog-core`. See +[Independent versioning](#independent-versioning). + +## What the numbers mean + +- **MAJOR** (`5.0.0`) — a public API is removed or renamed, an import path moves, + or a default changes such that working caller code stops working. +- **MINOR** (`4.10.0`) — new capability. New entities, new backends, new extras, + new commands, new options, and **changes to detection output**. +- **PATCH** (`4.10.1`) — fixes that do not change findings for a given input, + configuration, and locked dependency set. + +### Detection output is a minor change, never a patch + +The finding set is part of the contract. A patch release must not start finding +more email addresses, or fewer, for input that already worked. If a fix changes +what is found, it ships as a MINOR and the changelog says which entities moved +and in which direction. People write golden files against this output. + +### Dependency-default changes are minor + +Raising a floor, dropping support for a Python version, or changing which +backend is used by default is a MINOR. It is never a PATCH. + +## Deprecation window + +A public API is deprecated in a MINOR, warning at the call boundary, and removed +no earlier than the second following MINOR or 90 days, whichever is longer. + +A deprecation warning must name: + +1. the exact last version that supports the surface, as an installable pin + (`pip install "datafog==4.10.0"`), and +2. the MAJOR that removes it. + +Warnings fire at the public call boundary, before any optional import, so a plain +`import datafog` never pays for a deprecated surface. + +At the removal MAJOR, the removed name stays present as a stub raising +`ImportError` with the same pin, rather than disappearing into an +`AttributeError`. + +## Experimental surface + +Experimental code is marked as such in its documentation and docstrings. It may +break in a MINOR without a deprecation cycle. Anything unmarked is covered by +this policy. + +Currently experimental: + +- `datafog.v5` (native Core preview) +- `datafog.core` (from 4.10.0) +- GLiNER2 and other model sessions +- Browser/WASM entry points + +## Scope + +Covered by this policy: + +- top-level `datafog` callables listed in `datafog.__all__` +- `datafog.compat.v4` and its result classes +- the `datafog` CLI and the `datafog-hook` entry point +- `DataFogGuardrail` and the Claude Code hook integration +- documented configuration keys and environment variables +- the finding set for a given engine, entity selection, and locale + +Not covered: + +- private modules and names beginning with `_` +- `main_lean.py`, `main_original.py`, `__init___original.py` and other + transitional duplicates +- logging text, warning text, and internal error strings +- model accuracy targets for optional ML engines + +## Support window + +The current MAJOR receives fixes for its latest two MINOR releases. Older MINORs +receive security fixes only. Every published version stays installable on PyPI +permanently; a retired surface has a final supported version documented in +[`docs/retirement.rst`](docs/retirement.rst), and that version is the answer. + +## Independent versioning + +`datafog` and `datafog-core` are separate distributions on separate tracks. +`datafog-core` carries the engine; `datafog` carries the application surface. +The transport between them is `capabilities()` and its `contract_version`. +Bumping either `contract_version` or `workspace_contract_version` is MAJOR work +for `datafog-core`, and requires a matching floor in `datafog`. + +Both are exposed separately so a bug report is attributable: + +```python +import datafog, datafog.native +datafog.__version__ # application version +datafog.native.version() # engine version +``` + +## Yanking + +A broken PATCH may be yanked on PyPI and replaced by the next PATCH. A broken +MINOR or MAJOR is superseded, not yanked, because dependents are often already +pinned to it; the changelog opens with the replacement version. + +## Enforcement + +- `RELEASE_NOTES_.md` is required for stable releases. +- The changelog classifies every entry as Added / Changed / Deprecated / Removed + / Fixed and names the version at which each deprecation ends. +- `scripts/check_rust_install.py` and the parity suite run against the installed + wheel, not the checkout. + +## History + +Before 4.10.0 this policy was not written down, and the versioning record shows +it: the 5.0 retirement plan was revised mid-stream (commit `5f5bdbb3`), and the +4.9.0 final publish preceded its own `a1` and `a2` tags. 4.10.0 is the first +release prepared under this policy. diff --git a/datafog/__about__.py b/datafog/__about__.py index 21d77306..ade1e2b4 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.9.0a2" +__version__ = "4.12.0" diff --git a/datafog/__init__.py b/datafog/__init__.py index 591e0ef4..467f5a70 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -12,6 +12,7 @@ import warnings from .__about__ import __version__ +from ._legacy_retirement import LAST_SUPPORTED as _LAST_SUPPORTED from .agent import create_guardrail, filter_output, sanitize, scan_prompt # Core API functions - always available (lightweight) @@ -81,6 +82,13 @@ def __getattr__(name: str): elif name in _LAZY_EXPORTS: module_path, attr_name, extra_name = _LAZY_EXPORTS[name] + if attr_name is None: + # Submodule export (``datafog.native``): return the module itself. + import importlib + + value = importlib.import_module(module_path) + globals()[name] = value + return value try: module = __import__(module_path, fromlist=[attr_name]) value = getattr(module, attr_name) @@ -104,6 +112,7 @@ def _missing_dependency(*args, **kwargs): _LAZY_EXPORTS = { + "native": ("datafog.native", None, None), "app": ("datafog.client", "app", None), "DataFog": ("datafog.main", "DataFog", None), "TextPIIAnnotator": ("datafog.main", "TextPIIAnnotator", None), @@ -131,7 +140,8 @@ def _missing_dependency(*args, **kwargs): def _warn_v5_replacement(old_api: str, replacement: str) -> None: warnings.warn( f"datafog.{old_api}() is deprecated and will be removed in 5.0. Use {replacement} instead. " - "The earlier promise to retain this shim through 5.x has been revised.", + f'Last supported version is {_LAST_SUPPORTED}. Pin it with: pip install "datafog=={_LAST_SUPPORTED}". ' + "The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst.", FutureWarning, stacklevel=3, ) @@ -145,9 +155,9 @@ def scan( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> ScanResult: - """Scan using the 4.x compatibility API; Rust detection is opt-in.""" + """Scan using the 4.x compatibility API; native detection is the default.""" return _compat_v4.scan( text, engine, @@ -170,7 +180,9 @@ def redact( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Redact using the 4.x compatibility API and legacy strategies.""" return _compat_v4.redact( @@ -184,6 +196,8 @@ def redact( allowlist, allowlist_patterns, backend=backend, + hash_type=hash_type, + hash_key=hash_key, ) @@ -232,18 +246,15 @@ def _detect_impl(text: str) -> list: _start = _time.monotonic() - _lazy_import_regex_annotator() - annotator = RegexAnnotator() - # Use the structured output to get proper positions - _, result = annotator.annotate_with_spans(text) + result = scan(text) # Convert to simple format, filtering out empty matches entities = [] - for span in result.spans: + for span in result.entities: if span.text.strip(): # Only include non-empty matches entities.append( { - "type": span.label, + "type": span.type, "value": span.text, "start": span.start, "end": span.end, @@ -274,7 +285,14 @@ def _detect_impl(text: str) -> list: return entities -def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: +def process( + text: str, + anonymize: bool = False, + method: str = "redact", + *, + hash_type: str | None = None, + hash_key: bytes | None = None, +) -> dict: """ Process text to detect and optionally anonymize PII. @@ -282,6 +300,9 @@ def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: text: Input text to process anonymize: Whether to anonymize detected PII method: Anonymization method ('redact', 'replace', 'hash') + hash_type: Optional algorithm. The legacy plain-hash default is MD5; + keyed hashes default to SHA256. + hash_key: At least 32 bytes for a full keyed HMAC digest. Returns: Dictionary with original text, anonymized text (if requested), and findings @@ -301,31 +322,37 @@ def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: _start = _time.monotonic() + if method not in {"redact", "replace", "hash"}: + raise ValueError("method must be 'redact', 'replace', or 'hash'") + if anonymize and method == "hash": + from .engine import _hash_value, _validate_hash_parameters + + algorithm = ( + hash_type + if hash_type is not None + else ("sha256" if hash_key is not None else "md5") + ) + _validate_hash_parameters("hash", algorithm, hash_key) findings = _detect_impl(text) result = {"original": text, "findings": findings} if anonymize: - anonymized = text - # Simple anonymization - replace from end to start to preserve positions - for finding in sorted(findings, key=lambda x: x["start"], reverse=True): - start, end = finding["start"], finding["end"] - entity_type = finding["type"] - - if method == "redact": - replacement = f"[{entity_type}_REDACTED]" - elif method == "replace": - replacement = f"[{entity_type}_XXXXX]" - elif method == "hash": - import hashlib - - replacement = f"[{entity_type}_{hashlib.md5(finding['value'].encode()).hexdigest()[:8]}]" + parts = [] + cursor = 0 + suffix = "REDACTED" if method == "redact" else "XXXXX" + for finding in sorted(findings, key=lambda item: item["start"]): + if method == "hash": + digest = _hash_value( + finding["value"], algorithm, hash_key, plain_length=8 + ) + replacement = f"[{finding['type']}_{digest}]" else: - replacement = f"[{entity_type}]" - - anonymized = anonymized[:start] + replacement + anonymized[end:] - - result["anonymized"] = anonymized + replacement = f"[{finding['type']}_{suffix}]" + parts.extend((text[cursor : finding["start"]], replacement)) + cursor = finding["end"] + parts.append(text[cursor:]) + result["anonymized"] = "".join(parts) try: from .telemetry import _get_duration_bucket, track_function_call diff --git a/datafog/__init___lean.py b/datafog/__init___lean.py deleted file mode 100644 index 3f1165b8..00000000 --- a/datafog/__init___lean.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Historical shadow package export module. - -This file is not the live DataFog 4.5 package export surface. Use -``datafog/__init__.py`` for current package exports, lazy optional imports, -top-level helpers, and compatibility shims. This snapshot remains importable -only as historical reference until legacy cleanup can remove it safely. - -DataFog: Lightning-fast PII detection and anonymization library. - -Core package provides regex-based PII detection, 100x+ faster than NER-based -detection on identical payloads (reproduce: python benchmarks/run.py). -Optional extras available for advanced features: -- pip install datafog[nlp] - for spaCy integration -- pip install datafog[ocr] - for image/OCR processing -- pip install datafog[all] - for all features -""" - -from .__about__ import __version__ - -# Core imports - always available -from .models.annotator import AnnotationResult, AnnotatorRequest -from .models.anonymizer import ( - AnonymizationResult, - Anonymizer, - AnonymizerRequest, - AnonymizerType, -) -from .models.common import EntityTypes -from .processing.text_processing.regex_annotator import RegexAnnotator - -# Optional imports with graceful fallback -try: - from .client import app -except ImportError: - app = None - -try: - from .main import DataFog, TextPIIAnnotator -except ImportError: - DataFog = None - TextPIIAnnotator = None - -try: - from .services.text_service import TextService -except ImportError: - TextService = None - - -# Optional heavy features - only import if dependencies available -def _optional_import(name, module_path, extra_name): - """Helper to import optional modules with helpful error messages.""" - try: - module = __import__(module_path, fromlist=[name]) - return getattr(module, name) - except ImportError: - - def _missing_dependency(*args, **kwargs): - raise ImportError( - f"{name} requires additional dependencies. " - f"Install with: pip install datafog[{extra_name}]" - ) - - return _missing_dependency - - -# OCR/Image processing - requires 'ocr' extra -DonutProcessor = _optional_import( - "DonutProcessor", "datafog.processing.image_processing.donut_processor", "ocr" -) -PytesseractProcessor = _optional_import( - "PytesseractProcessor", - "datafog.processing.image_processing.pytesseract_processor", - "ocr", -) -ImageService = _optional_import("ImageService", "datafog.services.image_service", "ocr") - -# NLP processing - requires 'nlp' extra -SpacyPIIAnnotator = _optional_import( - "SpacyPIIAnnotator", "datafog.processing.text_processing.spacy_pii_annotator", "nlp" -) - -# Distributed processing - requires 'distributed' extra -SparkService = _optional_import( - "SparkService", "datafog.services.spark_service", "distributed" -) - - -# Simple API for core functionality -def detect(text: str) -> list: - """ - Detect PII in text using regex patterns. - - Args: - text: Input text to scan for PII - - Returns: - List of detected PII entities - - Example: - >>> from datafog import detect - >>> detect("Contact john@example.com") - [{'type': 'EMAIL', 'value': 'john@example.com', 'start': 8, 'end': 24}] - """ - annotator = RegexAnnotator() - result = annotator.annotate(text) - - # Convert to simple format - entities = [] - for entity_type, matches in result.items(): - for match in matches: - entities.append( - { - "type": entity_type, - "value": match, - "start": text.find(match), - "end": text.find(match) + len(match), - } - ) - - return entities - - -def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: - """ - Process text to detect and optionally anonymize PII. - - Args: - text: Input text to process - anonymize: Whether to anonymize detected PII - method: Anonymization method ('redact', 'replace', 'hash') - - Returns: - Dictionary with original text, anonymized text (if requested), and findings - - Example: - >>> from datafog import process - >>> process("Contact john@example.com", anonymize=True) - { - 'original': 'Contact john@example.com', - 'anonymized': 'Contact [EMAIL_REDACTED]', - 'findings': [{'type': 'EMAIL', 'value': 'john@example.com', ...}] - } - """ - findings = detect(text) - - result = {"original": text, "findings": findings} - - if anonymize: - anonymized = text - # Simple anonymization - replace from end to start to preserve positions - for finding in sorted(findings, key=lambda x: x["start"], reverse=True): - start, end = finding["start"], finding["end"] - entity_type = finding["type"] - - if method == "redact": - replacement = f"[{entity_type}_REDACTED]" - elif method == "replace": - replacement = f"[{entity_type}_XXXXX]" - elif method == "hash": - import hashlib - - replacement = f"[{entity_type}_{hashlib.md5(finding['value'].encode()).hexdigest()[:8]}]" - else: - replacement = f"[{entity_type}]" - - anonymized = anonymized[:start] + replacement + anonymized[end:] - - result["anonymized"] = anonymized - - return result - - -# Core exports -__all__ = [ - "__version__", - "detect", - "process", - "AnnotationResult", - "AnnotatorRequest", - "AnonymizationResult", - "Anonymizer", - "AnonymizerRequest", - "AnonymizerType", - "EntityTypes", - "RegexAnnotator", - # Optional exports (may be None if dependencies missing) - "DataFog", - "TextPIIAnnotator", - "TextService", - "app", - "DonutProcessor", - "PytesseractProcessor", - "ImageService", - "SpacyPIIAnnotator", - "SparkService", -] diff --git a/datafog/__init___original.py b/datafog/__init___original.py deleted file mode 100644 index 380511b1..00000000 --- a/datafog/__init___original.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Historical shadow package export module. - -This file is not the live DataFog 4.5 package export surface. Use -``datafog/__init__.py`` for current package exports, lazy optional imports, -top-level helpers, and compatibility shims. This eager-export snapshot remains -importable only as historical reference until legacy cleanup can remove it -safely. -""" - -from .__about__ import __version__ -from .client import app -from .config import OperationType, get_config -from .main import DataFog, TextPIIAnnotator -from .models.annotator import ( - AnalysisExplanation, - AnnotationResult, - AnnotationResultWithAnaysisExplanation, - AnnotatorRequest, -) -from .models.anonymizer import ( - AnonymizationResult, - Anonymizer, - AnonymizerRequest, - AnonymizerType, -) -from .models.common import AnnotatorMetadata, EntityTypes, Pattern, PatternRecognizer -from .models.spacy_nlp import SpacyAnnotator -from .processing.image_processing.donut_processor import DonutProcessor -from .processing.image_processing.image_downloader import ImageDownloader -from .processing.image_processing.pytesseract_processor import PytesseractProcessor -from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator -from .services.image_service import ImageService -from .services.spark_service import SparkService -from .services.text_service import TextService - -__all__ = [ - "DonutProcessor", - "DataFog", - "ImageService", - "OperationType", - "SparkService", - "TextPIIAnnotator", - "TextService", - "SpacyPIIAnnotator", - "ImageDownloader", - "PytesseractProcessor", - "__version__", - "app", - "AnalysisExplanation", - "AnnotationResult", - "AnnotationResultWithAnaysisExplanation", - "AnnotatorRequest", - "AnnotatorMetadata", - "EntityTypes", - "Pattern", - "PatternRecognizer", - "get_config", - "SpacyAnnotator", - "AnonymizerType", - "AnonymizerRequest", - "AnonymizationResult", - "Anonymizer", -] diff --git a/datafog/_core_capabilities.py b/datafog/_core_capabilities.py index a5a0df4b..77cd6c9d 100644 --- a/datafog/_core_capabilities.py +++ b/datafog/_core_capabilities.py @@ -85,7 +85,7 @@ def _requested_locales(locales: Any, lookup: dict[str, str]) -> set[str]: return {_resolve_locale(locale, lookup) for locale in locales or []} -def _activation_config(label: str, metadata: dict) -> dict: +def _activation_config(label: str, metadata: dict) -> dict | None: details = metadata.get(label) if not isinstance(details, dict): raise _invalid(f"missing entity metadata for {label!r}") @@ -95,10 +95,9 @@ def _activation_config(label: str, metadata: dict) -> dict: ): raise _invalid(f"invalid scopes for {label!r}") if "text" not in scopes: - raise ValueError( - f"Entity {label!r} is not available for Rust text scanning " - "(structured-only entity)" - ) + # Structured-only labels are served by the caller, usually from the + # embedded model, so a mixed selection is not an error. + return None activation = details.get("activation") if not isinstance(activation, dict): raise _invalid(f"missing activation metadata for {label!r}") @@ -120,13 +119,15 @@ def scan_configs(core: Any, requested: set[str], locales: Any) -> list[dict]: capabilities = getattr(core, "capabilities", None) if not callable(capabilities): raise _invalid( - "contract version 1 is required; install a compatible Core 0.4.x" + "contract version 1 is required; install a compatible datafog-core" ) supported, lookup, metadata = _read_capabilities(capabilities) selected_locales = _requested_locales(locales, lookup) common: dict[str, Any] = {} for label in sorted(requested & supported): config = _activation_config(label, metadata) + if config is None: + continue for key, value in config.items(): if key == "locale": selected_locales.add(_resolve_locale(value, lookup)) diff --git a/datafog/_legacy_retirement.py b/datafog/_legacy_retirement.py index 5bc96c13..41753955 100644 --- a/datafog/_legacy_retirement.py +++ b/datafog/_legacy_retirement.py @@ -2,6 +2,11 @@ import warnings +from .__about__ import __version__ + +#: Final release on which retired optional surfaces still work. +LAST_SUPPORTED = __version__ + class LegacySurfaceWarning(FutureWarning): """An optional DataFog surface scheduled for removal in 5.0.""" @@ -11,7 +16,8 @@ def warn_legacy_surface(surface: str) -> None: """Warn at the caller's public API boundary, before optional imports.""" warnings.warn( f"DataFog {surface} support is deprecated in 4.9 and will be removed in 5.0. " - "Remain on the final 4.x release if you need continued support.", + f'Last supported version is {LAST_SUPPORTED}. Pin it with: pip install "datafog=={LAST_SUPPORTED}". ' + "See docs/retirement.rst.", LegacySurfaceWarning, stacklevel=3, ) diff --git a/datafog/agent.py b/datafog/agent.py index e3cd7bdb..d8bfe56e 100644 --- a/datafog/agent.py +++ b/datafog/agent.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect import warnings from contextlib import contextmanager from dataclasses import dataclass @@ -101,14 +102,27 @@ def filter(self, text: str) -> RedactResult: return result def __call__(self, fn: F) -> F: - """Decorator that applies guardrail filtering to string return values.""" + """Filter string results, awaiting coroutine functions before scanning. + + Other result types raise TypeError rather than escaping unscanned. + """ + + def filtered(output: Any) -> str: + if not isinstance(output, str): + raise TypeError("A guarded function must return a string") + return self.filter(output).redacted_text + + if inspect.iscoroutinefunction(fn): + + @wraps(fn) + async def async_wrapped(*args: Any, **kwargs: Any) -> str: + return filtered(await fn(*args, **kwargs)) + + return async_wrapped # type: ignore[return-value] @wraps(fn) def wrapped(*args: Any, **kwargs: Any) -> Any: - output = fn(*args, **kwargs) - if isinstance(output, str): - return self.filter(output).redacted_text - return output + return filtered(fn(*args, **kwargs)) return wrapped # type: ignore[return-value] diff --git a/datafog/cli.py b/datafog/cli.py new file mode 100644 index 00000000..d5a86302 --- /dev/null +++ b/datafog/cli.py @@ -0,0 +1,17 @@ +"""Console bootstrap with an actionable optional dependency error.""" + + +def main(): + try: + import typer # noqa: F401 + except ImportError: + import sys + + print( + 'The datafog command requires the CLI extra: pip install "datafog[cli]"', + file=sys.stderr, + ) + raise SystemExit(1) from None + from .client import app + + app() diff --git a/datafog/client.py b/datafog/client.py index fd9a303d..cde33431 100644 --- a/datafog/client.py +++ b/datafog/client.py @@ -46,6 +46,15 @@ def list_entities(): app = typer.Typer() +def _operations(value: str) -> list[OperationType]: + try: + return [OperationType(op.strip()) for op in value.split(",")] + except ValueError: + raise typer.BadParameter( + "operations must use scan, extract, redact, replace, or hash" + ) from None + + @app.command() def scan_image( image_urls: List[str] = typer.Argument( @@ -70,9 +79,8 @@ def scan_image( typer.echo("No image URLs or file paths provided. Please provide at least one.") raise typer.Exit(code=1) - logging.basicConfig(level=logging.INFO) # Convert comma-separated string operations to a list of OperationType objects - operation_list = [OperationType(op.strip()) for op in operations.split(",")] + operation_list = _operations(operations) ocr_client = DataFog(operations=operation_list) try: results = asyncio.run(ocr_client.run_ocr_pipeline(image_urls=image_urls)) @@ -130,9 +138,8 @@ def scan_text( typer.echo("No texts provided.") raise typer.Exit(code=1) - logging.basicConfig(level=logging.INFO) # Convert comma-separated string operations to a list of OperationType objects - operation_list = [OperationType(op.strip()) for op in operations.split(",")] + operation_list = _operations(operations) text_client = DataFog(operations=operation_list, locales=locale) try: results = text_client.run_text_pipeline_sync(str_list=str_list) @@ -179,7 +186,11 @@ def show_config(): Prints the current DataFog configuration. """ - typer.echo(get_config()) + config = get_config() + values = dict(config) if isinstance(config, dict) else config.model_dump() + if values.get("api_key"): + values["api_key"] = "[REDACTED]" + typer.echo(values) @app.command() @@ -432,12 +443,11 @@ def hash_text( typer.echo("No text provided to hash.") raise typer.Exit(code=1) - # HashType is retained for backward-compatible CLI signature. - _ = hash_type result = scan_and_redact( text=text, engine="smart", strategy="hash", + hash_type=hash_type.value, locales=locale, ) typer.echo(result.redacted_text) diff --git a/datafog/compat/v4.py b/datafog/compat/v4.py index 0be6b4bb..ce6f00f4 100644 --- a/datafog/compat/v4.py +++ b/datafog/compat/v4.py @@ -6,6 +6,7 @@ from ..engine import Entity, RedactResult, ScanResult from ..engine import redact as _redact_entities +from ..engine import resolve_backend from ..engine import scan as _scan from ..engine import scan_and_redact as _scan_and_redact @@ -29,13 +30,13 @@ def scan( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, ) -> ScanResult: """ - Scan with the legacy result schema; Rust detection is opt-in. + Scan with the legacy result schema. - Defaults to the lightweight regex engine so the core install works without - optional dependency fallback warnings. + Backend selection follows ``datafog.engine.resolve_backend``: native when + ``datafog-core`` is importable, otherwise Python with a one-time warning. ``allowlist`` exempts exact entity texts (your own support address, doc placeholders); ``allowlist_patterns`` exempts entities whose full text @@ -64,7 +65,9 @@ def redact( allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, *, - backend: str = "python", + backend: str | None = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """ Redact with legacy strategies and result schema. @@ -75,8 +78,8 @@ def redact( full-text regex match respectively); they apply to the scan path and are rejected when explicit ``entities`` are supplied. """ - if backend not in ("python", "rust"): - raise ValueError("backend must be one of: python, rust") + # Validate the backend even when explicit entities skip the scan path. + resolve_backend(backend, engine) if preset is not None: try: @@ -91,7 +94,13 @@ def redact( "allowlist/allowlist_patterns cannot be combined with explicit " "entities; filter the entities before calling redact" ) - return _redact_entities(text=text, entities=entities, strategy=strategy) + return _redact_entities( + text=text, + entities=entities, + strategy=strategy, + hash_type=hash_type, + hash_key=hash_key, + ) return _scan_and_redact( text=text, @@ -102,4 +111,6 @@ def redact( allowlist=allowlist, allowlist_patterns=allowlist_patterns, backend=backend, + hash_type=hash_type, + hash_key=hash_key, ) diff --git a/datafog/core.py b/datafog/core.py index e56c2aa5..2153ee94 100644 --- a/datafog/core.py +++ b/datafog/core.py @@ -89,6 +89,9 @@ def anonymize_text( text: str, method: Union[str, AnonymizerType] = "redact", locales: List[str] | None = None, + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> str: """ Simple text anonymization using lightweight regex engine. @@ -103,7 +106,7 @@ def anonymize_text( Example: >>> result = anonymize_text("Contact john@example.com", method="redact") >>> print(result) - "Contact [EMAIL_REDACTED]" + "Contact [EMAIL_1]" """ import time as _time @@ -129,6 +132,8 @@ def anonymize_text( engine=REGEX_ENGINE, strategy=strategy_map[method], locales=locales, + hash_type=hash_type, + hash_key=hash_key, ) try: @@ -211,18 +216,29 @@ def scan_text( return result -def get_supported_entities(locales: List[str] | None = None) -> List[str]: +def get_supported_entities( + locales: List[str] | None = None, *, backend: str | None = None +) -> List[str]: """ - Get list of PII entity types supported by the regex engine. + Get the selected backend's supported entity types, including opt-in labels. Returns: List of supported entity type names Example: >>> entities = get_supported_entities() - >>> print(entities) - ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DATE', 'ZIP_CODE'] + >>> "EMAIL" in entities + True """ + from datafog.engine import _require_core, resolve_backend + + if resolve_backend(backend, "regex") == "rust": + from datafog._core_capabilities import _read_capabilities, scan_configs + + core = _require_core() + supported, _, _ = _read_capabilities(core.capabilities) + scan_configs(core, set(), locales) + return sorted(supported) annotator = RegexAnnotator(locales=locales) legacy_map = {"DOB": "DATE", "ZIP": "ZIP_CODE"} result = [] diff --git a/datafog/engine.py b/datafog/engine.py index 0e2a0fb4..c8041acf 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -3,13 +3,17 @@ from __future__ import annotations import hashlib +import hmac +import os import re import warnings from dataclasses import dataclass from functools import lru_cache from typing import Optional -from .exceptions import EngineNotAvailable +import regex as bounded_re + +from .exceptions import CoreUnavailableWarning, EngineNotAvailable from .processing.text_processing.regex_annotator import RegexAnnotator CANONICAL_TYPE_MAP = { @@ -27,6 +31,13 @@ # Presidio-compatible aliases, so configs migrate without renames. "EMAIL_ADDRESS": "EMAIL", "US_SSN": "SSN", + # Credential detectors are native-only; these spellings keep one vocabulary. + "APIKEY": "API_KEY", + "API-KEY": "API_KEY", + "TOKEN": "BEARER_TOKEN", + "BEARER": "BEARER_TOKEN", + "PRIVATEKEY": "PRIVATE_KEY", + "CREDENTIAL_URI": "CREDENTIAL_URI", } ALL_ENTITY_TYPES = { @@ -48,9 +59,44 @@ "ORGANIZATION", "LOCATION", "ADDRESS", + "API_KEY", + "BEARER_TOKEN", + "JWT", + "PRIVATE_KEY", + "CREDENTIAL_URI", + "ACCOUNT_NUMBER", + "STREET_ADDRESS", + "US_ROUTING_NUMBER", + "NPI", + "UUID", + "PASSPORT_NUMBER", + "MEDICAL_RECORD_NUMBER", + "LICENSE_NUMBER", + "URL", } -NER_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} +#: Labels the native rule engine reports as structured-only, served for text by +#: the embedded model instead. +MODEL_ENTITY_TYPES = {"PERSON", "ADDRESS"} + +#: Environment override for the default backend: ``auto``, ``python`` or ``rust``. +BACKEND_ENV = "DATAFOG_BACKEND" + +_core_warned = False +_cnn_cache: tuple[object, object] | None = None + +SPACY_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} + +NER_ENTITY_TYPES = { + "PERSON", + "ORGANIZATION", + "LOCATION", + "ADDRESS", + "PASSPORT_NUMBER", + "MEDICAL_RECORD_NUMBER", + "LICENSE_NUMBER", + "URL", +} ENTITY_TYPE_PRIORITY = { "DE_IBAN": 100, @@ -127,9 +173,13 @@ def _find_all_occurrences(text: str, needle: str) -> list[tuple[int, int]]: def _entities_from_dict( text: str, payload: dict[str, list[str]], engine: str, confidence: float ) -> list[Entity]: + """Compatibility for third-party annotators without offset support. + + Whole-word matching avoids interpreting "Ann" inside "Annual" as a name. + First-party annotators use their original character ranges instead. + """ entities: list[Entity] = [] value_offsets: dict[str, int] = {} - for raw_type, values in payload.items(): canonical_type = _canonical_type(raw_type) if canonical_type not in ALL_ENTITY_TYPES: @@ -137,25 +187,47 @@ def _entities_from_dict( for value in values: if not isinstance(value, str) or not value.strip(): continue - search_start = value_offsets.get(value, 0) - idx = text.find(value, search_start) - if idx < 0: - idx = text.find(value) - end = idx + len(value) if idx >= 0 else -1 - value_offsets[value] = end if end >= 0 else search_start + 1 + pattern = re.compile(r"(? list[Entity]: + method = getattr(annotator, "annotate_with_spans", None) + if not callable(method): + return _entities_from_dict(text, annotator.annotate(text), engine, confidence) + _, result = method(text) + entities = [] + for span in result.spans: + label = _canonical_type(span.label) + if label not in ALL_ENTITY_TYPES: + continue + if ( + not 0 <= span.start < span.end <= len(text) + or text[span.start : span.end] != span.text + ): + raise ValueError(f"{engine} returned an invalid entity span") + entities.append( + Entity(label, span.text, span.start, span.end, confidence, engine) + ) + return entities + + def _entity_length(entity: Entity) -> int: return max(entity.end - entity.start, 0) @@ -167,7 +239,31 @@ def _entities_overlap(left: Entity, right: Entity) -> bool: def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]: - selected: list[Entity] = [] + """Preserve longest-span precedence with O(n log n) interval queries.""" + coordinates = { + value: index + 1 + for index, value in enumerate( + sorted( + {value for entity in entities for value in (entity.start, entity.end)} + ) + ) + } + starts = [0] * (len(coordinates) + 1) + ends = [0] * (len(coordinates) + 1) + + def add(tree: list[int], index: int) -> None: + while index < len(tree): + tree[index] += 1 + index += index & -index + + def count(tree: list[int], index: int) -> int: + total = 0 + while index > 0: + total += tree[index] + index -= index & -index + return total + + selected = [] for entity in sorted( entities, key=lambda item: ( @@ -179,9 +275,12 @@ def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]: item.type, ), ): - if any(_entities_overlap(entity, kept) for kept in selected): + start, end = coordinates[entity.start], coordinates[entity.end] + if count(starts, end - 1) > count(ends, start): continue selected.append(entity) + add(starts, start) + add(ends, end) return sorted(selected, key=lambda item: (item.start, item.end, item.type)) @@ -209,6 +308,92 @@ def _regex_entities( return _suppress_overlapping_entities(entities) +def _core_module_or_none() -> object | None: + """Return the importable ``datafog_core`` module, or ``None``. + + Deliberately uncached: after the first import this is a ``sys.modules`` + lookup, and not caching keeps ``sys.modules`` patching in tests honest. + """ + try: + import datafog_core + except ImportError: + return None + return datafog_core + + +def _require_core() -> object: + core = _core_module_or_none() + if core is None: + raise ImportError( + "The Rust backend requires datafog-core. " + 'Install with: pip install "datafog[rust]"' + ) + return core + + +def core_available() -> bool: + """Whether the native engine can serve this interpreter.""" + return _core_module_or_none() is not None + + +def _set_core(module: object | None) -> None: + """Test seam: reset cached state derived from the core module.""" + global _cnn_cache + _cnn_cache = None + + +def _cnn_detector_or_none() -> object | None: + """Cached embedded detector, keyed on the module object it came from.""" + global _cnn_cache + core = _core_module_or_none() + if core is None: + return None + if _cnn_cache is not None and _cnn_cache[0] is core: + return _cnn_cache[1] + factory = getattr(core, "EmbeddedCnnDetector", None) + if factory is None: + return None + detector = factory() + _cnn_cache = (core, detector) + return detector + + +def _cnn_findings( + detector: object, requested: set[str], text: str +) -> list[tuple[int, object]]: + """Bound model work while retaining document character coordinates. + + 4096 characters fit the native byte/token caps even for four-byte Unicode. + Overlap gives names and address blocks context on either side of a seam. + """ + chunk_length, overlap = 4096, 256 + findings = [] + for offset in range(0, max(len(text), 1), chunk_length - overlap): + chunk = text[offset : offset + chunk_length] + try: + result = detector.scan_model(chunk) + except Exception as exc: + raise _model_error(exc) from exc + findings.extend( + (offset, finding) + for finding in result.findings + if finding.entity_type in requested + ) + if offset + chunk_length >= len(text): + break + return findings + + +def _model_error(exc: Exception) -> Exception: + """Re-raise a model failure as a ValueError with its native code attached.""" + code = getattr(exc, "code", "model_error") + reason = getattr(exc, "reason", None) + detail = f"{code}/{reason}" if reason else code + error = ValueError(f"Native model detection failed ({detail}): {exc}") + error.code = code # type: ignore[attr-defined] + return error + + def _rust_entities( text: str, entity_types: Optional[list[str]] = None, @@ -220,39 +405,48 @@ def _rust_entities( suppression before entity selection and allowlists, preserving that order without duplicating the native detectors. """ - try: - import datafog_core - except ImportError as exc: - raise ImportError( - "The Rust backend requires datafog-core. " - 'Install with: pip install "datafog[rust]"' - ) from exc + datafog_core = _require_core() from ._core_capabilities import scan_configs requested = {_canonical_type(value) for value in entity_types or []} - configs = scan_configs(datafog_core, requested, locales) + model_requested = requested & MODEL_ENTITY_TYPES + detector = _cnn_detector_or_none() if model_requested else None + if model_requested and detector is None: + raise ValueError( + f"Entity {', '.join(sorted(model_requested))} needs the embedded model, " + "which this datafog-core build does not provide. Upgrade it with: " + 'pip install --upgrade "datafog-core"' + ) + configs = scan_configs(datafog_core, requested - MODEL_ENTITY_TYPES, locales) entities: list[Entity] = [] seen: set[tuple[str, int, int]] = set() - for config in configs: - for finding in datafog_core.scan(text, config=config): - canonical_type = _canonical_type(finding.entity_type) - start = finding.codepoint_range.start - end = finding.codepoint_range.end - key = (canonical_type, start, end) - if key in seen or not finding.matched_text.strip(): - continue - seen.add(key) - entities.append( - Entity( - type=canonical_type, - text=finding.matched_text, - start=start, - end=end, - confidence=1.0, - engine="regex", - ) + native_text = re.sub(r"[\ud800-\udfff]", "\ufffd", text) + findings = [ + (0, finding) + for config in configs + for finding in datafog_core.scan(native_text, config=config) + ] + if detector is not None: + findings.extend(_cnn_findings(detector, model_requested, native_text)) + for offset, finding in findings: + canonical_type = _canonical_type(finding.entity_type) + start = offset + finding.codepoint_range.start + end = offset + finding.codepoint_range.end + key = (canonical_type, start, end) + if key in seen or not finding.matched_text.strip(): + continue + seen.add(key) + entities.append( + Entity( + type=canonical_type, + text=text[start:end], + start=start, + end=end, + confidence=1.0, + engine="regex", ) + ) return _suppress_overlapping_entities(entities) @@ -260,16 +454,22 @@ def _spacy_entities(text: str) -> list[Entity]: annotator = _get_spacy_annotator() if isinstance(annotator, _UnavailableAnnotator): raise EngineNotAvailable(annotator.message) - payload = annotator.annotate(text) - return _entities_from_dict(text, payload, engine="spacy", confidence=0.7) + return [ + entity + for entity in _annotator_entities(text, annotator, "spacy", 0.7) + if entity.type in SPACY_ENTITY_TYPES + ] -def _gliner_entities(text: str) -> list[Entity]: - annotator = _get_gliner_annotator() +def _gliner_entities(text: str, model_name: str | None = None) -> list[Entity]: + annotator = ( + _get_gliner_annotator() + if model_name is None + else _get_gliner_annotator(model_name) + ) if isinstance(annotator, _UnavailableAnnotator): raise EngineNotAvailable(annotator.message) - payload = annotator.annotate(text) - return _entities_from_dict(text, payload, engine="gliner", confidence=0.8) + return _annotator_entities(text, annotator, "gliner", 0.8) @lru_cache(maxsize=1) @@ -289,15 +489,15 @@ def _get_spacy_annotator(): ) -@lru_cache(maxsize=1) -def _get_gliner_annotator(): +@lru_cache(maxsize=4) +def _get_gliner_annotator(model_name: str = "urchade/gliner_multi_pii-v1"): try: from .processing.text_processing.gliner_annotator import GLiNERAnnotator except ImportError as exc: return _UnavailableAnnotator(str(exc)) try: - annotator = GLiNERAnnotator.create() + annotator = GLiNERAnnotator.create(model_name=model_name) except ImportError as exc: return _UnavailableAnnotator(str(exc)) except Exception as exc: @@ -329,44 +529,39 @@ def _filter_entity_types( return [entity for entity in entities if entity.type in allowed] -# Python's re module backtracks; a quantified group containing another -# quantifier (e.g. ``(a+)+``) can take exponential time on adversarial -# input, and entity text can be attacker-influenced (LLM messages, tool -# output). Reject that construct outright rather than matching under it. -_NESTED_QUANTIFIER = re.compile( - r"\((?:[^()\\]|\\.)*(? list["re.Pattern[str]"]: +def _compile_allowlist_patterns(allowlist_patterns: Optional[list[str]]) -> list: compiled = [] + if isinstance(allowlist_patterns, str): + raise ValueError("allowlist_patterns must be a list of patterns") for raw in allowlist_patterns or []: - if len(raw) > MAX_ALLOWLIST_PATTERN_LENGTH: - raise ValueError( - "allowlist_patterns entries must be at most " - f"{MAX_ALLOWLIST_PATTERN_LENGTH} characters" - ) - if _NESTED_QUANTIFIER.search(raw): + if not isinstance(raw, str) or len(raw) > MAX_ALLOWLIST_PATTERN_LENGTH: raise ValueError( - "allowlist_patterns contains a quantified group with a nested " - f"quantifier ({raw!r}), which risks catastrophic backtracking; " - "rewrite the pattern without nesting quantifiers" + "allowlist_patterns entries must be strings of at most 512 characters" ) try: - compiled.append(re.compile(raw)) - except re.error as exc: + compiled.append(bounded_re.compile(raw)) + except bounded_re.error as exc: raise ValueError( f"allowlist_patterns contains an invalid regex: {raw!r} ({exc})" ) from None return compiled +def _allowlist_matches(pattern: object, value: str) -> bool: + if len(value) > MAX_PATTERN_SUBJECT_LENGTH: + return False + try: + return pattern.fullmatch(value, timeout=ALLOWLIST_MATCH_TIMEOUT) is not None + except TimeoutError: + # A slow exemption must never suppress a sensitive finding. + return False + + def _apply_allowlist( entities: list[Entity], allowlist: Optional[list[str]], @@ -389,11 +584,7 @@ def _apply_allowlist( entity for entity in entities if entity.text not in exact - and not any( - pattern.fullmatch(entity.text) - for pattern in patterns - if len(entity.text) <= MAX_PATTERN_SUBJECT_LENGTH - ) + and not any(_allowlist_matches(pattern, entity.text) for pattern in patterns) ] @@ -404,11 +595,71 @@ def _needs_ner(entity_types: Optional[list[str]]) -> bool: return bool(requested & NER_ENTITY_TYPES) +def _warn_core_missing() -> None: + global _core_warned + if _core_warned: + return + _core_warned = True + warnings.warn( + "datafog-core is not installed, so detection falls back to the pure-Python " + f"engine. Install it to use the native engine, or set {BACKEND_ENV}=python " + "to silence this warning.", + CoreUnavailableWarning, + stacklevel=4, + ) + + +def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: + """Resolve ``auto`` to a concrete backend without importing Core eagerly. + + An explicit ``backend`` wins over ``DATAFOG_BACKEND``. ``auto`` uses the + native engine whenever Core is importable and degrades to Python otherwise. + Every engine uses this backend for structured PII; NER adds spans. + """ + requested = backend or os.environ.get(BACKEND_ENV) or "auto" + normalized = requested.strip().lower() + if normalized not in {"auto", "python", "rust"}: + raise ValueError( + f"backend must be one of: auto, python, rust (from {BACKEND_ENV} or the " + "backend argument)" + ) + if normalized == "auto": + return "rust" if core_available() else "python" + return normalized + + def _validate_backend(backend: str, engine: str) -> None: if backend not in {"python", "rust"}: raise ValueError("backend must be one of: python, rust") - if backend == "rust" and engine != "regex": - raise ValueError("backend='rust' supports only engine='regex'") + + +def _validate_entity_types( + entity_types: Optional[list[str]], backend: str, engine: str +) -> Optional[list[str]]: + if entity_types is None: + return None + if not isinstance(entity_types, (list, tuple)) or any( + not isinstance(value, str) or not value.strip() for value in entity_types + ): + raise ValueError("entity_types must be a list of entity labels") + requested = [_canonical_type(value) for value in entity_types] + if backend == "rust": + from ._core_capabilities import _read_capabilities + + supported, _, _ = _read_capabilities(_require_core().capabilities) + supported = supported | MODEL_ENTITY_TYPES + else: + supported = {_canonical_type(label) for label in RegexAnnotator.LABELS} + if engine == "spacy": + supported |= SPACY_ENTITY_TYPES + elif engine in {"smart", "gliner"}: + supported |= NER_ENTITY_TYPES | {"DATE"} + unsupported = set(requested) - supported + if unsupported: + raise ValueError( + f"Unsupported entity_types for {backend}/{engine}: {', '.join(sorted(unsupported))}" + ) + return requested def scan( @@ -419,13 +670,21 @@ def scan( allowlist: Optional[list[str]] = None, allowlist_patterns: Optional[list[str]] = None, *, - backend: str = "python", + backend: Optional[str] = None, + gliner_model: Optional[str] = None, ) -> ScanResult: """Scan text for PII entities. - ``backend="rust"`` opts into experimental native detection for - ``engine="regex"``. Results retain legacy regex provenance and confidence; - these are compatibility values, not native confidence estimates. + ``backend`` defaults to ``auto``: the native engine when ``datafog-core`` is + importable, otherwise the pure-Python engine with a one-time warning. Set + ``DATAFOG_BACKEND`` or pass ``backend="python"`` to force the fallback. The + selected backend supplies structured PII for every engine mode; optional + NER engines add entity spans. + Prose names and address blocks come from the embedded model on the native + path, so ``PERSON`` and ``ADDRESS`` work without spaCy or a model download. + + Results retain legacy regex provenance and confidence; these are + compatibility values, not native confidence estimates. ``allowlist`` exempts exact entity texts (e.g. your own support email); ``allowlist_patterns`` exempts entities whose full text matches a regex @@ -437,7 +696,14 @@ def scan( if engine not in {"regex", "spacy", "gliner", "smart"}: raise ValueError("engine must be one of: regex, spacy, gliner, smart") + requested_backend = ( + (backend or os.environ.get(BACKEND_ENV) or "auto").strip().lower() + ) + backend = resolve_backend(backend, engine) _validate_backend(backend, engine) + entity_types = _validate_entity_types(entity_types, backend, engine) + if backend == "python" and requested_backend == "auto" and not core_available(): + _warn_core_missing() # Validate patterns up front so config errors fail fast even when the # text contains no entities. @@ -477,7 +743,11 @@ def scan( if engine == "gliner" and _needs_ner(entity_types): try: - gliner_entities = _gliner_entities(text) + gliner_entities = ( + _gliner_entities(text) + if gliner_model is None + else _gliner_entities(text, gliner_model) + ) combined.extend(gliner_entities) engines_used.add("gliner") except EngineNotAvailable: @@ -492,13 +762,16 @@ def scan( if engine == "smart" and _needs_ner(entity_types): try: - gliner_entities = _gliner_entities(text) + gliner_entities = ( + _gliner_entities(text) + if gliner_model is None + else _gliner_entities(text, gliner_model) + ) combined.extend(gliner_entities) engines_used.add("gliner") - except EngineNotAvailable: + except EngineNotAvailable as exc: warnings.warn( - "GLiNER not available, smart scan falling back to spaCy. " - "Install with: pip install datafog[nlp-advanced]", + f"GLiNER unavailable; smart scan falling back to spaCy. {exc}", UserWarning, stacklevel=2, ) @@ -516,6 +789,8 @@ def scan( filtered = _filter_entity_types(combined, entity_types) filtered = _apply_allowlist(filtered, allowlist, allowlist_patterns) + # Optional NER adds candidates without suppressing structured findings. + # Transformation resolves overlaps after selection, preserving scan labels. deduped = _dedupe_entities(filtered) return ScanResult( entities=deduped, @@ -524,10 +799,62 @@ def scan( ) +def _validate_hash_parameters( + strategy: str, hash_type: str, hash_key: bytes | None +) -> None: + if hash_key is not None and (not isinstance(hash_key, bytes) or len(hash_key) < 32): + raise ValueError("hash_key must contain at least 32 bytes") + if strategy == "hash": + try: + hashlib.new(hash_type).hexdigest() + except (ValueError, TypeError): + raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None + + +def _hash_value( + text: str, + hash_type: str, + hash_key: bytes | None, + *, + plain_length: int = 12, +) -> str: + """Retain legacy plain-hash widths; keyed HMAC always uses its full digest. + + Unkeyed hashes are deterministic pseudonyms and remain vulnerable to + dictionary attacks regardless of digest width. Supply a private key to + prevent direct enumeration of low-entropy identifiers. + """ + _validate_hash_parameters("hash", hash_type, hash_key) + encoded = text.encode("utf-8", "surrogatepass") + if hash_key is not None: + return hmac.new(hash_key, encoded, hash_type).hexdigest() + return hashlib.new(hash_type, encoded).hexdigest()[:plain_length] + + +def _validated_redaction_entities(text: str, entities: list[Entity]) -> list[Entity]: + valid_entities = [] + for entity in entities: + if ( + not 0 <= entity.start < entity.end <= len(text) + or not entity.text + or text[entity.start : entity.end] != entity.text + ): + raise ValueError( + "entities must contain valid ranges whose text matches the source" + ) + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", entity.type): + raise ValueError("entity types must be uppercase labels") + valid_entities.append(entity) + return _suppress_overlapping_entities(valid_entities) + + def redact( text: str, entities: list[Entity], strategy: str = "token", + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Redact PII entities from text. @@ -537,6 +864,12 @@ def redact( type priority and then confidence. Suppressed spans are omitted from ``RedactResult.entities`` and ``mapping``. + Plain hashes retain the historical 12-character digest prefix and provide + deterministic pseudonymization. With ``hash_key`` (at least 32 bytes), HMAC + uses the complete digest. Unkeyed hashes do not resist dictionary attacks. + + Invalid ranges, mismatched source text, and malformed labels raise ValueError. + ``mapping`` for the ``mask`` strategy is keyed by per-type indexed keys (``[EMAIL_MASK_1]``) rather than the mask string, because distinct same-length values produce identical masks and would collide. @@ -546,22 +879,19 @@ def redact( if strategy not in {"token", "mask", "hash", "pseudonymize"}: raise ValueError("strategy must be one of: token, mask, hash, pseudonymize") - redacted_text = text + _validate_hash_parameters(strategy, hash_type, hash_key) + mapping: dict[str, str] = {} counters: dict[str, int] = {} pseudonym_by_value: dict[tuple[str, str], str] = {} - valid_entities = [ - entity - for entity in entities - if 0 <= entity.start < entity.end <= len(text) and entity.text - ] - # Returns non-overlapping spans sorted by document position. - valid_entities = _suppress_overlapping_entities(valid_entities) + valid_entities = _validated_redaction_entities(text, entities) + + occupied_tokens = set(re.findall(r"\[[A-Z][A-Za-z0-9_]*\]", text)) # Assign replacements in document order so the per-type counters used by - # the token/pseudonymize strategies number the first occurrence _1; spans - # are applied right-to-left below so earlier offsets stay valid. + # the token/pseudonymize strategies number the first occurrence _1. + # Assemble the result in one pass without repeated document copies. replacements: list[tuple[Entity, str]] = [] for entity in valid_entities: original = text[entity.start : entity.end] @@ -570,30 +900,57 @@ def redact( replacement = "*" * max(len(original), 1) # Distinct same-length values share a mask, so the mask string # cannot key the mapping without collisions. - counters[entity.type] = counters.get(entity.type, 0) + 1 - mapping_key = f"[{entity.type}_MASK_{counters[entity.type]}]" + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + mapping_key = f"[{entity.type}_MASK_{counters[entity.type]}]" + if mapping_key not in occupied_tokens: + break + occupied_tokens.add(mapping_key) elif strategy == "hash": - digest = hashlib.sha256(original.encode("utf-8")).hexdigest()[:12] - replacement = f"[{entity.type}_{digest}]" + digest = _hash_value(original, hash_type, hash_key) + key = (entity.type, original) + if key not in pseudonym_by_value: + candidate = f"[{entity.type}_{digest}]" + suffix = 1 + while candidate in occupied_tokens: + suffix += 1 + candidate = f"[{entity.type}_{digest}_{suffix}]" + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) + replacement = pseudonym_by_value[key] elif strategy == "pseudonymize": key = (entity.type, original) if key not in pseudonym_by_value: - counters[entity.type] = counters.get(entity.type, 0) + 1 - pseudonym_by_value[key] = ( - f"[{entity.type}_PSEUDO_{counters[entity.type]}]" - ) + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + candidate = f"[{entity.type}_PSEUDO_{counters[entity.type]}]" + if candidate not in occupied_tokens: + break + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) replacement = pseudonym_by_value[key] else: # token - counters[entity.type] = counters.get(entity.type, 0) + 1 - replacement = f"[{entity.type}_{counters[entity.type]}]" + key = (entity.type, original) + if key not in pseudonym_by_value: + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + candidate = f"[{entity.type}_{counters[entity.type]}]" + if candidate not in occupied_tokens: + break + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) + replacement = pseudonym_by_value[key] replacements.append((entity, replacement)) mapping[mapping_key if mapping_key is not None else replacement] = original - for entity, replacement in reversed(replacements): - redacted_text = ( - redacted_text[: entity.start] + replacement + redacted_text[entity.end :] - ) + parts = [] + offset = 0 + for entity, replacement in replacements: + parts.extend((text[offset : entity.start], replacement)) + offset = entity.end + parts.append(text[offset:]) + redacted_text = "".join(parts) return RedactResult( redacted_text=redacted_text, @@ -611,9 +968,14 @@ def scan_and_redact( allowlist: Optional[list[str]] = None, allowlist_patterns: Optional[list[str]] = None, *, - backend: str = "python", + backend: Optional[str] = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: - """Scan with the selected backend, then apply legacy Python redaction.""" + """Scan with the selected backend, then apply legacy Python redaction. + + Backend selection matches :func:`scan`. + """ scan_result = scan( text=text, engine=engine, @@ -623,4 +985,10 @@ def scan_and_redact( allowlist_patterns=allowlist_patterns, backend=backend, ) - return redact(text=text, entities=scan_result.entities, strategy=strategy) + return redact( + text=text, + entities=scan_result.entities, + strategy=strategy, + hash_type=hash_type, + hash_key=hash_key, + ) diff --git a/datafog/exceptions.py b/datafog/exceptions.py index 98bc8d0d..7ac32340 100644 --- a/datafog/exceptions.py +++ b/datafog/exceptions.py @@ -70,6 +70,10 @@ def __init__(self, message: str): super().__init__(message, status_code=None) +class CoreUnavailableWarning(RuntimeWarning): + """Emitted once when ``backend="auto"`` falls back to the pure-Python engine.""" + + def raise_for_status_code(status_code: int, error_message: str): """ Raise the appropriate exception based on the status code. diff --git a/datafog/integrations/claude_code.py b/datafog/integrations/claude_code.py index 28c4e787..5eb72fb3 100644 --- a/datafog/integrations/claude_code.py +++ b/datafog/integrations/claude_code.py @@ -22,8 +22,9 @@ whose full text matches are exempt (note: a pattern containing a comma cannot be expressed here). -Failure policy: fail open. A hook bug must never brick a Claude Code -session, so any unexpected error exits non-blocking with no output. +Oversized inputs receive the configured ask/deny decision rather than proceeding +with an unscanned tail. Unexpected hook errors fail open with a non-blocking +error, so a hook bug does not prevent using the Claude Code session. """ import json @@ -50,6 +51,10 @@ _EXIT_FAIL_OPEN = 1 +class ScanLimitExceeded(ValueError): + """The payload cannot be fully inspected within the configured budget.""" + + def _entity_types(env: Mapping[str, str]) -> list[str]: raw = env.get("DATAFOG_HOOK_ENTITIES", "") parsed = [t.strip().upper() for t in raw.split(",") if t.strip()] @@ -82,6 +87,7 @@ def _iter_strings(value: Any) -> Iterator[str]: yield current elif isinstance(current, dict): stack.extend(current.values()) + stack.extend(current.keys()) elif isinstance(current, (list, tuple)): stack.extend(current) @@ -98,9 +104,9 @@ def _scan_findings( counts: dict[str, int] = {} total_budget = TOTAL_SCAN_CHARS for text in _iter_strings(value): - if total_budget <= 0: - break - chunk = text[: min(MAX_SCAN_CHARS, total_budget)] + if len(text) > MAX_SCAN_CHARS or len(text) > total_budget: + raise ScanLimitExceeded("Input exceeds the scan budget") + chunk = text total_budget -= len(chunk) result = datafog.scan( chunk, @@ -191,8 +197,20 @@ def run(payload: dict, env: Mapping[str, str]) -> tuple[int, str]: if handler is None: return _EXIT_OK, "" return _EXIT_OK, handler(payload, env) + except ScanLimitExceeded: + event = payload.get("hook_event_name") + reason = "DataFog PII firewall: input exceeds the scan budget and was not fully scanned. Split the input before proceeding." + if event == "PreToolUse": + return _EXIT_OK, _emit( + event, + { + "permissionDecision": _action(env), + "permissionDecisionReason": reason, + }, + ) + return _EXIT_OK, _emit(event, {"additionalContext": reason}) except Exception as exc: # noqa: BLE001 — fail open by design - print(f"datafog-hook error (fail-open): {exc}", file=sys.stderr) + print(f"datafog-hook error (fail-open): {type(exc).__name__}", file=sys.stderr) return _EXIT_FAIL_OPEN, "" @@ -206,7 +224,10 @@ def main() -> None: if not isinstance(payload, dict): payload = {} except Exception as exc: # noqa: BLE001 — fail open by design - print(f"datafog-hook: invalid hook payload (fail-open): {exc}", file=sys.stderr) + print( + f"datafog-hook: invalid hook payload (fail-open): {type(exc).__name__}", + file=sys.stderr, + ) sys.exit(_EXIT_FAIL_OPEN) code, stdout = run(payload, os.environ) diff --git a/datafog/integrations/litellm_guardrail.py b/datafog/integrations/litellm_guardrail.py index 7e33ed55..74a56169 100644 --- a/datafog/integrations/litellm_guardrail.py +++ b/datafog/integrations/litellm_guardrail.py @@ -30,7 +30,9 @@ ``datafog`` core; the LiteLLM proxy, where this runs, always ships fastapi). """ +import copy import logging +import re from typing import Any, Optional from fastapi import HTTPException @@ -42,15 +44,74 @@ VALID_ACTIONS = {"redact", "block"} VALID_FAIL_POLICIES = {"open", "closed"} +MAX_STREAM_CHARS = 8_000_000 +MAX_STREAM_CHUNKS = 100_000 logger = logging.getLogger(__name__) +class _RedactionSession: + """Token assignments belong to one request or response, never the adapter.""" + + def __init__(self, strings=()): + self.tokens: dict[tuple[str, str], str] = {} + self.counters: dict[str, int] = {} + self.reserved = { + token for text in strings for token in re.findall(r"\[[A-Z_]+_\d+\]", text) + } + + def token(self, entity): + key = (entity.type, entity.text) + if key not in self.tokens: + number = self.counters.get(entity.type, 0) + 1 + while f"[{entity.type}_{number}]" in self.reserved: + number += 1 + token = f"[{entity.type}_{number}]" + self.counters[entity.type] = number + self.tokens[key] = token + self.reserved.add(token) + return self.tokens[key] + + +def _strings(value): + stack = [value] + visited = set() + while stack: + item = stack.pop() + if isinstance(item, (dict, list)): + if id(item) in visited: + continue + visited.add(id(item)) + if isinstance(item, str): + yield item + elif isinstance(item, dict): + stack.extend(item.values()) + stack.extend(item.keys()) + elif isinstance(item, list): + stack.extend(item) + + +def _get(value, name, default=None): + return ( + value.get(name, default) + if isinstance(value, dict) + else getattr(value, name, default) + ) + + +def _set(value, name, item): + if isinstance(value, dict): + value[name] = item + else: + setattr(value, name, item) + + def _redact_text( text: str, entity_types: list[str], allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, + session: _RedactionSession | None = None, ) -> tuple[str, dict[str, int]]: """Redact ``text``; return (redacted_text, counts per entity type).""" import datafog @@ -65,7 +126,14 @@ def _redact_text( counts: dict[str, int] = {} for entity in result.entities: counts[entity.type] = counts.get(entity.type, 0) + 1 - return result.redacted_text, counts + session = session or _RedactionSession([text]) + parts = [] + cursor = 0 + for entity in sorted(result.entities, key=lambda entity: entity.start): + parts.extend((text[cursor : entity.start], session.token(entity))) + cursor = entity.end + parts.append(text[cursor:]) + return "".join(parts), counts def _summary(counts: dict[str, int]) -> str: @@ -97,39 +165,51 @@ def __init__( self.allowlist_patterns = allowlist_patterns super().__init__(**kwargs) - def _process_content(self, content: Any) -> tuple[Any, dict[str, int]]: - """Redact a message content value (str or list of content parts).""" + def _process_content( + self, content: Any, session=None + ) -> tuple[Any, dict[str, int]]: + """Walk textual input, including nested tool arguments and content parts.""" + session = session or _RedactionSession(_strings(content)) counts: dict[str, int] = {} if isinstance(content, str): redacted, counts = _redact_text( - content, self.entity_types, self.allowlist, self.allowlist_patterns + content, + self.entity_types, + self.allowlist, + self.allowlist_patterns, + session, ) return redacted, counts - if isinstance(content, list): - new_parts = [] - skipped_parts = 0 - for part in content: - if isinstance(part, dict) and isinstance(part.get("text"), str): - redacted, part_counts = _redact_text( - part["text"], - self.entity_types, - self.allowlist, - self.allowlist_patterns, - ) - new_parts.append({**part, "text": redacted}) - for etype, n in part_counts.items(): - counts[etype] = counts.get(etype, 0) + n + if isinstance(content, (list, dict)): + items = enumerate(content) if isinstance(content, list) else content.items() + output = [] if isinstance(content, list) else {} + for key, value in items: + # Media payloads are opaque; preserve them while scanning any + # textual sibling. name/id/url in ordinary user data are text. + media_type = content.get("type") if isinstance(content, dict) else None + if not isinstance(media_type, str): + media_type = None + opaque = { + "image_url": {"image_url"}, + "input_image": {"image_url", "file_id"}, + "image": {"source"}, + "input_audio": {"input_audio"}, + "audio": {"source", "data"}, + "video_url": {"video_url"}, + "file": {"file"}, + "input_file": {"file_id", "file_data"}, + } + if key in opaque.get(media_type, set()): + new_value, found = value, {} else: - # Images and other non-text parts are not scanned — - # count them so the blind spot is auditable. - new_parts.append(part) - skipped_parts += 1 - if skipped_parts: - logger.debug( - "DataFog guardrail: %d non-text content parts not scanned", - skipped_parts, - ) - return new_parts, counts + new_value, found = self._process_content(value, session) + if isinstance(output, list): + output.append(new_value) + else: + output[key] = new_value + for label, count in found.items(): + counts[label] = counts.get(label, 0) + count + return output, counts return content, counts def _handle_engine_error(self, exc: Exception) -> None: @@ -158,21 +238,20 @@ async def async_pre_call_hook( data: dict, call_type: str, ) -> dict: - messages = data.get("messages") - if not isinstance(messages, list): - return data - total_counts: dict[str, int] = {} - new_messages = [] + new_data = dict(data) + fields = ("messages", "prompt", "input") + session = _RedactionSession( + text for field in fields for text in _strings(data.get(field)) + ) try: - for message in messages: - if isinstance(message, dict) and "content" in message: - new_content, counts = self._process_content(message["content"]) - new_messages.append({**message, "content": new_content}) - for etype, n in counts.items(): - total_counts[etype] = total_counts.get(etype, 0) + n - else: - new_messages.append(message) + for field in fields: + if field in data: + new_data[field], counts = self._process_content( + data[field], session + ) + for label, count in counts.items(): + total_counts[label] = total_counts.get(label, 0) + count except Exception as exc: # noqa: BLE001 — fail policy decides self._handle_engine_error(exc) return data @@ -197,7 +276,6 @@ async def async_pre_call_hook( }, ) - new_data = {**data, "messages": new_messages} self._record_guardrail_logging(new_data, total_counts) return new_data @@ -215,42 +293,130 @@ def _record_guardrail_logging( except Exception: # noqa: BLE001 — observability must never break traffic logger.debug("could not record guardrail logging information") + @staticmethod + def _response_fields(response): + """Yield only generated text fields, preserving response metadata.""" + for choice in _get(response, "choices", []) or []: + if isinstance(_get(choice, "text"), str): + yield choice, "text" + message = _get(choice, "message") + if message is None: + continue + yield message, "content" + for call in _get(message, "tool_calls", []) or []: + function = _get(call, "function") + if function is not None: + yield function, "arguments" + function = _get(message, "function_call") + if function is not None: + yield function, "arguments" + async def async_post_call_success_hook( self, data: dict, user_api_key_dict: Any, response: Any, ) -> Any: - """Redact PII from model responses. + """Redact generated text in place after every field has been scanned.""" + try: + fields = list(self._response_fields(response)) + session = _RedactionSession( + text + for target, field in fields + for text in _strings(_get(target, field)) + ) + changes = [] + for target, field in fields: + value, _ = self._process_content(_get(target, field), session) + changes.append((target, field, value)) + for target, field, value in changes: + _set(target, field, value) + except Exception as exc: # noqa: BLE001 — fail policy decides + self._handle_engine_error(exc) + return response - Mutates ``response`` in place — deliberate: litellm post_call - guardrails share the response object rather than cloning it, and - an unredacted clone escaping through another callback would defeat - the purpose. + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict, response, request_data + ): + """Buffer to completion before emitting, protecting split entities. + + Native text and tool argument deltas are joined per choice/tool. The + first corresponding delta carries the redacted text and later deltas + are empty; metadata and finish events retain their original order. + Incomplete or oversized streams are rejected without emitting chunks. """ - choices = getattr(response, "choices", None) - if not choices: - return response + chunks = [] + groups = {} + size = 0 try: - skipped_parts = 0 - for choice in choices: - message = getattr(choice, "message", None) - if message is not None and isinstance(message.content, str): - redacted, counts = _redact_text( - message.content, - self.entity_types, - self.allowlist, - self.allowlist_patterns, - ) - if counts: - message.content = redacted - elif message is not None and message.content is not None: - skipped_parts += 1 - if skipped_parts: - logger.debug( - "DataFog guardrail: %d non-text response parts not scanned", - skipped_parts, + async for original in response: + # Iterators may reuse a mutable chunk object. Keep each emitted + # delta's value and metadata as they existed when received. + chunk = copy.deepcopy(original) + chunks.append(chunk) + for position, choice in enumerate(_get(chunk, "choices", []) or []): + index = _get(choice, "index", position) + fields = [((index, "text"), choice, "text")] + delta = _get(choice, "delta") + if delta is not None: + fields.append(((index, "content"), delta, "content")) + for call_position, call in enumerate( + _get(delta, "tool_calls", []) or [] + ): + function = _get(call, "function") + if function is not None: + fields.append( + ( + ( + index, + "tool", + _get(call, "index", call_position), + ), + function, + "arguments", + ) + ) + function = _get(delta, "function_call") + if function is not None: + fields.append( + ((index, "function_call"), function, "arguments") + ) + for key, target, field in fields: + value = _get(target, field) + if isinstance(value, str): + size += len(value) + groups.setdefault(key, []).append((target, field, value)) + if size > MAX_STREAM_CHARS or len(chunks) > MAX_STREAM_CHUNKS: + raise _StreamLimitExceeded() + except _StreamLimitExceeded: + raise RuntimeError( + "DataFog streaming response exceeded the scan budget; rejecting unscanned output" + ) from None + except Exception as exc: + # A failed upstream stream cannot be safely completed or replayed. + # Provider exceptions can contain private prompt/response text. + raise RuntimeError( + f"DataFog streaming response did not complete ({type(exc).__name__}); rejecting incomplete output" + ) from None + try: + texts = [ + "".join(value for _, _, value in group) for group in groups.values() + ] + session = _RedactionSession(texts) + changes = [] + for group, text in zip(groups.values(), texts, strict=True): + redacted, _ = self._process_content(text, session) + changes.extend( + (target, field, redacted if i == 0 else "") + for i, (target, field, _) in enumerate(group) ) - except Exception as exc: # noqa: BLE001 — fail policy decides + for target, field, value in changes: + _set(target, field, value) + except Exception as exc: self._handle_engine_error(exc) - return response + for chunk in chunks: + yield chunk + + +class _StreamLimitExceeded(Exception): + """Internal marker; stream-budget failures always reject output.""" diff --git a/datafog/main.py b/datafog/main.py index 0dc6f5d5..30fd1eed 100644 --- a/datafog/main.py +++ b/datafog/main.py @@ -15,12 +15,12 @@ from datafog._legacy_retirement import warn_legacy_surface from .config import OperationType +from .core import get_supported_entities from .engine import scan, scan_and_redact from .models.anonymizer import Anonymizer, AnonymizerType, HashType from .processing.text_processing.regex_annotator import RegexAnnotator logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) class DataFog: @@ -42,6 +42,7 @@ def __init__( hash_type: HashType = HashType.SHA256, anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, locales: List[str] | None = None, + hash_key: bytes | None = None, ): self.locales = locales self.regex_annotator = RegexAnnotator(locales=locales) @@ -59,6 +60,7 @@ def __init__( hash_type=hash_type, anonymizer_type=anonymizer_type ) self.hash_type = hash_type + self.hash_key = hash_key self.anonymizer_type = anonymizer_type self.logger = logging.getLogger(__name__) self.logger.info("Initializing lightweight DataFog class with regex engine") @@ -106,7 +108,15 @@ def run_text_pipeline_sync(self, str_list: List[str]) -> List: _start = _time.monotonic() try: self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: + if any( + op in self.operations + for op in ( + OperationType.SCAN, + OperationType.REDACT, + OperationType.REPLACE, + OperationType.HASH, + ) + ): annotated_text = [self.detect(text) for text in str_list] self.logger.info( @@ -190,8 +200,11 @@ def detect(self, text: str) -> dict: # Only pre-populate keys for labels active under the configured # locales so the default output shape matches v4.4.0 (no DE_* keys # unless German locale support is enabled). - result = {label: [] for label in RegexAnnotator.active_labels_for(self.locales)} legacy_map = {"DATE": "DOB", "ZIP_CODE": "ZIP"} + result = { + legacy_map.get(label, label): [] + for label in get_supported_entities(self.locales) + } for entity in scan_result.entities: label = legacy_map.get(entity.type, entity.type) result.setdefault(label, []).append(entity.text) @@ -239,6 +252,8 @@ def process( _start = _time.monotonic() + if method not in {"redact", "replace", "hash"}: + raise ValueError("method must be 'redact', 'replace', or 'hash'") annotations_dict = self.detect(text) result = {"original": text, "findings": annotations_dict} @@ -249,12 +264,18 @@ def process( "replace": "pseudonymize", "hash": "hash", } - strategy = strategy_map.get(method, "token") + strategy = strategy_map[method] redact_result = scan_and_redact( text=text, engine="regex", strategy=strategy, locales=self.locales, + hash_type=( + self.hash_type.value + if isinstance(self.hash_type, HashType) + else self.hash_type + ), + hash_key=self.hash_key, ) result["anonymized"] = redact_result.redacted_text @@ -298,6 +319,7 @@ class TextPIIAnnotator: """ def __init__(self, locales: List[str] | None = None): + self.locales = locales self.regex_annotator = RegexAnnotator(locales=locales) def run(self, text, output_path=None): @@ -312,7 +334,13 @@ def run(self, text, output_path=None): Dictionary mapping entity types to lists of found entities """ try: - annotated_text = self.regex_annotator.annotate(text) + result = scan(text=text, engine="regex", locales=self.locales) + legacy = {"DATE": "DOB", "ZIP_CODE": "ZIP"} + annotated_text = {} + for entity in result.entities: + annotated_text.setdefault( + legacy.get(entity.type, entity.type), [] + ).append(entity.text) # Optionally, output the results to a JSON file if output_path: diff --git a/datafog/main_lean.py b/datafog/main_lean.py deleted file mode 100644 index f67501e6..00000000 --- a/datafog/main_lean.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Historical shadow main module. - -This file is not the live DataFog 4.5 ``DataFog`` implementation. Use -``datafog/main.py`` for the current backward-compatible class and text/OCR -compatibility methods. This lean snapshot remains importable only as historical -reference until legacy cleanup can remove it safely. - -Lean main module for DataFog core functionality. - -This module contains the lightweight core classes for DataFog: -- DataFog: Main class for regex-based PII detection -- TextPIIAnnotator: Class for annotating PII in text using regex patterns - -These classes provide the core PII detection functionality without heavy dependencies. -""" - -import json -import logging -from typing import List - -from .config import OperationType -from .models.anonymizer import Anonymizer, AnonymizerType, HashType -from .processing.text_processing.regex_annotator import RegexAnnotator - -logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) - - -class DataFog: - """ - Lightweight main class for regex-based PII detection and anonymization. - - Handles text processing operations using fast regex patterns for PII detection. - For advanced features like OCR, spaCy, or Spark, install additional extras. - - Attributes: - regex_annotator: Core regex-based PII annotator. - operations: List of operations to perform. - anonymizer: Anonymizer for PII redaction, replacement, or hashing. - """ - - def __init__( - self, - operations: List[OperationType] = [OperationType.SCAN], - hash_type: HashType = HashType.SHA256, - anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, - ): - self.regex_annotator = RegexAnnotator() - self.operations: List[OperationType] = operations - self.anonymizer = Anonymizer( - hash_type=hash_type, anonymizer_type=anonymizer_type - ) - self.logger = logging.getLogger(__name__) - self.logger.info("Initializing lightweight DataFog class with regex engine") - self.logger.info(f"Operations: {operations}") - self.logger.info(f"Hash Type: {hash_type}") - self.logger.info(f"Anonymizer Type: {anonymizer_type}") - - def run_text_pipeline_sync(self, str_list: List[str]) -> List[str]: - """ - Run the text pipeline synchronously on a list of input text. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List[str]: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: - annotated_text = [] - - for text in str_list: - # Use regex annotator for core PII detection - annotations = self.regex_annotator.annotate(text) - annotated_text.append(annotations) - - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if any( - op in self.operations - for op in [ - OperationType.REDACT, - OperationType.REPLACE, - OperationType.HASH, - ] - ): - return [ - self.anonymizer.anonymize(text, annotations).anonymized_text - for text, annotations in zip( - str_list, annotated_text, strict=True - ) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return str_list - except Exception as e: - self.logger.error(f"Error in run_text_pipeline_sync: {str(e)}") - raise - - def detect(self, text: str) -> dict: - """ - Simple PII detection using regex patterns. - - Args: - text: Input text to scan for PII - - Returns: - Dictionary mapping entity types to lists of found entities - """ - return self.regex_annotator.annotate(text) - - def process( - self, text: str, anonymize: bool = False, method: str = "redact" - ) -> dict: - """ - Process text to detect and optionally anonymize PII. - - Args: - text: Input text to process - anonymize: Whether to anonymize detected PII - method: Anonymization method ('redact', 'replace', 'hash') - - Returns: - Dictionary with original text, anonymized text (if requested), and findings - """ - annotations = self.detect(text) - - result = {"original": text, "findings": annotations} - - if anonymize: - if method == "redact": - anonymizer_type = AnonymizerType.REDACT - elif method == "replace": - anonymizer_type = AnonymizerType.REPLACE - elif method == "hash": - anonymizer_type = AnonymizerType.HASH - else: - anonymizer_type = AnonymizerType.REDACT - - anonymized_result = self.anonymizer.anonymize( - text, annotations, anonymizer_type - ) - result["anonymized"] = anonymized_result.anonymized_text - - return result - - -class TextPIIAnnotator: - """ - Lightweight class for annotating PII in text using regex patterns. - - Provides functionality to detect and annotate Personally Identifiable Information (PII) - in text using fast regex patterns instead of heavy NLP models. - - Attributes: - regex_annotator: RegexAnnotator instance for text annotation. - """ - - def __init__(self): - self.regex_annotator = RegexAnnotator() - - def run(self, text, output_path=None): - """ - Run PII annotation on text using regex patterns. - - Args: - text: Input text to annotate - output_path: Optional path to save results as JSON - - Returns: - Dictionary mapping entity types to lists of found entities - """ - try: - annotated_text = self.regex_annotator.annotate(text) - - # Optionally, output the results to a JSON file - if output_path: - with open(output_path, "w") as f: - json.dump(annotated_text, f) - - return annotated_text - - except Exception as e: - logging.error(f"Error in TextPIIAnnotator.run: {str(e)}") - raise diff --git a/datafog/main_original.py b/datafog/main_original.py deleted file mode 100644 index 9594c824..00000000 --- a/datafog/main_original.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Historical shadow main module. - -This file is not the live DataFog 4.5 ``DataFog`` implementation. Use -``datafog/main.py`` for the current backward-compatible class and use the -optional OCR/Spark service modules directly when those surfaces are needed. -This original full-featured snapshot remains importable for legacy tests until -cleanup can remove it safely. - -Main module for DataFog. - -This module contains the core classes for DataFog: -- DataFog: Main class for running OCR and text processing pipelines. -- TextPIIAnnotator: Class for annotating PII in text. - -These classes provide high-level interfaces for image and text processing, -including OCR, PII detection, annotation, and anonymization. -""" - -import json -import logging -from typing import List - -from .config import OperationType -from .models.anonymizer import Anonymizer, AnonymizerType, HashType -from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator -from .services.image_service import ImageService -from .services.spark_service import SparkService -from .services.text_service import TextService - -logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) - - -class DataFog: - """ - Main class for running OCR and text processing pipelines. - - Handles image and text processing operations, including OCR, PII detection, and anonymization. - - Attributes: - image_service: Service for image processing and OCR. - text_service: Service for text processing and annotation. - spark_service: Optional Spark service for distributed processing. - operations: List of operations to perform. - anonymizer: Anonymizer for PII redaction, replacement, or hashing. - """ - - def __init__( - self, - image_service=None, - text_service=None, - spark_service=None, - operations: List[OperationType] = [OperationType.SCAN], - hash_type: HashType = HashType.SHA256, - anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, - ): - self.image_service = image_service or ImageService() - self.text_service = text_service or TextService() - self.spark_service: SparkService = spark_service - self.operations: List[OperationType] = operations - self.anonymizer = Anonymizer( - hash_type=hash_type, anonymizer_type=anonymizer_type - ) - self.logger = logging.getLogger(__name__) - self.logger.info( - "Initializing DataFog class with the following services and operations:" - ) - self.logger.info(f"Image Service: {type(self.image_service)}") - self.logger.info(f"Text Service: {type(self.text_service)}") - self.logger.info( - f"Spark Service: {type(self.spark_service) if self.spark_service else 'None'}" - ) - self.logger.info(f"Operations: {operations}") - self.logger.info(f"Hash Type: {hash_type}") - self.logger.info(f"Anonymizer Type: {anonymizer_type}") - - async def run_ocr_pipeline(self, image_urls: List[str]): - """ - Run the OCR pipeline asynchronously on a list of images provided via URL. - - This method performs optical character recognition (OCR) on the images specified by the URLs. - If PII annotation is enabled, it also annotates the extracted text for personally identifiable information. - If redaction, replacement, or hashing is enabled, it applies the corresponding anonymization. - - Args: - image_urls (List[str]): A list of URLs pointing to the images to be processed. - - Returns: - List: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the OCR or text processing. - """ - try: - extracted_text = await self.image_service.ocr_extract(image_urls) - self.logger.info(f"OCR extraction completed for {len(image_urls)} images.") - - return await self._process_text(extracted_text) - except Exception as e: - logging.error(f"Error in run_ocr_pipeline: {str(e)}") - return [f"Error: {str(e)}"] - - async def run_text_pipeline(self, str_list: List[str]): - """ - Run the text pipeline asynchronously on a list of input text. - - This method processes a list of text strings, potentially annotating them for personally - identifiable information (PII) and applying anonymization if enabled. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - return await self._process_text(str_list) - except Exception as e: - self.logger.error(f"Error in run_text_pipeline: {str(e)}") - raise - - async def _process_text(self, text_list: List[str]): - """ - Internal method to process text based on enabled operations. - """ - if OperationType.SCAN in self.operations: - annotated_text = await self.text_service.batch_annotate_text_async( - text_list - ) - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if OperationType.REDACT in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.REDACT - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - elif OperationType.REPLACE in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.REPLACE - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - elif OperationType.HASH in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.HASH - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return text_list - - def run_text_pipeline_sync(self, str_list: List[str]) -> List[str]: - """ - Run the text pipeline synchronously on a list of input text. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List[str]: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: - annotated_text = self.text_service.batch_annotate_text_sync(str_list) - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if any( - op in self.operations - for op in [ - OperationType.REDACT, - OperationType.REPLACE, - OperationType.HASH, - ] - ): - return [ - self.anonymizer.anonymize(text, annotations).anonymized_text - for text, annotations in zip( - str_list, annotated_text, strict=True - ) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return str_list - except Exception as e: - self.logger.error(f"Error in run_text_pipeline_sync: {str(e)}") - raise - - def _add_attributes(self, attributes: dict): - """ - Add multiple attributes to the DataFog instance. - - This private method allows for the dynamic addition of multiple attributes to the - DataFog instance. It iterates through the provided dictionary of attributes and - adds each key-value pair as an attribute. - - Args: - attributes (dict): A dictionary where keys are attribute names and values are - the corresponding attribute values to be added. - - Note: - This method is intended for internal use and may be used for extending the - functionality of the DataFog class dynamically. Care should be taken when - using this method to avoid overwriting existing attributes. - """ - for key, value in attributes.items(): - setattr(self, key, value) - - -class TextPIIAnnotator: - """ - Class for annotating PII in text. - - Provides functionality to detect and annotate Personally Identifiable Information (PII) in text. - - Attributes: - text_annotator: SpacyPIIAnnotator instance for text annotation. - spark_processor: Optional SparkService for distributed processing. - """ - - def __init__(self): - self.text_annotator = SpacyPIIAnnotator.create() - self.spark_processor: SparkService = None - - def run(self, text, output_path=None): - try: - annotated_text = self.text_annotator.annotate(text) - - # Optionally, output the results to a JSON file - if output_path: - with open(output_path, "w") as f: - json.dump(annotated_text, f) - - return annotated_text - - finally: - # Ensure Spark resources are released - if self.spark_processor: - self.spark_processor.stop() diff --git a/datafog/models/annotator.py b/datafog/models/annotator.py index 2f915ea4..489689b5 100644 --- a/datafog/models/annotator.py +++ b/datafog/models/annotator.py @@ -22,12 +22,12 @@ class AnnotatorRequest(BaseModel): text: str language: str - correlation_id: Optional[str] - score_threshold: Optional[float] - entities: Optional[List[EntityTypes]] - return_decision_process: Optional[bool] - ad_hoc_recognizers: Optional[List[PatternRecognizer]] - context: Optional[List[str]] + correlation_id: Optional[str] = None + score_threshold: Optional[float] = None + entities: Optional[List[EntityTypes]] = None + return_decision_process: Optional[bool] = None + ad_hoc_recognizers: Optional[List[PatternRecognizer]] = None + context: Optional[List[str]] = None class AnnotationResult(BaseModel): @@ -39,16 +39,19 @@ class AnnotationResult(BaseModel): start: int end: int - score: Optional[float] + score: Optional[float] = None entity_type: str - recognition_metadata: Optional[AnnotatorMetadata] + recognition_metadata: Optional[AnnotatorMetadata] = None - @field_validator("entity_type") + @field_validator("entity_type", mode="before") @classmethod def validate_entity_type(cls, v): - if v not in EntityTypes.__members__: - return "UNKNOWN" - return v + from datafog.engine import ALL_ENTITY_TYPES, CANONICAL_TYPE_MAP + + if isinstance(v, EntityTypes): + v = v.name + v = CANONICAL_TYPE_MAP.get(v, v) + return v if v in ALL_ENTITY_TYPES else "UNKNOWN" class AnalysisExplanation(BaseModel): @@ -60,14 +63,14 @@ class AnalysisExplanation(BaseModel): """ recognizer: str - pattern_name: Optional[str] - pattern: Optional[str] + pattern_name: Optional[str] = None + pattern: Optional[str] = None original_score: float score: float - textual_explanation: Optional[str] + textual_explanation: Optional[str] = None score_context_improvement: float supportive_context_word: str - validation_result: Optional[float] + validation_result: Optional[float] = None class AnnotationResultWithAnaysisExplanation(AnnotationResult): @@ -75,4 +78,4 @@ class AnnotationResultWithAnaysisExplanation(AnnotationResult): Extends AnnotationResult with detailed analysis explanation. """ - analysis_explanation: Optional[AnalysisExplanation] + analysis_explanation: Optional[AnalysisExplanation] = None diff --git a/datafog/models/anonymizer.py b/datafog/models/anonymizer.py index 6b81f5be..3ec36664 100644 --- a/datafog/models/anonymizer.py +++ b/datafog/models/anonymizer.py @@ -50,6 +50,33 @@ class Anonymizer(BaseModel): def __init__(self, **data): super().__init__(**data) + def _selected_annotations(self, text, annotations): + from datafog.engine import CANONICAL_TYPE_MAP + + selected = ( + None + if not self.entities + else { + CANONICAL_TYPE_MAP.get(entity.name, entity.name) + for entity in self.entities + } + ) + merged = [] + for annotation in sorted(annotations, key=lambda a: (a.start, -a.end)): + if not 0 <= annotation.start < annotation.end <= len(text): + raise ValueError( + "Annotation range must be nonempty and inside the text" + ) + if selected is not None and annotation.entity_type not in selected: + continue + if merged and annotation.start < merged[-1].end: + merged[-1] = merged[-1].model_copy( + update={"end": max(merged[-1].end, annotation.end)} + ) + else: + merged.append(annotation) + return reversed(merged) + def anonymize( self, text: str, annotations: List[AnnotationResult] ) -> AnonymizationResult: @@ -68,19 +95,18 @@ def replace_pii( ) -> AnonymizationResult: """Replace PII in text with anonymized values.""" replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if not self.entities or annotation.entity_type in self.entities: - replacement = self._generate_replacement( - text[annotation.start : annotation.end], annotation.entity_type - ) - replacements.append( - { - "original": text[annotation.start : annotation.end], - "replacement": replacement, - "entity_type": annotation.entity_type, - } - ) - text = text[: annotation.start] + replacement + text[annotation.end :] + for annotation in self._selected_annotations(text, annotations): + replacement = self._generate_replacement( + text[annotation.start : annotation.end], annotation.entity_type + ) + replacements.append( + { + "original": text[annotation.start : annotation.end], + "replacement": replacement, + "entity_type": annotation.entity_type, + } + ) + text = text[: annotation.start] + replacement + text[annotation.end :] return AnonymizationResult( anonymized_text=text, anonymized_entities=replacements @@ -88,13 +114,13 @@ def replace_pii( def _generate_replacement(self, original: str, entity_type: EntityTypes) -> str: """Generate a replacement for the given entity.""" - if entity_type == EntityTypes.PERSON: + if entity_type == "PERSON": return f"[PERSON_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.ORGANIZATION: + elif entity_type == "ORGANIZATION": return f"[ORGANIZATION_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.LOCATION: + elif entity_type == "LOCATION": return f"[LOCATION_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.DATE: + elif entity_type == "DATE": return "[REDACTED_DATE]" else: return f"[{entity_type}_{secrets.token_hex(4).upper()}]" @@ -104,10 +130,7 @@ def hash_pii( ) -> AnonymizationResult: """Hash PII in text.""" replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if self.entities and annotation.entity_type not in self.entities: - continue - + for annotation in self._selected_annotations(text, annotations): start, end = annotation.start, annotation.end original = text[start:end] replacement = self._hash_text(original) @@ -139,10 +162,7 @@ def redact_pii( self, text: str, annotations: List[AnnotationResult] ) -> AnonymizationResult: replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if self.entities and annotation.entity_type not in self.entities: - continue - + for annotation in self._selected_annotations(text, annotations): start, end = annotation.start, annotation.end original = text[start:end] replacement = "[REDACTED]" diff --git a/datafog/models/common.py b/datafog/models/common.py index f01d6651..1f72d4aa 100644 --- a/datafog/models/common.py +++ b/datafog/models/common.py @@ -40,8 +40,8 @@ class PatternRecognizer(BaseModel): name: str supported_language: str patterns: List[Pattern] - deny_list: Optional[List[str]] - context: Optional[List[str]] + deny_list: Optional[List[str]] = None + context: Optional[List[str]] = None supported_entity: str diff --git a/datafog/native.py b/datafog/native.py new file mode 100644 index 00000000..44b6fb29 --- /dev/null +++ b/datafog/native.py @@ -0,0 +1,66 @@ +"""Native detection API, re-exported from the installed ``datafog-core``. + +``datafog`` selects the native engine by default, so most callers never need +this module. Import it to reach the native schema directly: byte and code-point +ranges, transformation records, provider-backed tokenization, structured scans, +the embedded model, and ``capabilities()``. + +The names are re-exports, not a second implementation. Importing ``datafog`` +alone does not load them. + +Experimental: this module may change outside the deprecation policy until 5.0. +""" + +from __future__ import annotations + +import importlib +import importlib.metadata +from typing import Any + +_MODULE = "datafog_core" + +__all__ = [ + "EmbeddedCnnDetector", + "Finding", + "ModelScanResult", + "capabilities", + "contract_version", + "scan", + "scan_and_transform", + "scan_structured", + "transform", + "transform_structured", + "version", +] + + +def _core() -> Any: + try: + return importlib.import_module(_MODULE) + except ImportError as exc: # pragma: no cover - base install always has it + raise ImportError( + "datafog.native requires datafog-core. Install with: " + 'pip install "datafog[rust]"' + ) from exc + + +def __getattr__(name: str) -> Any: + if name in __all__: + return getattr(_core(), name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def version() -> str: + """Installed Core version, for bug reports and reproducibility locks.""" + _core() + try: + return importlib.metadata.version("datafog-core") + except importlib.metadata.PackageNotFoundError as exc: + raise ImportError( + "datafog-core distribution metadata is missing; reinstall datafog[rust]" + ) from exc + + +def contract_version() -> int: + """Capability contract the installed Core speaks. Currently 1.""" + return int(_core().capabilities()["contract_version"]) diff --git a/datafog/processing/spark_processing/pyspark_udfs.py b/datafog/processing/spark_processing/pyspark_udfs.py index aaa53ece..bb9b8c18 100644 --- a/datafog/processing/spark_processing/pyspark_udfs.py +++ b/datafog/processing/spark_processing/pyspark_udfs.py @@ -28,8 +28,9 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]: if text: if len(text) > MAXIMAL_STRING_SIZE: - # Cut the strings for required sizes - text = text[:MAXIMAL_STRING_SIZE] + raise ValueError( + "Spark PII input exceeds the maximum supported string length" + ) nlp = broadcasted_nlp.value doc = nlp(text) @@ -38,8 +39,15 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]: _label: [] for _label in PII_ANNOTATION_LABELS } for ent in doc.ents: - # Add entities from extracted values - classified_entities[ent.label_].append(ent.text) + label = { + "PERSON": "PER", + "GPE": "LOC", + "NORP": "NRP", + "DATE": "DATE_TIME", + "TIME": "DATE_TIME", + }.get(ent.label_, ent.label_) + if label in classified_entities: + classified_entities[label].append(ent.text) return [_ent for _ent in classified_entities.values()] else: diff --git a/datafog/processing/text_processing/gliner_annotator.py b/datafog/processing/text_processing/gliner_annotator.py index 5a195f89..b871cf63 100644 --- a/datafog/processing/text_processing/gliner_annotator.py +++ b/datafog/processing/text_processing/gliner_annotator.py @@ -10,6 +10,8 @@ from pydantic import BaseModel, ConfigDict +from .regex_annotator.regex_annotator import AnnotationResult, Span + # Default entity types for PII detection using GLiNER # These can be customized based on specific use cases DEFAULT_PII_ENTITIES = [ @@ -92,55 +94,37 @@ def create( f"datafog download-model {model_name} --engine gliner" ) from e - def annotate(self, text: str) -> Dict[str, List[str]]: - """ - Annotate text for PII entities using GLiNER. - - Args: - text: Text to analyze for PII entities - - Returns: - Dictionary mapping entity types to lists of detected entities - """ - try: - if not text: - return { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } - - if len(text) > MAXIMAL_STRING_SIZE: - text = text[:MAXIMAL_STRING_SIZE] - logging.warning(f"Text truncated to {MAXIMAL_STRING_SIZE} characters") - - # Predict entities using GLiNER - entities = self.model.predict_entities(text, self.entity_types) - - # Organize results by entity type - classified_entities: Dict[str, List[str]] = { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } - + def annotate_with_spans(self, text: str): + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text) > MAXIMAL_STRING_SIZE: + raise ValueError( + f"GLiNER input exceeds {MAXIMAL_STRING_SIZE} characters; split the document before scanning" + ) + classified_entities = { + label.upper().replace(" ", "_"): [] for label in self.entity_types + } + spans = [] + if text: + try: + entities = self.model.predict_entities(text, self.entity_types) + except Exception as exc: + raise RuntimeError("GLiNER annotation failed") from exc for entity in entities: - entity_label = entity["label"].upper().replace(" ", "_") - entity_text = entity["text"] + label = entity["label"].upper().replace(" ", "_") + classified_entities.setdefault(label, []).append(entity["text"]) + spans.append( + Span( + label=label, + start=entity["start"], + end=entity["end"], + text=entity["text"], + ) + ) + return classified_entities, AnnotationResult(text=text, spans=spans) - if entity_label in classified_entities: - classified_entities[entity_label].append(entity_text) - else: - # Handle cases where GLiNER returns entity types not in our list - classified_entities[entity_label] = [entity_text] - - return classified_entities - - except Exception as e: - logging.error(f"Error processing text with GLiNER: {str(e)}") - # Return empty annotations in case of error - return { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } + def annotate(self, text: str) -> Dict[str, List[str]]: + return self.annotate_with_spans(text)[0] def set_entity_types(self, entity_types: List[str]) -> None: """ diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py index 44dfa2e0..03641e82 100644 --- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py +++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py @@ -40,11 +40,14 @@ class RegexAnnotator: ] LABELS = BASE_LABELS + GERMAN_LABELS DEFAULT_LABELS = BASE_LABELS - SUPPORTED_LOCALES = {"de", "de-de", "de_de"} + SUPPORTED_LOCALES = {"de", "de-de", "de_de", "en", "en-us", "fr"} LOCALE_LABELS = { "de": GERMAN_LABELS, "de-de": GERMAN_LABELS, "de_de": GERMAN_LABELS, + "en": [], + "en-us": [], + "fr": [], } def __init__( @@ -63,14 +66,13 @@ def __init__( # Note: This is broader than the spec to catch more potential emails "EMAIL": re.compile( r""" - (? (? (? (? (? tuple[str, . locales = [locales] normalized = [] for locale in locales: + if not isinstance(locale, str) or not locale.strip(): + raise ValueError("locales must contain nonempty locale identifiers") value = locale.strip().lower() - if not value: - continue if value not in cls.SUPPORTED_LOCALES: allowed = ", ".join(sorted(cls.SUPPORTED_LOCALES)) raise ValueError(f"locale must be one of: {allowed}") @@ -360,15 +362,27 @@ def active_labels_for( active.update(label.strip().upper() for label in enabled_labels) return [label for label in cls.LABELS if label in active] - @staticmethod - def _match_text(match: Match[str]) -> str: - return match.groupdict().get("value") or match.group() + @classmethod + def _match_text(cls, match: Match[str]) -> str: + start, end = cls._match_span(match) + return match.string[start:end] @staticmethod def _match_span(match: Match[str]) -> tuple[int, int]: if "value" in match.groupdict() and match.group("value") is not None: return match.start("value"), match.end("value") - return match.start(), match.end() + start, end = match.start(), match.end() + value = match.group() + if "@" in value and any(char.isalnum() for char in value[: value.rfind("@")]): + local = value[: value.rfind("@")] + # Query/assignment separators and prose wrappers are context, + # not part of the mailbox. Consume the whole lexical run before + # trimming so failed candidates never retry every suffix. + boundary = max((local.rfind(char) for char in "&?=|/"), default=-1) + start += boundary + 1 + while start < end and match.string[start] in ".!?'*{}`": + start += 1 + return start, end @classmethod def create(cls, **kwargs) -> "RegexAnnotator": diff --git a/datafog/processing/text_processing/spacy_pii_annotator.py b/datafog/processing/text_processing/spacy_pii_annotator.py index d2d1d765..1cb2cbd2 100644 --- a/datafog/processing/text_processing/spacy_pii_annotator.py +++ b/datafog/processing/text_processing/spacy_pii_annotator.py @@ -1,8 +1,9 @@ -import logging from typing import Any, Dict, List from pydantic import BaseModel, ConfigDict +from .regex_annotator.regex_annotator import AnnotationResult, Span + PII_ANNOTATION_LABELS = [ "CARDINAL", "DATE", @@ -28,7 +29,7 @@ class SpacyPIIAnnotator(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) + model_config = ConfigDict(arbitrary_types_allowed=True, protected_namespaces=()) nlp: Any model_name: str = DEFAULT_SPACY_MODEL @@ -53,22 +54,32 @@ def create(cls, model_name: str = DEFAULT_SPACY_MODEL) -> "SpacyPIIAnnotator": return cls(nlp=nlp, model_name=model_name) - def annotate(self, text: str) -> Dict[str, List[str]]: - try: - if not text: - return {label: [] for label in PII_ANNOTATION_LABELS} - if len(text) > MAXIMAL_STRING_SIZE: - text = text[:MAXIMAL_STRING_SIZE] - doc = self.nlp(text) - classified_entities: Dict[str, List[str]] = { - label: [] for label in PII_ANNOTATION_LABELS - } + def annotate_with_spans(self, text: str): + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text) > MAXIMAL_STRING_SIZE: + raise ValueError( + f"spaCy input exceeds {MAXIMAL_STRING_SIZE} characters; split the document before scanning" + ) + classified_entities = {label: [] for label in PII_ANNOTATION_LABELS} + spans = [] + if text: + try: + doc = self.nlp(text) + except Exception as exc: + raise RuntimeError("spaCy annotation failed") from exc for ent in doc.ents: if ent.label_ in classified_entities: classified_entities[ent.label_].append(ent.text) - return classified_entities - except Exception as e: - logging.error(f"Error processing text for PII annotations: {str(e)}") - return { - label: [] for label in PII_ANNOTATION_LABELS - } # Return empty annotations in case of error + spans.append( + Span( + label=ent.label_, + start=ent.start_char, + end=ent.end_char, + text=ent.text, + ) + ) + return classified_entities, AnnotationResult(text=text, spans=spans) + + def annotate(self, text: str) -> Dict[str, List[str]]: + return self.annotate_with_spans(text)[0] diff --git a/datafog/services/text_service.py b/datafog/services/text_service.py index 6b80b26d..a7826a7e 100644 --- a/datafog/services/text_service.py +++ b/datafog/services/text_service.py @@ -1,8 +1,8 @@ """Lean text processing service for PII annotation. -Provides synchronous and asynchronous methods for annotating text with personally -identifiable information (PII) using regex patterns. Supports chunking long texts -and batch processing. SpaCy integration available as optional extra. +Provides synchronous, asynchronous and batch access to the shared PII engine. +Structured PII spans are scanned across complete documents; optional NLP engines +add spans with their original character coordinates. """ import asyncio @@ -31,8 +31,8 @@ class TextService: Lightweight service for annotating text with PII entities using regex patterns. This service provides methods to detect and annotate personally identifiable information (PII) - in text using fast regex patterns. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. + in text through the shared engine. Complete-document structured scanning preserves + entities across boundaries, and optional NLP engines add canonical entity spans. For advanced NLP-based detection using spaCy, install the 'nlp' extra: pip install datafog[nlp] @@ -49,7 +49,8 @@ def __init__( Initialize the TextService with specified chunk length and annotation engine. Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. + text_chunk_length: Positive compatibility setting. Structured detection scans the + complete document; the shared engine bounds native model chunks separately. engine: The annotation engine to use. Options are: - "regex": (Default) Use RegexAnnotator for fast pattern-based entity detection - "spacy": Use SpacyPIIAnnotator for NLP-based entity detection (requires nlp extra) @@ -61,10 +62,17 @@ def __init__( German structured identifiers. Raises: - AssertionError: If an invalid engine type is provided + ValueError: If the engine or text_chunk_length is invalid ImportError: If spacy/gliner engine is requested but corresponding extra is not installed """ - assert engine in {"regex", "spacy", "gliner", "auto", "smart"}, "Invalid engine" + if engine not in {"regex", "spacy", "gliner", "auto", "smart"}: + raise ValueError("engine must be one of: regex, spacy, gliner, auto, smart") + if ( + not isinstance(text_chunk_length, int) + or isinstance(text_chunk_length, bool) + or text_chunk_length <= 0 + ): + raise ValueError("text_chunk_length must be a positive integer") self.engine = engine self.text_chunk_length = text_chunk_length self.gliner_model = gliner_model @@ -379,17 +387,32 @@ def annotate_text_sync( _start = _time.monotonic() - if len(text) <= self.text_chunk_length: - # Single chunk processing - result = self._annotate_single_chunk(text, structured) + from datafog.engine import scan + + # Structured PII is scanned as a complete document. Fixed-size chunks + # lose entities at their boundaries; native model work chunks separately. + options = { + "engine": "smart" if self.engine == "auto" else self.engine, + "locales": self.locales, + } + if self.gliner_model != "urchade/gliner_multi_pii-v1": + options["gliner_model"] = self.gliner_model + entities = scan(text, **options).entities + if structured: + SpanClass = _get_span_class() + result = [ + SpanClass( + label=entity.type, + start=entity.start, + end=entity.end, + text=entity.text, + ) + for entity in entities + ] else: - # Multi-chunk processing - chunks = self._chunk_text(text) - - if structured: - result = self._annotate_multiple_chunks_structured(chunks) - else: - result = self._annotate_multiple_chunks_dict(chunks) + result = {} + for entity in entities: + result.setdefault(entity.type, []).append(entity.text) try: from datafog.telemetry import ( diff --git a/datafog/services/text_service_lean.py b/datafog/services/text_service_lean.py deleted file mode 100644 index 1262db3f..00000000 --- a/datafog/services/text_service_lean.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Historical shadow text service module. - -This file is not the live DataFog 4.5 text service. Use -``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER, -auto, and smart engine boundary. This lean snapshot remains importable only as -historical reference until legacy cleanup can remove it safely. - -Lean text processing service for PII annotation. - -Provides synchronous and asynchronous methods for annotating text with personally -identifiable information (PII) using regex patterns. Supports chunking long texts -and batch processing. SpaCy integration available as optional extra. -""" - -import asyncio -from typing import Dict, List, Union - -from datafog.processing.text_processing.regex_annotator.regex_annotator import ( - RegexAnnotator, - Span, -) - - -class TextService: - """ - Lightweight service for annotating text with PII entities using regex patterns. - - This service provides methods to detect and annotate personally identifiable information (PII) - in text using fast regex patterns. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. - - For advanced NLP-based detection using spaCy, install the 'nlp' extra: - pip install datafog[nlp] - """ - - def __init__(self, text_chunk_length: int = 1000, engine: str = "regex"): - """ - Initialize the TextService with specified chunk length and annotation engine. - - Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. - engine: The annotation engine to use. Options are: - - "regex": (Default) Use RegexAnnotator for fast pattern-based entity detection - - "spacy": Use SpacyPIIAnnotator for NLP-based entity detection (requires nlp extra) - - "auto": Try RegexAnnotator first and fall back to SpacyPIIAnnotator if no entities found - - Raises: - AssertionError: If an invalid engine type is provided - ImportError: If spacy engine is requested but nlp extra is not installed - """ - assert engine in {"regex", "spacy", "auto"}, "Invalid engine" - self.engine = engine - self.regex_annotator = RegexAnnotator() - self.text_chunk_length = text_chunk_length - - # Only initialize spacy if needed and available - self.spacy_annotator = None - if engine in {"spacy", "auto"}: - try: - from datafog.processing.text_processing.spacy_pii_annotator import ( - SpacyPIIAnnotator, - ) - - self.spacy_annotator = SpacyPIIAnnotator.create() - except ImportError: - if engine == "spacy": - raise ImportError( - "SpaCy engine requires additional dependencies. " - "Install with: pip install datafog[nlp]" - ) - # For auto mode, just continue with regex only - self.spacy_annotator = None - - def _chunk_text(self, text: str) -> List[str]: - """Split the text into chunks of specified length.""" - return [ - text[i : i + self.text_chunk_length] - for i in range(0, len(text), self.text_chunk_length) - ] - - def _combine_annotations( - self, chunk_annotations: List[Dict[str, List[str]]] - ) -> Dict[str, List[str]]: - """Combine annotations from multiple chunks.""" - combined = {} - for annotations in chunk_annotations: - for entity_type, entities in annotations.items(): - if entity_type not in combined: - combined[entity_type] = [] - combined[entity_type].extend(entities) - return combined - - def annotate_text_sync( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text synchronously for PII entities. - - Args: - text: The text to annotate - structured: If True, return structured Span objects. If False, return dict format. - - Returns: - Dictionary mapping entity types to lists of entities, or list of Span objects - """ - if len(text) <= self.text_chunk_length: - # Single chunk processing - if self.engine == "regex": - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return self.regex_annotator.annotate(text) - elif self.engine == "spacy": - if self.spacy_annotator is None: - raise ImportError( - "SpaCy engine not available. Install with: pip install datafog[nlp]" - ) - return self.spacy_annotator.annotate(text) - elif self.engine == "auto": - # Try regex first - regex_result = self.regex_annotator.annotate(text) - - # Check if regex found any entities - if any(entities for entities in regex_result.values()): - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return regex_result - - # Fall back to spacy if available - if self.spacy_annotator is not None: - return self.spacy_annotator.annotate(text) - - # Return regex result even if empty - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return regex_result - else: - # Multi-chunk processing - chunks = self._chunk_text(text) - chunk_annotations = [] - - for chunk in chunks: - chunk_result = self.annotate_text_sync(chunk, structured=False) - chunk_annotations.append(chunk_result) - - if structured: - # For structured output with chunking, we need to recalculate positions - # This is more complex, so for now return dict format - return self._combine_annotations(chunk_annotations) - - return self._combine_annotations(chunk_annotations) - - async def annotate_text_async( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text asynchronously for PII entities. - - Args: - text: The text to annotate - structured: If True, return structured Span objects. If False, return dict format. - - Returns: - Dictionary mapping entity types to lists of entities, or list of Span objects - """ - # For regex processing, we can just run synchronously since it's fast - return self.annotate_text_sync(text, structured) - - def batch_annotate_text_sync(self, texts: List[str]) -> List[Dict[str, List[str]]]: - """ - Annotate multiple texts synchronously. - - Args: - texts: List of texts to annotate - - Returns: - List of annotation dictionaries, one per input text - """ - return [self.annotate_text_sync(text) for text in texts] - - async def batch_annotate_text_async( - self, texts: List[str] - ) -> List[Dict[str, List[str]]]: - """ - Annotate multiple texts asynchronously. - - Args: - texts: List of texts to annotate - - Returns: - List of annotation dictionaries, one per input text - """ - # For better performance with many texts, we can process them concurrently - tasks = [self.annotate_text_async(text) for text in texts] - return await asyncio.gather(*tasks) diff --git a/datafog/services/text_service_original.py b/datafog/services/text_service_original.py deleted file mode 100644 index 8ad576a9..00000000 --- a/datafog/services/text_service_original.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Historical shadow text service module. - -This file is not the live DataFog 4.5 text service. Use -``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER, -auto, and smart engine boundary. This original snapshot remains importable for -legacy tests until cleanup can remove it safely. - -Text processing service for PII annotation. - -Provides synchronous and asynchronous methods for annotating text with personally identifiable information (PII) using SpaCy or regex patterns. Supports chunking long texts and batch processing. -""" - -import asyncio -from typing import Dict, List, Union - -from datafog.processing.text_processing.regex_annotator.regex_annotator import ( - RegexAnnotator, - Span, -) -from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator - - -class TextService: - """ - Service for annotating text with PII entities. - - This service provides methods to detect and annotate personally identifiable information (PII) - in text using different annotation engines. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. - """ - - def __init__(self, text_chunk_length: int = 1000, engine: str = "auto"): - """ - Initialize the TextService with specified chunk length and annotation engine. - - Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. - engine: The annotation engine to use. Options are: - - "regex": Use only the RegexAnnotator for pattern-based entity detection - - "spacy": Use only the SpacyPIIAnnotator for NLP-based entity detection - - "auto": (Default) Try RegexAnnotator first and fall back to SpacyPIIAnnotator if no entities are found - - Raises: - AssertionError: If an invalid engine type is provided - """ - assert engine in {"regex", "spacy", "auto"}, "Invalid engine" - self.engine = engine - self.spacy_annotator = SpacyPIIAnnotator.create() - self.regex_annotator = RegexAnnotator() - self.text_chunk_length = text_chunk_length - - def _chunk_text(self, text: str) -> List[str]: - """Split the text into chunks of specified length.""" - return [ - text[i : i + self.text_chunk_length] - for i in range(0, len(text), self.text_chunk_length) - ] - - def _combine_annotations( - self, annotations: List[Dict[str, List[str]]] - ) -> Dict[str, List[str]]: - """Combine annotations from multiple chunks.""" - combined: Dict[str, List[str]] = {} - for annotation in annotations: - for key, value in annotation.items(): - if key not in combined: - combined[key] = [] - combined[key].extend(value) - return combined - - def _annotate_with_engine( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text using the selected engine based on the engine parameter. - - This method implements the engine selection logic: - - For "regex" mode: Uses only the RegexAnnotator - - For "spacy" mode: Uses only the SpacyPIIAnnotator - - For "auto" mode: Tries RegexAnnotator first and falls back to SpacyPIIAnnotator if no entities are found - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary of annotations by entity type where keys are entity types (e.g., "EMAIL", "PERSON", "ORG") - and values are lists of detected entities of that type - If structured=True: List of Span objects with entity information - """ - if structured: - # Handle structured output mode - if self.engine == "regex": - _, annotation_result = self.regex_annotator.annotate_with_spans(text) - return annotation_result.spans - elif self.engine == "spacy": - # For spaCy, we need to convert the dictionary format to spans - spacy_dict = self.spacy_annotator.annotate(text) - spacy_spans: List[Span] = [] - for label, entities in spacy_dict.items(): - for entity in entities: - # Find the start and end positions of the entity in the text - start = text.find(entity) - if start >= 0: - end = start + len(entity) - span = Span(start=start, end=end, label=label, text=entity) - spacy_spans.append(span) - return spacy_spans - else: # "auto" mode - # Try regex first - _, annotation_result = self.regex_annotator.annotate_with_spans(text) - if annotation_result.spans: - return annotation_result.spans - - # If regex found nothing, fall back to spaCy - spacy_dict = self.spacy_annotator.annotate(text) - auto_spans: List[Span] = [] - for label, entities in spacy_dict.items(): - for entity in entities: - # Find the start and end positions of the entity in the text - start = text.find(entity) - if start >= 0: - end = start + len(entity) - span = Span(start=start, end=end, label=label, text=entity) - auto_spans.append(span) - return auto_spans - else: - # Handle legacy dictionary output mode - if self.engine == "regex": - return self.regex_annotator.annotate(text) - elif self.engine == "spacy": - return self.spacy_annotator.annotate(text) - else: # auto mode - # Try regex first - regex_dict = self.regex_annotator.annotate(text) - - # Check if any VALID entities were found (ignore empty strings) - has_entities = any( - any(entity.strip() for entity in entities) - for entities in regex_dict.values() - ) - - # If regex found entities, return those results - if has_entities: - return regex_dict - - # Otherwise, fall back to spaCy - return self.spacy_annotator.annotate(text) - - def annotate_text_sync( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Synchronously annotate a text string. - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary mapping entity types to lists of strings - If structured=True: List of Span objects with entity information - """ - if not text: - return [] if structured else {} - - chunks = self._chunk_text(text) - - if structured: - # Handle structured output mode - all_spans: List[Span] = [] - chunk_offset = 0 # Track the offset for each chunk in the original text - - for chunk in chunks: - # Process each chunk and get spans - chunk_spans = self._annotate_with_engine(chunk, structured=True) - if not isinstance(chunk_spans, list): - continue # Skip if not a list of spans - - # Adjust span positions based on chunk offset in the original text - for span in chunk_spans: - if not isinstance(span, Span): - continue # Skip if not a Span object - span.start += chunk_offset - span.end += chunk_offset - # Verify the span text matches the text at the adjusted position - if span.start < len(text) and span.end <= len(text): - span.text = text[span.start : span.end] - all_spans.append(span) - - # Update offset for the next chunk - chunk_offset += len(chunk) - - print(f"Done processing {text.split()[0]}") - return all_spans - else: - # Handle legacy dictionary output mode - annotations: List[Dict[str, List[str]]] = [] - for chunk in chunks: - res = self._annotate_with_engine(chunk) - if isinstance(res, dict): - annotations.append(res) - combined = self._combine_annotations(annotations) - print(f"Done processing {text.split()[0]}") - return combined - - def batch_annotate_text_sync( - self, texts: List[str], structured: bool = False - ) -> Dict[str, Union[Dict[str, List[str]], List[Span]]]: - """ - Synchronously annotate a list of text input. - - Args: - texts: List of text strings to annotate - structured: If True, return structured output (list of Span objects) for each text - - Returns: - Dictionary mapping each input text to its annotation result - """ - results = [ - self.annotate_text_sync(text, structured=structured) for text in texts - ] - return dict(zip(texts, results, strict=True)) - - async def annotate_text_async( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Asynchronously annotate a text string. - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary mapping entity types to lists of strings - If structured=True: List of Span objects with entity information - """ - if not text: - return [] if structured else {} - - chunks = self._chunk_text(text) - - if structured: - # Handle structured output mode asynchronously - all_spans: List[Span] = [] - chunk_offset = 0 # Track the offset for each chunk in the original text - - for chunk in chunks: - # We can't easily parallelize this due to the need to track offsets sequentially - # In a production environment, you might want a more sophisticated approach - chunk_spans = self._annotate_with_engine(chunk, structured=True) - if not isinstance(chunk_spans, list): - continue # Skip if not a list of spans - - # Adjust span positions based on chunk offset in the original text - for span in chunk_spans: - if not isinstance(span, Span): - continue # Skip if not a Span object - span.start += chunk_offset - span.end += chunk_offset - # Verify the span text matches the text at the adjusted position - if span.start < len(text) and span.end <= len(text): - span.text = text[span.start : span.end] - all_spans.append(span) - - # Update offset for the next chunk - chunk_offset += len(chunk) - - return all_spans - else: - # Handle legacy dictionary output mode asynchronously - tasks = [ - asyncio.to_thread(self._annotate_with_engine, chunk) for chunk in chunks - ] - results = await asyncio.gather(*tasks) - annotations: List[Dict[str, List[str]]] = [ - r for r in results if isinstance(r, dict) - ] - return self._combine_annotations(annotations) - - async def batch_annotate_text_async( - self, texts: List[str], structured: bool = False - ) -> Dict[str, Union[Dict[str, List[str]], List[Span]]]: - """ - Asynchronously annotate a list of text input. - - Args: - texts: List of text strings to annotate - structured: If True, return structured output (list of Span objects) for each text - - Returns: - Dictionary mapping each input text to its annotation result - """ - tasks = [ - self.annotate_text_async(text, structured=structured) for text in texts - ] - results = await asyncio.gather(*tasks) - return dict(zip(texts, results, strict=True)) diff --git a/docs/cli.rst b/docs/cli.rst index b07a080a..b704508b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -24,7 +24,8 @@ dependencies are: Spark/distributed workflows are Python SDK surfaces rather than first-path CLI commands. Spark support is also deprecated in 4.9 for removal in 5.0. Install ``datafog[distributed]`` when using ``SparkService`` during 4.x. Users needing -OCR/Spark after the cutover can remain on the final 4.x release. See +OCR/Spark after the cutover should pin ``datafog==4.10.0``, the last supported +version. See :doc:`retirement` and :doc:`optional-surfaces` and the :download:`4.9 migration guide `. diff --git a/docs/important-concepts.rst b/docs/important-concepts.rst index 239471bc..4f48c39d 100644 --- a/docs/important-concepts.rst +++ b/docs/important-concepts.rst @@ -39,7 +39,7 @@ Services ``TextService`` remains available. ``ImageService`` and ``SparkService`` are optional legacy services, deprecated in 4.9 for removal in 5.0. They remain functional throughout 4.9; users requiring them after the cutover can remain on -the final 4.x release. See :doc:`optional-surfaces`. +:doc:`retirement`. Existing services: diff --git a/docs/index.rst b/docs/index.rst index df61a539..bb9b3d9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,13 +11,30 @@ Start with :doc:`getting-started` if you want the shortest route from install to scanning text. The roadmap and historical planning pages remain available, but the live user docs are the first path for current text APIs. +4.10.0 native default +===================== + +DataFog 4.10.0 turns the native engine on by default. ``pip install datafog`` +now installs ``datafog-core``, and detection uses it whenever it imports; the +pure-Python engine remains the fallback and ``DATAFOG_BACKEND=python`` forces +it. No public API is removed in 4.10.0 and no existing call changes shape — the +``backend`` keyword is still accepted, and now also accepts ``auto``. + +:download:`Complete 4.10 migration guide `. + +Prose ``PERSON`` and ``ADDRESS`` detection now comes from an embedded model, so +names and addresses no longer need spaCy or a model download. The native engine +adds credential, account and German detectors. Reviewed differences from the +Python engine are recorded in ``tests/contracts/rust-1.2.0.json`` and the +migration guide below. + 4.9.0 migration bridge ====================== -DataFog 4.9.0 is the migration bridge to the planned Rust-backed 5.0 API. -Upgrade the base package with ``python -m pip install --upgrade datafog==4.9.0``; -the native dependency remains optional. The ``rust`` extra requires -``datafog-core>=0.4.0,<0.5`` and capability contract version 1. +DataFog 4.9.0 was the migration bridge to the Rust-backed 5.0 API. It added the +explicit ``backend="rust"`` opt-in, the ``datafog.compat.v4`` facade and the +``datafog.v5`` preview. 4.10.0 built on it by making the native path the +default. 4.9 preserves existing imports, result classes, Python detection defaults, and redaction behavior. It adds explicit experimental Rust detection through @@ -28,7 +45,7 @@ Core schema preview. See :doc:`python-sdk` and the DataFog 4.9.0 deprecates ``detect()``/``process()``, OCR, and Spark for removal in 5.0. The earlier promise to retain ``detect()``/``process()`` throughout 5.x is revised. OCR/Spark remain functional in 4.9; users needing them after the -cutover can remain on the final 4.x release. See :doc:`optional-surfaces`. +cutover can pin :doc:`retirement`. Use DataFog =========== @@ -42,6 +59,7 @@ Use DataFog cli optional-surfaces important-concepts + retirement Reference ========= diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md new file mode 100644 index 00000000..6ea5db00 --- /dev/null +++ b/docs/migration-4.10.md @@ -0,0 +1,109 @@ +# Migrating to DataFog 4.10.0 + +4.10.0 turns on the native engine. Nothing is removed; most callers change +nothing at all. This page covers what a version number cannot: what you will +notice, and what to do if you do not want it. + +```bash +python -m pip install --upgrade "datafog==4.10.0" +``` + +`datafog-core` is now a normal dependency, so this pulls a platform wheel. On +Linux (glibc or musl, x86_64/aarch64/s390x/ppc64le), macOS and Windows +(amd64/arm64) that is a wheel. Elsewhere it builds from source and needs Rust +1.88+. If Core is missing at runtime, detection falls back to the Python engine +and warns once instead of failing. + +## What changes + +`backend` defaults to `auto`. `auto` selects the native engine when Core +imports, and the Python engine otherwise. `backend="python"` and +`backend="rust"` work exactly as before. `DATAFOG_BACKEND` sets the default +without touching code: + +```bash +DATAFOG_BACKEND=python # always the pure-Python engine, no warning +DATAFOG_BACKEND=auto # the default +``` + +`auto` never picks the native engine for `engine="spacy"`, `"gliner"` or +`"smart"`, because the native path serves `engine="regex"` plus embedded-model +findings. Those engines still run on the Python ML stack and still need their +extras. + +## Detection differences + +The native engine finds things the Python regex engine never did: `API_KEY`, +`BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, +`US_ROUTING_NUMBER`, `NPI`, `UUID`, and the German identifier set behind the +`de` locale. Credential detectors ship enabled by default; German and `UUID` +stay opt-in. + +These differences go the other way and are recorded, not hidden, in +[tests/contracts/rust-1.4.0.json](../tests/contracts/rust-1.4.0.json): + +| Input | Python | Native | Why | +| --- | --- | --- | --- | +| `4111 1111 1111 1112` | `CREDIT_CARD` | none | Core validates the Luhn checksum | +| `x123-45-6789y` | `SSN` | none | Core requires non-alphanumeric boundaries | +| `Invoice 1234567890` | `PHONE` | none | Core requires an area code starting 2-9, which is what keeps Unix timestamps out of `PHONE` | +| `unsupported` locale | locale ignored | `ValueError` | Core validates locales against its inventory | + +`PERSON` and `ADDRESS` used to need spaCy or GLiNER through `[nlp]` and +`[nlp-advanced]`. They now come from an embedded model on the native path, with +no download and no interpreter. `PERSON` for a text scan on `backend="python"` +is still Python-engine territory: the pure-Python regex engine has no name +detection, so select `PERSON` without forcing `backend="python"`. + +## The accuracy corpus + +`tests/corpus` is the frozen accuracy corpus, and its expectations describe the +Python engine, which is what they were written against. `pytest +tests/test_detection_accuracy.py` pins `backend="python"`. Set +`DATAFOG_ACCURACY_BACKEND=rust` to score the native engine; its twelve known +divergences — eight misses, one added finding and three shifted email +boundaries — are enumerated with reasons in +[tests/contracts/native-corpus.json](../tests/contracts/native-corpus.json), and +CI runs both variants. + +The three shifted email boundaries are the one worth filing a follow-up for: +Core finds `alice@example.com` inside ``email=alice@example.com`` only under its +`format: "env"` setting, and inside a quoted value only under +`format: "sql"`. Log-shaped input wants that behaviour by default; Core 1.4 +should decide that rather than leaving it to each caller. + +## If the output must not move + +Pin the backend, not just the version: + +```python +import datafog + +datafog.scan(text, backend="python") # exact 4.9.0 output +``` + +Or pin the whole package: `pip install "datafog==4.9.0"`. + +Detection output is part of the contract. Per +[VERSIONING.md](../VERSIONING.md), a change to the finding set is a MINOR, never +a PATCH, so a patch release will not move your golden files. + +## Reproducibility + +Core versions independently. If output must be byte-identical across machines, +lock it: + +```bash +pip install "datafog==4.10.0" "datafog-core==1.3.0" +``` + +`datafog.native.version()` reports the installed engine version and +`datafog.native.contract_version()` the contract it speaks, for bug reports. +Core follows its own [versioning policy](https://github.com/DataFog/datafog-core/blob/main/VERSIONING.md); +a contract bump is a Core MAJOR and raises the floor here. + +## What is not changing yet + +`detect()`, `process()`, OCR and Spark all still work and still warn. They are +removed in 5.0, not here. The last supported version in the current release train is 4.12.0 — see the +[retirement schedule](retirement.rst). diff --git a/docs/migration-4.9.md b/docs/migration-4.9.md index 3bf2b900..4e06e3fe 100644 --- a/docs/migration-4.9.md +++ b/docs/migration-4.9.md @@ -179,8 +179,9 @@ They are now scheduled for removal in 5.0. Use `scan` or `redact` in 4.9 and eva the native preview before upgrading to 5.0. `process(anonymize=True)` used older placeholder/hash semantics; migrating to `redact` can intentionally change output. -Users needing OCR/Spark can remain on the final 4.x release. No successor package -or indefinite support commitment is introduced here. Deprecation notices do not +Users needing OCR/Spark can pin `pip install "datafog==4.10.0"`, the last +supported version. No successor package is introduced here. See +[the retirement schedule](retirement.md). Deprecation notices do not trigger downloads or import heavy dependencies during ordinary text imports. ## Verification and performance diff --git a/docs/optional-surfaces.rst b/docs/optional-surfaces.rst index 96d1494d..006bec49 100644 --- a/docs/optional-surfaces.rst +++ b/docs/optional-surfaces.rst @@ -18,7 +18,8 @@ The default path is: OCR and Spark are deprecated optional surfaces in 4.9 and will be removed in 5.0. Their APIs, installation extras, and existing behavior remain available throughout -4.9. Remain on the final 4.x release if you need continued OCR or Spark support. +4.9. The last supported version is 4.10.0 — pin it with ``pip install +"datafog==4.10.0"``. See :doc:`retirement`. No replacement package is introduced by this migration. Use sites emit ``FutureWarning`` notices, visible with Python's default warning diff --git a/docs/python-sdk.rst b/docs/python-sdk.rst index 2a8a2ddf..581afc91 100644 --- a/docs/python-sdk.rst +++ b/docs/python-sdk.rst @@ -27,10 +27,11 @@ available for existing users. ``TextService(engine="regex")`` is the dependency-light service path; ``spacy``, ``gliner``, ``smart``, OCR, and Spark surfaces require their explicit extras. -4.9.0 compatibility and Core preview ------------------------------------- +4.10.0 native default and Core preview +-------------------------------------- -DataFog 4.9.0 provides an experimental Rust backend and native schema preview. +DataFog 4.10.0 uses the native engine by default and keeps the native schema +preview at ``datafog.v5`` and ``datafog.native``. Existing top-level ``scan``/``redact`` functions retain their result shapes and use the Python backend by default. They delegate through the new facade, whose classes are the same objects as the established result types: @@ -44,9 +45,9 @@ continue to work at the top level in 4.9 but warn of removal in 5.0, revising th previous promise to retain them throughout 5.x. Moving from ``process`` to ``redact`` can change old placeholder and hash output; compare results explicitly. -The adapter requires ``datafog-core>=0.4.0,<0.5``. Install with -``python -m pip install --upgrade "datafog[rust]==4.9.0"`` to evaluate Rust. -A base install of ``datafog==4.9.0`` has no native dependency; installing the +The adapter requires ``datafog-core>=1.3.0,<2``, which the base install pulls. Install with +``datafog-core`` is installed by every ``datafog`` install; ``backend="python"`` +or ``DATAFOG_BACKEND=python`` forces the pure-Python engine. Installing the extra does not change the default Python backend: .. code-block:: python @@ -89,7 +90,7 @@ a separate decision. The adapter requires capability contract version 1 and discovers supported labels, locales, and activation settings from the installed Core. Future finding labels -are preserved. Core 0.4.x updates may change detection output; lock the version +are preserved. Core 1.x updates may change detection output; lock the version when reproducibility is required. German aliases ``de``, ``de-DE``, and ``de_DE`` activate German detectors; ``en-US`` and ``fr`` activate only base detectors. Locale validation trims ASCII whitespace and ignores ASCII case; unsupported @@ -101,7 +102,7 @@ through Core metadata. Core's ``PERSON`` is structured-only and explicitly selecting it for Rust text scanning raises an error. The Python default backend keeps its existing behavior. -Core 0.4.0 adds JWT, private-key, contextual routing-number, and contextual NPI +Core 1.2 adds the same JWT, private-key, contextual routing-number and NPI findings. The legacy overlap policy can prefer ``PHONE`` over a same-span ``NPI``; filtering for NPI then returns no entities. Native ``datafog.v5.scan`` retains NPI, and native transformation can select it. Legacy result and transformation @@ -158,7 +159,8 @@ OCR and Spark remain available as optional surfaces throughout 4.9: DataFog 4.9.0 deprecates OCR and Spark for removal in 5.0. Use-time ``FutureWarning`` notices are visible under normal Python warning filters. Users -needing these features can remain on the final 4.x release; this migration does +needing these features can pin ``datafog==4.10.0``, the last supported version; +this migration does not introduce successor packages or promise indefinite maintenance. See :doc:`optional-surfaces` for install notes and limitations. diff --git a/docs/retirement.rst b/docs/retirement.rst new file mode 100644 index 00000000..01b2fc53 --- /dev/null +++ b/docs/retirement.rst @@ -0,0 +1,115 @@ +=================== +Retirement schedule +=================== + +This page is the standing record of surfaces DataFog is removing. It is updated +once per release, and a row is added in the MINOR that deprecates the surface — +never retroactively. + +Ordering and removal rules are defined in +`VERSIONING.md `__. + +Removed or scheduled +==================== + +.. list-table:: + :header-rows: 1 + + * - Surface + - Last supported version + - Replacement + - Removed in + - Deprecated in + * - ``OCRMService`` + - ``4.12.0`` + - none + - ``5.0.0`` + - ``4.9.0`` + * - ``DonutProcessor`` + - ``4.12.0`` + - none + - ``5.0.0`` + - ``4.9.0`` + * - ``[ocr]`` extra (Donut, Tesseract, Pillow) + - ``4.12.0`` + - none + - ``5.0.0`` + - ``4.9.0`` + * - ``[distributed]`` extra (PySpark, pandas) + - ``4.12.0`` + - :ref:`spark-recipe` + - ``5.0.0`` + - ``4.9.0`` + * - ``detect()`` + - ``4.12.0`` + - ``scan()`` + - ``5.0.0`` + - ``4.9.0`` + * - ``process()`` + - ``4.12.0`` + - ``redact()`` + - ``5.0.0`` + - ``4.9.0`` + +After the 4.12.0 release is published, installing ``pip install "datafog==4.12.0"`` keeps every row above working. +That version stays installable on PyPI permanently. + +OCR +=== + +There is no successor package and no 5.x reimplementation. OCR quality was never +the reason anyone adopted DataFog, and a mediocre replacement is worse than a +pin. + +.. code-block:: bash + + pip install "datafog[ocr]==4.12.0" + +If you need continued OCR support beyond that, stay on ``4.12.0``. Track the +surfaces you depend on by name, not by staying on a major version range, so a +lock file review shows the pin. + +.. _spark-recipe: + +Spark +===== + +No distributed API ships in 5.0. Partition the dataframe and call ``scan`` per +partition: one Spark job with a ``mapInPandas`` step, or Dask's +``map_partitions``. Both run the local engine and need no plugin. + +.. code-block:: python + + # Spark + def scan_partition(rows): + for pdf in rows: + pdf["findings"] = pdf["text"].map(lambda t: len(datafog.scan(t))) + yield pdf + + df.mapInPandas(scan_partition, schema="text string, findings long") + +.. code-block:: python + + # Dask + ddf["findings"] = ddf["text"].map_partitions( + lambda s: s.map(lambda t: len(datafog.scan(t))) + ) + +Per-partition ``scan`` is a pure local call with no network and no shared state, +so worker fan-out is just normal framework behaviour. What you lose versus +``SparkService`` is the UDF packaging, not correctness. + +detect and process +================== + +``detect()`` and ``process()`` were the pre-4.0 entry points. +``process(anonymize=True)`` used older placeholder and hash semantics, so +migrating to ``redact()`` can intentionally change output. Read the output +before you commit the change. + +DataFog Core +============ + +``datafog-core`` retires surfaces on its own schedule, documented in that +repository. The Python surface here inherits Core's ``contract_version``; a +contract bump is a Core MAJOR and raises the floor in ``datafog`` accordingly. diff --git a/docs/review-remediation.md b/docs/review-remediation.md new file mode 100644 index 00000000..7618b363 --- /dev/null +++ b/docs/review-remediation.md @@ -0,0 +1,123 @@ +# November 2026 review remediation + +This candidate addresses the review of DataFog 4.11.0 and Core 1.4.0. The +review's suggested fixes were assessed against the local implementations; +regression tests cover the implemented behavior. + +## Observable behavior + +All engine modes and legacy scanning APIs share the structured scan boundary. +Optional NER adds spans with its own character offsets. TextService scans the +whole document; `text_chunk_length` remains validated for compatibility. +Unsupported labels, invalid ranges and invalid locales fail explicitly. Source +code-point offsets remain stable when lone surrogate characters are replaced +for native transport. `get_supported_entities()` reports supported labels, +including opt-in ones, rather than only currently active default labels. + +The `create_guardrail` decorator supports sync and async functions returning +strings. Non-string results raise TypeError; they cannot silently bypass the +filter. Apply a guardrail directly to the text fields of structured results. + +LiteLLM scans messages, nested tool/function arguments, prompt and input fields. +Image and binary parts are retained without OCR. Tokens are assigned per request +or response and repeated values reuse their token. Streamed response text and +tool arguments are buffered until completion, then scanned before any chunk is +emitted. This adds completion latency. Exceeding eight million text characters +or 100,000 chunks rejects the stream. Engine failures follow the explicit +`fail_policy`; deployments requiring protected egress should select `closed`. +The Claude Code hook requests review or denies oversized PreToolUse inputs; +nonblocking events receive an explicit incomplete-scan notice. Unexpected hook +failures retain the documented fail-open contract and log exception types only. + +`hash_type` selects md5, sha256 or sha3_256 on the CLI and legacy model APIs. +The engine also accepts fixed-length algorithms provided by hashlib. Existing +plain-hash marker formats remain compatible: engine, class facade and CLI +markers use 12 hex characters with SHA256 as their default; deprecated top-level +`process` uses eight hex characters and retains its MD5 default. Its explicit +`hash_type` option uses that same legacy width. The legacy Anonymizer model keeps +its existing full plain digests. Explicit keyed hashes use the full HMAC digest; +an omitted algorithm selects SHA256 for this new keyed mode. +Plain deterministic hashes are pseudonymization: SSNs, phones and +other guessable inputs can be enumerated. Use `hash_key` (at least 32 random +bytes) for keyed HMAC and keep the key outside logs and source control: + +```python +import secrets +import datafog +key = secrets.token_bytes(32) # store securely to reproduce pseudonyms +result = datafog.redact("mail jane@example.com", strategy="hash", hash_key=key) +``` + +## Release sequence and qualification + +The Python candidate is 4.12.0. Its standalone CI uses the published Core 1.4 +minor series, pinned with `~=1.4.0`. Core 1.5.0 is prepared separately as a minor +release correcting detection and transformation behavior through the existing +APIs. Public APIs remain available and capability contract 1 is unchanged. Before +shipping the combined fixes, publish the qualified Core candidate, then raise +both Python dependency declarations to `~=1.5.0`, update the reviewed native +contract/corpus, and run Python's installed-wheel and release qualification +against the published artifacts. Source-candidate integration is additional +evidence; it does not substitute for that post-publication gate. + +Scheduled Python jobs only qualify candidates. Manual `release.yml` runs default +to `dry_run: true`; explicitly selecting false is the publication boundary. +Candidate bases must be newer than every stable Git tag and the latest stable +PyPI release. Development qualification refuses a ref missing `main` history. +The workflow atomically pushes the tested version commit and immutable tag +before uploading to PyPI, then fast-forwards `dev` after a stable publication. +The user must authorize deployment before any publishing workflow is dispatched. + +If registry upload fails after the atomic ref push, keep the tag. Inspect PyPI +for partial artifacts, compare hashes to the retained workflow artifact and tag, +and resume only the missing upload from those exact validated artifacts. Do not +move the tag, reuse the version for different bytes, or rerun version generation +with an existing release tag. A changed branch must be qualified again. + +Core's individual registries have separate availability; never infer that a +Python version is present on npm or crates.io. Native wheel coverage and source +build requirements are documented in Core's platform policy. + +## Python finding disposition + +All 35 Python findings have a concrete resolution below. Core-owned fixes are +separately tracked in Core's `docs/reviews/2026-11-core-findings.md`; their combined +release still requires the Core publication and Python dependency update above. + +| Finding | Resolution and proof | +| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| PY-H1 | Replace lone surrogates only for native transport, preserving source coordinates. Engine and hook/LiteLLM block regressions cover mixed payloads. | +| PY-H2 | Interval queries replace pairwise overlap checks; redaction and legacy assembly are linear after sorting. Dense scan/redaction benchmarks and overlap tests cover the change. | +| PY-H3 | TextService scans complete documents through the shared engine. Boundary-crossing email/SSN tests pass. | +| PY-H4 | spaCy/GLiNER return actual character spans. Stub and real-spaCy tests distinguish Ann from Annual and repeated names. | +| PY-H5 | Every engine and legacy/CLI path uses the same structured layer before optional NER. Discover/IPv6/credential tests cover the paths. | +| PY-M1 | Coroutine decorators await results; non-string results raise a documented TypeError rather than silently escaping inspection. | +| PY-M2 | Redact, replace and hash pipeline operations imply scanning. All three are tested without an explicit SCAN operation. | +| PY-M3 | Fallback email consumes each lexical run once; German separator expressions avoid ambiguous repetition. Adversarial 80K-character measurements and positive matches pass. | +| PY-M4 | Native backend selection also supplies the structured stage of smart/spaCy/GLiNER. CLI commands work with DATAFOG_BACKEND=rust. | +| PY-M5 | Request-scoped tokens, nested tool/function arguments, prompt/input fields and buffered streaming protect all tested request/response shapes. Live proxy deployment is not claimed. | +| PY-M6 | Bare strings, invalid labels and labels unavailable to the selected backend/engine raise explicit errors. Supported aliases remain accepted. | +| PY-M7 | APIs and CLI honor hash_type, preserve existing plain-hash formats and support full-length keyed HMAC. Plain-hash guessing limitations are documented; algorithm/key and compatibility tests pass. | +| PY-M8 | Hook per-string/total budget exhaustion emits review/deny or an incomplete-scan notice. Tests put PII past the cap. | +| PY-M9 | Allowlist expressions run with match timeouts; timed-out expressions retain findings. Adversarial patterns and safe quantified groups are tested. | +| PY-M10 | Dependency resolution stays within the explicitly reviewed Core minor, replacing the broad less-than-2 range. Exact output locks should still pin patch versions. | +| PY-M11 | Reconcile stable history, reject stale/PyPI-occupied versions, gate 3.13, sync version metadata and push immutable refs before upload. Scheduled runs are nonpublishing; selector tests and actionlint pass. | +| PY-M12 | Spark maps ordinary PERSON/ORG/GPE/NORP/DATE labels into its fixed output schema. Stub tests cover real spaCy label spellings and oversized input errors. | +| PY-M13 | Optional Pydantic fields have defaults, labels use canonical names, enum filters work and overlaps merge before replacement. Model/anonymizer regressions pass. | +| PY-M14 | Preserve supported GLiNER labels, use exact offsets, exclude relative spaCy DATE mentions, propagate model failures and reject oversized annotator input explicitly. | +| PY-M15 | Embedded model calls use bounded overlapping chunks. Large PERSON/ADDRESS plus structured-PII tests succeed with published and candidate Core. Native model limits remain enforced. | +| PY-M16 | Core 1.5 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | +| PY-L1 | Console bootstrap prints the CLI-extra install hint on a base install. A clean installed-wheel check confirms no traceback. | +| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 1.5 suppresses the labelled-tax-ID PHONE false positive. | +| PY-L3 | Retirement notices derive the current supported release version, and the native install hint no longer names an older release. | +| PY-L4 | Supported-entity reporting follows the selected native/Python capability inventory, including opt-in labels. Tests separate supported from default-active labels. | +| PY-L5 | Document actual token/pseudonym outputs and reject invalid methods in both process APIs. | +| PY-L6 | Legacy APIs and TextPIIAnnotator route through the shared engine; service/engine integration covers native-only entities and locale behavior. Explicit Python fallback remains a narrower documented detector set. | +| PY-L7 | Tokens reuse values and avoid existing markers; invalid explicit ranges/text raise errors. Collision, repeated-value and validation tests pass. | +| PY-L8 | Remove unused implementation copies, declare an isolated build, include test packages/corpora/scripts in sdists, align formatter pins and extend tox through 3.14. Optional OCR imports skip cleanly when absent. | +| PY-L9 | CLI configuration hides API keys; import and commands preserve host logging levels. CLI/logger regressions pass. | +| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 1.5 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | +| PY-L11 | TextService validates positive chunk lengths and engine names with explicit exceptions, including optimized Python execution. | +| PY-L12 | Remove incompatible Torch upper bound and include spaCy in nlp-advanced. Resolving it with Core GLiNER2 succeeds with Torch 2.12.0. | +| PY-L13 | Declare Pydantic 2.1 minimum. A clean Python 3.10 base install with Pydantic 2.1 imports and scans successfully. | +| PY-L14 | Correct native-default documentation, normalize missing native metadata to ImportError, clarify missing-model hints, validate locales consistently and surface invalid CLI operations as usage errors. | diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..cb600497 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..536f394a --- /dev/null +++ b/ruff.toml @@ -0,0 +1,7 @@ +# Only the delta from the default ruff rules. Rule selection stays ruff's +# default so this hook behaves like pre-4.10; flake8 config is in .flake8. + +[lint.per-file-ignores] +# datafog/native.py re-exports the installed datafog-core surface through a +# dynamic __getattr__, so its __all__ names resolve at runtime. +"datafog/native.py" = ["F822"] diff --git a/scripts/check_rust_install.py b/scripts/check_rust_install.py index 26120232..57604d3f 100644 --- a/scripts/check_rust_install.py +++ b/scripts/check_rust_install.py @@ -1,8 +1,10 @@ -"""Smoke-test installed 4.9 bridge wheels with python -I, outside checkout imports.""" +"""Smoke-test an installed wheel with python -I, outside checkout imports.""" import importlib.metadata import sys +from packaging.requirements import Requirement + def main(): assert sys.flags.isolated, "Run with python -I" @@ -13,18 +15,33 @@ def main(): from datafog import v5 from datafog.compat import v4 - assert importlib.metadata.version("datafog-core").startswith("0.4.") + installed = importlib.metadata.version("datafog-core") + requirements = [ + Requirement(value) for value in importlib.metadata.requires("datafog") or [] + ] + core_requirements = [ + requirement + for requirement in requirements + if requirement.name == "datafog-core" and requirement.marker is None + ] + assert ( + core_requirements + ), "Installed DataFog must declare its Core compatibility range" + assert all( + installed in requirement.specifier for requirement in core_requirements + ), installed assert datafog_core.capabilities()["contract_version"] == 1 assert v4.Entity is datafog.Entity assert v5.Finding is datafog_core.Finding text = "👋 Contact alice@example.com" - legacy = datafog.scan(text, backend="rust") + # No explicit backend: the native engine must be the resolved default. + legacy = datafog.scan(text) assert isinstance(legacy, datafog.ScanResult) assert legacy.entities[0].text == "alice@example.com" assert ( text[legacy.entities[0].start : legacy.entities[0].end] == "alice@example.com" ) - assert datafog.redact(text, backend="rust").redacted_text == "👋 Contact [EMAIL_1]" + assert datafog.redact(text).redacted_text == "👋 Contact [EMAIL_1]" native = v5.scan(text) assert isinstance(native[0], datafog_core.Finding) assert ( diff --git a/scripts/release_version.py b/scripts/release_version.py new file mode 100644 index 00000000..f78afd28 --- /dev/null +++ b/scripts/release_version.py @@ -0,0 +1,75 @@ +"""Select a release version while refusing stale or already published bases.""" + +import argparse +import json +import re +import subprocess +import urllib.request +from pathlib import Path + +from packaging.version import Version + + +def select_version( + current, release_type, tags, latest_stable, override=None, published=() +): + if release_type not in {"alpha", "beta", "stable"}: + raise ValueError("Unknown release type") + source = Version(current) + occupied = set(tags) | {f"v{Version(version)}" for version in published} + base = override or source.base_version + if not re.fullmatch(r"\d+\.\d+\.\d+", base): + raise ValueError("Release base must be numeric major.minor.patch") + if Version(base) <= Version(latest_stable): + raise ValueError("Release base must be newer than the latest stable release") + if release_type == "stable": + candidate = base + else: + suffix = "a" if release_type == "alpha" else "b" + number = 1 + while f"v{base}{suffix}{number}" in occupied: + number += 1 + candidate = f"{base}{suffix}{number}" + if f"v{candidate}" in occupied: + raise ValueError( + "Release tag already exists; inspect publication state before retrying" + ) + return candidate + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--type", required=True, choices=["alpha", "beta", "stable"]) + parser.add_argument("--override", default="") + parser.add_argument( + "--latest-stable", help="Offline preflight only; workflow reads PyPI" + ) + args = parser.parse_args() + tags = subprocess.check_output(["git", "tag", "-l"], text=True).splitlines() + stable_tags = [tag[1:] for tag in tags if re.fullmatch(r"v\d+\.\d+\.\d+", tag)] + if args.latest_stable: + published = args.latest_stable + published_versions = [published] + else: + with urllib.request.urlopen( + "https://pypi.org/pypi/datafog/json", timeout=30 + ) as response: + payload = json.load(response) + releases = [ + Version(version) + for version in payload["releases"] + if not Version(version).is_prerelease + ] + published = str(max(releases)) + published_versions = list(payload["releases"]) + latest = str(max(Version(version) for version in [published, *stable_tags])) + about = Path("datafog/__about__.py") + current = re.search(r'^__version__ = "([^"]+)"', about.read_text(), re.M).group(1) + candidate = select_version( + current, args.type, tags, latest, args.override or None, published_versions + ) + print(candidate) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 72cb2da3..bffb7ff5 100644 --- a/setup.py +++ b/setup.py @@ -18,9 +18,13 @@ # Core lightweight dependencies only core_deps = [ - "pydantic>=2.0,<3.0", + # Core minor releases may change findings; stay on this reviewed minor. + "datafog-core~=1.4.0", + "pydantic>=2.1,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", + # Match timeouts bound untrusted allowlist expressions. + "regex>=2024.11.6", ] # Optional heavy dependencies @@ -31,7 +35,7 @@ nlp_advanced_deps = [ "gliner>=0.2.5", - "torch>=2.1.0,<2.7", + "torch>=2.1.0,<3", "transformers>=4.20.0", "huggingface-hub>=0.16.0", "sentencepiece>=0.2.0", @@ -84,9 +88,10 @@ ] extras_require = { - "rust": ["datafog-core>=0.4.0,<0.5"], + # Retained for explicitness; the base install already pulls Core. + "rust": ["datafog-core~=1.4.0"], "nlp": nlp_deps, - "nlp-advanced": nlp_advanced_deps, + "nlp-advanced": nlp_deps + nlp_advanced_deps, "ocr": ocr_deps, "distributed": distributed_deps, "web": web_deps, @@ -113,6 +118,8 @@ version=version, author="Sid Mohan", author_email="sid@datafog.ai", + license="MIT", + license_files=["LICENSE"], description="Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)", long_description=long_description, long_description_content_type="text/markdown", @@ -123,14 +130,13 @@ python_requires=">=3.10", entry_points={ "console_scripts": [ - "datafog=datafog.client:app [cli]", # Requires cli extra + "datafog=datafog.cli:main", "datafog-hook=datafog.integrations.claude_code:main", # Core only ], }, classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/setup_lean.py b/setup_lean.py deleted file mode 100644 index 57315788..00000000 --- a/setup_lean.py +++ /dev/null @@ -1,109 +0,0 @@ -from setuptools import find_packages, setup - -# Historical shadow packaging snapshot. The live DataFog packaging input is -# setup.py. Do not use this file for builds, releases, or dependency changes. - -# Read README for the long description -with open("README.md", "r") as f: - long_description = f.read() - -# Use a single source of truth for the version -__version__ = "4.1.1" - -project_urls = { - "Homepage": "https://datafog.ai", - "Documentation": "https://docs.datafog.ai", - "Discord": "https://discord.gg/bzDth394R4", - "Twitter": "https://twitter.com/datafoginc", - "GitHub": "https://github.com/datafog/datafog-python", -} - -# Core lightweight dependencies only -core_deps = [ - "pydantic>=2.0,<3.0", - "typing-extensions>=4.0", -] - -# Optional heavy dependencies -extras_require = { - "nlp": [ - "spacy>=3.7.0,<4.0", - ], - "ocr": [ - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - ], - "distributed": [ - "pandas>=2.0.0", - "numpy>=1.24.0", - ], - "web": [ - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - ], - "cli": [ - "typer>=0.12.0", - ], - "crypto": [ - "cryptography>=40.0.0", - ], - "dev": [ - "pytest>=7.0.0", - "pytest-asyncio>=0.21.0", - "pytest-cov>=4.0.0", - "sphinx>=7.0.0", - ], - # Convenience bundles - "all": [ - "spacy>=3.7.0,<4.0", - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - "pandas>=2.0.0", - "numpy>=1.24.0", - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - "typer>=0.12.0", - "cryptography>=40.0.0", - ], -} - -setup( - name="datafog", - version=__version__, - author="Sid Mohan", - author_email="sid@datafog.ai", - description="Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)", - long_description=long_description, - long_description_content_type="text/markdown", - packages=find_packages(), - install_requires=core_deps, - extras_require=extras_require, - python_requires=">=3.10", - entry_points={ - "console_scripts": [ - "datafog=datafog.client:app [cli]", # Requires cli extra - ], - }, - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Text Processing", - "Topic :: Security", - ], - project_urls=project_urls, - keywords="pii detection anonymization privacy regex performance", -) diff --git a/setup_original.py b/setup_original.py deleted file mode 100644 index 006a3608..00000000 --- a/setup_original.py +++ /dev/null @@ -1,92 +0,0 @@ -from setuptools import find_packages, setup - -# Historical shadow packaging snapshot. The live DataFog packaging input is -# setup.py. Do not use this file for builds, releases, or dependency changes. - -# Read README for the long description -with open("README.md", "r") as f: - long_description = f.read() - -# Use a single source of truth for the version -__version__ = "4.1.1" - -project_urls = { - "Homepage": "https://datafog.ai", - "Documentation": "https://docs.datafog.ai", - "Discord": "https://discord.gg/bzDth394R4", - "Twitter": "https://twitter.com/datafoginc", - "GitHub": "https://github.com/datafog/datafog-python", -} - -setup( - name="datafog", - version=__version__, - author="Sid Mohan", - author_email="sid@datafog.ai", - description="Scan, redact, and manage PII in your documents before they get uploaded to a Retrieval Augmented Generation (RAG) system.", - long_description=long_description, - long_description_content_type="text/markdown", - packages=find_packages(), - install_requires=[ - "pandas", - "requests==2.32.3", - "spacy==3.7.5", - "pydantic", - "Pillow", - "sentencepiece", - "protobuf", - "pytesseract", - "aiohttp", - "pytest-asyncio", - "numpy", - "fastapi", - "asyncio", - "setuptools", - "pydantic-settings==2.3.4", - "typer==0.12.3", - "sphinx", - "cryptography", - ], - python_requires=">=3.10", - entry_points={ - "console_scripts": [ - "datafog=datafog.client:app", - ], - }, - classifiers=[ - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Framework :: tox", - "Framework :: Pytest", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "Intended Audience :: Information Technology", - "Intended Audience :: System Administrators", - ], - keywords="pii, redaction, nlp, rag, retrieval augmented generation", - maintainer="DataFog", - maintainer_email="hi@datafog.ai", - url="https://datafog.ai", - project_urls=project_urls, - license="MIT", - extras_require={ - "dev": [ - "just", - "isort", - "black", - "blacken-docs", - "flake8", - "tox", - "pytest", - "pytest-codeblocks", - "pytest-cov", - "build", - "twine", - ], - }, -) diff --git a/tests/benchmark_text_service.py b/tests/benchmark_text_service.py index 3178c61e..e802c40e 100644 --- a/tests/benchmark_text_service.py +++ b/tests/benchmark_text_service.py @@ -107,7 +107,7 @@ def test_spacy_performance(benchmark, sample_text_10kb, spacy_service): # Verify spaCy found expected entities assert "PERSON" in result or "PER" in result - assert "ORG" in result + assert "ORGANIZATION" in result # Print some stats about the results entity_counts = {key: len(values) for key, values in result.items() if values} @@ -178,7 +178,7 @@ def test_auto_engine_fallback_performance(benchmark, spacy_only_text, auto_servi ) # Check if we have spaCy entities (depends on spaCy availability) - spacy_entities = ["PERSON", "ORG", "GPE", "CARDINAL", "DATE", "TIME"] + spacy_entities = ["PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"] has_spacy_entities = any(entity in result for entity in spacy_entities) # If no spaCy entities, check if spaCy is available diff --git a/tests/contracts/native-corpus.json b/tests/contracts/native-corpus.json new file mode 100644 index 00000000..b9c3a893 --- /dev/null +++ b/tests/contracts/native-corpus.json @@ -0,0 +1,47 @@ +{ + "core_version": "1.3.0", + "cases": { + "adjacent-pii-no-separator": { + "missing": [["SSN", "123-45-6789"]], + "reason": "Core requires non-alphanumeric SSN boundaries; the adjacent-email case embeds it" + }, + "phone-two-values": { + "missing": [["PHONE", "555-000-1111"]], + "reason": "Core requires the exchange code to start 1-9; 000-1111 is a fictional exchange" + }, + "ssn-second-valid": { + "missing": [["SSN", "987-65-4321"]], + "reason": "Core rejects SSN after 'Backup SSN' context token in this boundary case" + }, + "date-slash-short": { + "missing": [["DATE", "3/5/2020"]], + "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" + }, + "date-dash-short": { + "missing": [["DATE", "3-5-2020"]], + "reason": "Core DATE requires explicit month/day ordering; single-digit short form is unsupported" + }, + "date-year-only": { + "missing": [["DATE", "year 2024"]], + "reason": "Core DATE requires a full date; bare years are not matched" + }, + "de-vat-id-locale": { + "missing": [["DE_VAT_ID", "DE 123456789"]], + "reason": "Core German VAT requires the digits to follow the country prefix without a space" + }, + "de-iban-locale": { + "missing": [["DE_IBAN", "DE44 5001 0517 5407 3249 31"]], + "reason": "Core German IBAN accepts the grouped form but this spacing variant differs" + } + }, + "extra": { + "product-code-not-phone": { + "added": [["PHONE", "555-123-4567"]], + "reason": "Core accepts exchange codes starting with 1, so this product code is PHONE now; the area-code 2-9 check still keeps Unix timestamps out" + }, + "unicode-chinese-name": { + "offsets_shifted": true, + "reason": "The corpus fixture itself contains eleven ASCII '?' characters (a mangled CJK name); '?' is a valid RFC local-part character, so the email span includes them in every engine. Overlap is the reviewed expectation." + } + } +} diff --git a/tests/contracts/rust-0.4.0.json b/tests/contracts/rust-1.4.0.json similarity index 70% rename from tests/contracts/rust-0.4.0.json rename to tests/contracts/rust-1.4.0.json index 541cc3a3..6d3aeb4b 100644 --- a/tests/contracts/rust-0.4.0.json +++ b/tests/contracts/rust-1.4.0.json @@ -1,5 +1,5 @@ { - "core_version": "0.4.0", + "core_version": "1.4.0", "cases": { "scan-observation-invalid-card": { "classification": "detector-difference", @@ -41,6 +41,21 @@ }, "warnings": [] } + }, + "scan-observation-ambiguous-number": { + "classification": "detector-difference", + "reason": "Core's North American area-code check requires a leading digit of 2-9, so a bare 10-digit run starting with 1 stays undetected. This is the deliberate Unix-timestamp protection; Python 4.8.1 matches it.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Invoice 1234567890", + "engine_used": "regex" + } + }, + "warnings": [] + } } } } diff --git a/tests/contracts/rust-1.5.0.json b/tests/contracts/rust-1.5.0.json new file mode 100644 index 00000000..13f39961 --- /dev/null +++ b/tests/contracts/rust-1.5.0.json @@ -0,0 +1,77 @@ +{ + "core_version": "1.5.0", + "cases": { + "scan-observation-invalid-card": { + "classification": "detector-difference", + "reason": "Core validates card checksums; Python 4.8.1 accepts this card-shaped value. Experimental precision difference, not a claim of full parity.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "4111 1111 1111 1112", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "scan-observation-boundary": { + "classification": "detector-difference", + "reason": "Core rejects the SSN embedded in an ASCII alphanumeric token; Python 4.8.1 detects the numeric substring.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "x123-45-6789y", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "locale-unknown": { + "classification": "validation-difference", + "reason": "The capability adapter rejects the unadvertised locale with a backend-specific diagnostic instead of the Python regex locale inventory. The exception remains ValueError.", + "expected": { + "error": { + "type": "ValueError", + "message": "Unsupported locale for the Rust backend: 'xx'" + }, + "warnings": [] + } + }, + "scan-observation-ambiguous-number": { + "classification": "detector-difference", + "reason": "Core's North American area-code check requires a leading digit of 2-9, so a bare 10-digit run starting with 1 stays undetected. This is the deliberate Unix-timestamp protection; Python 4.8.1 matches it.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Invoice 1234567890", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "locale-default-DE_TAX_ID": { + "classification": "detector-difference", + "reason": "CORE-17/PY-L2 suppress a German tax identifier from the generic PHONE detector even when German detectors are not activated. Explicit locale selection still detects DE_TAX_ID.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Steuer-ID 12345678901 liegt vor.", + "engine_used": "regex" + } + }, + "warnings": [] + } + } + }, + "review": "Core 1.5 candidate qualification retains reviewed historical differences and documents the contextual tax-ID correction. Wrapper 4.12 fixes are layered separately in wrapper-4.12.0.json." +} diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json new file mode 100644 index 00000000..cd78ae0d --- /dev/null +++ b/tests/contracts/wrapper-4.12.0.json @@ -0,0 +1,972 @@ +{ + "wrapper_version": "4.12.0", + "core_versions": ["1.4.0", "1.5.0"], + "source_oracle": "4.8.1.json (unchanged)", + "review": "2026-11-15 PDF remediation: explicit reviewed wrapper changes, not a regenerated historical oracle", + "cases": { + "selection-unknown": { + "classification": "validation-difference", + "reason": "PY-M6 rejects unknown entity labels instead of silently disabling detection.", + "expected": { + "error": { + "type": "ValueError", + "message": "Unsupported entity_types for rust/regex: NOT_A_TYPE" + }, + "warnings": [] + } + }, + "redact-token": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses one collision-safe token for repeated values.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_2] [EMAIL_1] [SSN_1]", + "mapping": { + "[EMAIL_1]": "a@example.com", + "[EMAIL_2]": "b@example.com", + "[SSN_1]": "123-45-6789" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "SSN", + "text": "123-45-6789", + "start": 42, + "end": 53, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "engine-redact-token": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses one collision-safe token for repeated values.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_2] [EMAIL_1]", + "mapping": { + "[EMAIL_1]": "a@example.com", + "[EMAIL_2]": "b@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "explicit-invalid-bounds": { + "classification": "validation-difference", + "reason": "PY-L7 rejects malformed explicit spans rather than silently skipping them.", + "expected": { + "error": { + "type": "ValueError", + "message": "entities must contain valid ranges whose text matches the source" + }, + "warnings": [] + } + }, + "explicit-mismatched-text": { + "classification": "validation-difference", + "reason": "PY-L7 rejects explicit span text that differs from the source.", + "expected": { + "error": { + "type": "ValueError", + "message": "entities must contain valid ranges whose text matches the source" + }, + "warnings": [] + } + }, + "error-invalid-pattern": { + "classification": "validation-difference", + "reason": "PY-M9 uses a timeout-capable matcher, whose syntax diagnostic counts positions differently.", + "expected": { + "error": { + "type": "ValueError", + "message": "allowlist_patterns contains an invalid regex: '[' (unterminated character set at position 1)" + }, + "warnings": [] + } + }, + "error-unsafe-pattern": { + "classification": "behavioral-fix", + "reason": "PY-M9 allows nested groups under a hard matching timeout instead of a bypassable syntax heuristic.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "error-oversized-pattern": { + "classification": "validation-difference", + "reason": "PY-M9 validates both pattern type and length with an explicit diagnostic.", + "expected": { + "error": { + "type": "ValueError", + "message": "allowlist_patterns entries must be strings of at most 512 characters" + }, + "warnings": [] + } + }, + "facade-sanitize": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": "[EMAIL_1] [EMAIL_1]", + "warnings": [] + } + }, + "facade-filter_output": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_1]", + "mapping": { + "[EMAIL_1]": "alice@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 0, + "end": 17, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 18, + "end": 35, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "facade-detect": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + }, + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 18, + "end": 35 + } + ], + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.detect() is deprecated and will be removed in 5.0. Use datafog.scan() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "facade-anonymize_text": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": "[EMAIL_1] [EMAIL_1]", + "warnings": [] + } + }, + "facade-process": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + }, + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 18, + "end": 35 + } + ] + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-redact": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_REDACTED]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-replace": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_XXXXX]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-hash": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release. Legacy default MD5 and its eight-character prefix remain compatible; explicit algorithms and keyed HMAC are additional options.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_c160f8cc]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-invalid": { + "classification": "validation-difference", + "reason": "PY-L5 rejects an invalid anonymization method instead of choosing an undocumented fallback; PY-L3 updates the notice.", + "expected": { + "error": { + "type": "ValueError", + "message": "method must be 'redact', 'replace', or 'hash'" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "service-default": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route TextService through shared native detection and return populated canonical labels.", + "expected": { + "value": { + "EMAIL": ["alice@example.com"] + }, + "warnings": [] + } + }, + "service-german": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route TextService through shared native detection with overlap resolution and populated canonical labels.", + "expected": { + "value": { + "DE_TAX_ID": ["12345678901"] + }, + "warnings": [] + } + }, + "service-batch": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route every TextService batch member through shared native detection, returning populated canonical labels.", + "expected": { + "value": [ + { + "EMAIL": ["alice@example.com"] + }, + {} + ], + "warnings": [] + } + }, + "public-signatures": { + "classification": "signature-addition", + "reason": "PY-M7 adds keyword-only hash_type/hash_key while preserving existing unkeyed output widths and omitted legacy hash defaults; TextService custom models use the engine gliner_model keyword; backend remains keyword-only.", + "expected": { + "value": { + "datafog:scan": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "preset", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:protect": { + "parameters": [ + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "on_detect", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + } + ] + }, + "datafog:detect": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:process": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "anonymize", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": false + }, + { + "name": "method", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:sanitize": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:scan_prompt": { + "parameters": [ + { + "name": "prompt", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:filter_output": { + "parameters": [ + { + "name": "output", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:create_guardrail": { + "parameters": [ + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "on_detect", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + } + ] + }, + "datafog.engine:Entity": { + "parameters": [ + { + "name": "type", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "start", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "end", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "confidence", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": ["type", "text", "start", "end", "confidence", "engine"] + }, + "datafog.engine:ScanResult": { + "parameters": [ + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine_used", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": ["entities", "text", "engine_used"] + }, + "datafog.engine:RedactResult": { + "parameters": [ + { + "name": "redacted_text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "mapping", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": ["redacted_text", "mapping", "entities"] + }, + "datafog.engine:scan": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "smart" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "gliner_model", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog.engine:redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog.engine:scan_and_redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "smart" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + } + }, + "warnings": [] + } + } + } +} diff --git a/tests/corpus/negative_cases.json b/tests/corpus/negative_cases.json index 44eb1fcf..c5b67b76 100644 --- a/tests/corpus/negative_cases.json +++ b/tests/corpus/negative_cases.json @@ -49,7 +49,16 @@ { "id": "markdown-link", "input": "See [RFC 1918](https://example.com/rfc1918)", - "expected_entities": [] + "expected_entities": [], + "permitted_gliner_entities": [ + { + "type": "URL", + "text": "https://example.com/rfc1918", + "start": 15, + "end": 42 + } + ], + "permitted_gliner_reason": "PY-M14 restores the existing GLiNER URL label. This exact public URL is valid URL detection; it must not become EMAIL, PHONE or IP_ADDRESS." }, { "id": "code-symbol", @@ -74,6 +83,15 @@ { "id": "url-with-at", "input": "https://example.com/@user/profile is a URL path", - "expected_entities": [] + "expected_entities": [], + "permitted_gliner_entities": [ + { + "type": "URL", + "text": "https://example.com/@user/profile", + "start": 0, + "end": 33 + } + ], + "permitted_gliner_reason": "PY-M14 restores the existing GLiNER URL label. This exact public URL is valid URL detection; it must not become EMAIL, PHONE or IP_ADDRESS." } ] diff --git a/tests/corpus/unstructured_pii.json b/tests/corpus/unstructured_pii.json index ad91c35b..1e5ddf79 100644 --- a/tests/corpus/unstructured_pii.json +++ b/tests/corpus/unstructured_pii.json @@ -168,8 +168,8 @@ { "type": "LOCATION", "text": "Jordan", - "start": 0, - "end": 6 + "start": 33, + "end": 39 } ] }, diff --git a/tests/rust_contract.py b/tests/rust_contract.py index 3b477231..9e673091 100644 --- a/tests/rust_contract.py +++ b/tests/rust_contract.py @@ -9,7 +9,11 @@ from tests.contract_481 import FIXTURE, observe -DEVIATIONS = Path(__file__).parent / "contracts" / "rust-0.4.0.json" +CORE_VERSION = importlib.metadata.version("datafog-core") +CORE_LINE = ".".join(CORE_VERSION.split(".")[:2]) +DEVIATIONS = Path(__file__).parent / "contracts" / f"rust-{CORE_LINE}.0.json" +WRAPPER_DEVIATIONS = Path(__file__).parent / "contracts" / "wrapper-4.12.0.json" + TARGETS = { "datafog:scan", "datafog:redact", @@ -20,6 +24,18 @@ } +def reviewed_deviation(case): + """Layer explicit wrapper fixes on the unchanged historical wheel oracle.""" + wrapper = json.loads(WRAPPER_DEVIATIONS.read_text(encoding="utf-8")) + native = json.loads(DEVIATIONS.read_text(encoding="utf-8")) + return native["cases"].get(case["id"]) or wrapper["cases"].get(case["id"]) + + +def expected_observation(case): + deviation = reviewed_deviation(case) + return copy.deepcopy(deviation["expected"] if deviation else case["expected"]) + + def applicable(case): return ( case.get("operation", "call") == "call" @@ -39,15 +55,14 @@ def main(): parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() contract = json.loads(FIXTURE.read_text(encoding="utf-8")) - deviations = json.loads(DEVIATIONS.read_text(encoding="utf-8")) results = [] for case in contract["cases"]: if not applicable(case): results.append({"id": case["id"], "status": "outside-backend-scope"}) continue actual = native_observation(case) - deviation = deviations["cases"].get(case["id"]) - expected = deviation["expected"] if deviation else case["expected"] + deviation = reviewed_deviation(case) + expected = expected_observation(case) status = "regression" if actual != expected else "match" if status == "match" and deviation: status = deviation["classification"] diff --git a/tests/simple_performance_test.py b/tests/simple_performance_test.py index 6da82d88..589ef59b 100644 --- a/tests/simple_performance_test.py +++ b/tests/simple_performance_test.py @@ -90,7 +90,7 @@ def test_simple_spacy_performance(): # Verify functionality assert "PERSON" in stats["result"] or "PER" in stats["result"] - assert "ORG" in stats["result"] + assert "ORGANIZATION" in stats["result"] return stats diff --git a/tests/test_allowlist.py b/tests/test_allowlist.py index 1f3bef6e..fc4b2865 100644 --- a/tests/test_allowlist.py +++ b/tests/test_allowlist.py @@ -10,7 +10,8 @@ EMAIL = "jane.doe@" "example.com" OTHER_EMAIL = "sid@" "example.com" -TIMESTAMP_LIKE = "17830" "25668" # ten digits: matches the PHONE pattern +TIMESTAMP_LIKE = "44123" "45678" # ten digits, NANP-shaped: both backends match +LEADING_ONE_TIMESTAMP = "17830" "25668" # native reads the leading 1 as a country code class TestExactAllowlist: @@ -54,6 +55,23 @@ def test_pattern_suppresses_matching_entities(self): ) assert result.entities == [] + def test_leading_one_timestamp_is_a_documented_native_delta(self): + # The native NANP check reads a leading 1 as a country code and needs a + # nine-digit national number after it, so this misses; the Python engine + # matches. Recorded rather than silently accepted. + assert ( + datafog.scan(f"created {LEADING_ONE_TIMESTAMP}", engine="regex").entities + == [] + ) + assert ( + len( + datafog.scan( + f"created {LEADING_ONE_TIMESTAMP}", engine="regex", backend="python" + ).entities + ) + == 1 + ) + def test_pattern_matches_full_entity_text_only(self): result = datafog.scan( f"mail {EMAIL}", engine="regex", allowlist_patterns=[r"^jane\."] @@ -75,13 +93,15 @@ def test_patterns_and_values_combine(self): class TestReDoSGuards: - def test_catastrophic_pattern_rejected(self): - with pytest.raises(ValueError, match="catastrophic backtracking"): - datafog.scan("text", engine="regex", allowlist_patterns=[r"(a+)+$"]) - - def test_nested_star_rejected(self): - with pytest.raises(ValueError, match="catastrophic backtracking"): - datafog.scan("text", engine="regex", allowlist_patterns=[r"(.*)*"]) + def test_nested_quantifier_uses_bounded_matching(self): + assert datafog.scan( + "a@b.com", engine="regex", allowlist_patterns=[r"(a+)+$"] + ).entities + + def test_safe_nested_star_is_supported(self): + assert not datafog.scan( + "a@b.com", engine="regex", allowlist_patterns=[r"(.*)*"] + ).entities def test_overlong_pattern_rejected(self): with pytest.raises(ValueError, match="at most"): diff --git a/tests/test_api_bridge_49.py b/tests/test_api_bridge_49.py index 483aa7f8..d47ff561 100644 --- a/tests/test_api_bridge_49.py +++ b/tests/test_api_bridge_49.py @@ -32,7 +32,7 @@ def test_existing_positional_calls_match_compatibility_facade(): def test_backend_is_additive_keyword_only(function): parameter = inspect.signature(function).parameters["backend"] assert parameter.kind is inspect.Parameter.KEYWORD_ONLY - assert parameter.default == "python" + assert parameter.default is None @pytest.mark.parametrize("name", ["scan", "redact"]) diff --git a/tests/test_backend_resolution.py b/tests/test_backend_resolution.py new file mode 100644 index 00000000..55abb06d --- /dev/null +++ b/tests/test_backend_resolution.py @@ -0,0 +1,114 @@ +"""Backend resolution: native by default, Python fallback, env override.""" + +import builtins + +import pytest + +import datafog +from datafog import engine +from datafog.exceptions import CoreUnavailableWarning + + +@pytest.fixture +def without_core(monkeypatch): + """Hide ``datafog_core`` from the import machinery.""" + original_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + if name == "datafog_core" or name.startswith("datafog_core."): + raise ImportError("hidden for test") + return original_import(name, *args, **kwargs) + + monkeypatch.delitem(__import__("sys").modules, "datafog_core", raising=False) + monkeypatch.setattr(builtins, "__import__", guarded_import) + monkeypatch.setattr(engine, "_cnn_cache", None) + + +def test_auto_resolves_to_native_when_core_is_installed(): + pytest.importorskip("datafog_core") + assert engine.resolve_backend(None, "regex") == "rust" + + +def test_auto_resolves_to_python_without_core(without_core): + assert engine.resolve_backend(None, "regex") == "python" + + +def test_auto_uses_native_structured_layer_for_every_engine(): + pytest.importorskip("datafog_core") + for engine_name in ("smart", "spacy", "gliner"): + assert engine.resolve_backend(None, engine_name) == "rust" + + +def test_python_fallback_warns_once_then_stays_quiet(without_core): + with pytest.warns(CoreUnavailableWarning) as caught: + for _ in range(2): + datafog.scan("mail jane@example.com") + assert len(caught) == 1 + + +def test_explicit_python_does_not_warn(): + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("error", CoreUnavailableWarning) + assert datafog.scan("mail jane@example.com", backend="python").entities + + +def test_env_override_selects_the_python_engine(monkeypatch): + pytest.importorskip("datafog_core") + monkeypatch.setenv(engine.BACKEND_ENV, "python") + assert engine.resolve_backend(None, "regex") == "python" + + +def test_explicit_backend_beats_the_env(monkeypatch): + pytest.importorskip("datafog_core") + monkeypatch.setenv(engine.BACKEND_ENV, "python") + assert engine.resolve_backend("rust", "regex") == "rust" + + +def test_env_is_case_and_space_insensitive(monkeypatch): + pytest.importorskip("datafog_core") + monkeypatch.setenv(engine.BACKEND_ENV, " AUTO ") + assert engine.resolve_backend(None, "regex") == "rust" + + +def test_unknown_env_value_rejected(monkeypatch): + monkeypatch.setenv(engine.BACKEND_ENV, "gpu") + with pytest.raises(ValueError, match="backend must be one of"): + engine.scan("text", "regex") + + +def test_explicit_rust_without_core_still_raises(without_core): + with pytest.raises(ImportError, match="datafog\\[rust\\]"): + datafog.scan("mail jane@example.com", backend="rust") + + +def test_person_selection_without_model_raises_a_pin(monkeypatch): + from types import SimpleNamespace + + meta = { + "contract_version": 1, + "supported_entities": ["EMAIL", "PERSON"], + "default_entities": ["EMAIL"], + "locales": {"en-US": {"enabled_entities": []}}, + "entities": { + "EMAIL": { + "scopes": ["structured", "text"], + "activation": {"kind": "default"}, + }, + "PERSON": {"scopes": ["structured"], "activation": {"kind": "default"}}, + }, + } + stub = SimpleNamespace(capabilities=lambda: meta, scan=lambda text, config=None: []) + monkeypatch.setitem(__import__("sys").modules, "datafog_core", stub) + monkeypatch.setattr(engine, "_cnn_cache", None) + with pytest.raises(ValueError, match="embedded model"): + datafog.scan("Jane Doe", entity_types=["PERSON"], backend="rust") + + +def test_native_findings_are_classified_as_regex_provenance(): + pytest.importorskip("datafog_core") + result = datafog.scan("mail jane@example.com") + assert result.engine_used == "regex" + assert result.entities[0].engine == "regex" + assert result.entities[0].confidence == 1.0 diff --git a/tests/test_client.py b/tests/test_client.py index d5b042d1..24734be1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -165,7 +165,7 @@ def test_anonymizer_outputs(): ), AnnotationResult( start=33, - end=41, + end=len(text), score=1.0, entity_type=EntityTypes.LOCATION, recognition_metadata=AnnotatorMetadata(), diff --git a/tests/test_contract_481.py b/tests/test_contract_481.py index 5aa5b767..b54ecf1e 100644 --- a/tests/test_contract_481.py +++ b/tests/test_contract_481.py @@ -1,51 +1,33 @@ """Frozen observations from the published 4.8.1 wheel; never regenerate in CI.""" -import copy import json import pytest from tests.contract_481 import FIXTURE, observe +from tests.rust_contract import WRAPPER_DEVIATIONS, expected_observation CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) +REVIEWED_WRAPPER = json.loads(WRAPPER_DEVIATIONS.read_text(encoding="utf-8")) def expected_49(case): - """Allow only the four added backend parameters and revised shim notices.""" - expected = copy.deepcopy(case["expected"]) - if case["id"] == "public-signatures": - for target in ( - "datafog:scan", - "datafog:redact", - "datafog.engine:scan", - "datafog.engine:scan_and_redact", - ): - expected["value"][target]["parameters"].append( - { - "name": "backend", - "kind": "KEYWORD_ONLY", - "required": False, - "default": "python", - } - ) - if case.get("target") in {"datafog:detect", "datafog:process"}: - name = case["target"].split(":")[1] - replacement = ( - "datafog.scan()" - if name == "detect" - else "datafog.scan() or datafog.redact()" - ) - expected["warnings"] = [ - { - "type": "FutureWarning", - "message": ( - f"datafog.{name}() is deprecated and will be removed in 5.0. " - f"Use {replacement} instead. " - "The earlier promise to retain this shim through 5.x has been revised." - ), - } - ] - return expected + """The immutable oracle plus individually explained native/wrapper fixes.""" + return expected_observation(case) + + +def test_reviewed_wrapper_deviations_are_explicit_and_in_scope(): + cases = {case["id"]: case for case in CONTRACT["cases"]} + assert set(REVIEWED_WRAPPER["cases"]) <= set(cases) + for case_id, deviation in REVIEWED_WRAPPER["cases"].items(): + assert deviation["reason"] + assert deviation["classification"] in { + "behavioral-fix", + "validation-difference", + "diagnostic-difference", + "signature-addition", + } + assert deviation["expected"] != cases[case_id]["expected"] @pytest.mark.parametrize("case", CONTRACT["cases"], ids=lambda case: case["id"]) diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index 272060af..9f952bac 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -1,4 +1,4 @@ -"""Exercise actual Core 0.4 wheels through the Python compatibility adapter. +"""Exercise reviewed native wheels through the Python compatibility adapter. Examples originate in Core's fixtures/{german,jwt,private-key,npi, us-routing-number,uuid}.jsonl. Payloads are synthetic detector fixtures, @@ -6,6 +6,7 @@ """ import importlib.metadata +import json import re import pytest @@ -13,6 +14,7 @@ import datafog from datafog import engine, v5 from datafog.compat import v4 +from tests.rust_contract import DEVIATIONS core = pytest.importorskip("datafog_core") @@ -50,7 +52,8 @@ def test_supported_wheel_version_and_capability_contract(): version = importlib.metadata.version("datafog-core") - assert version.split(".")[:2] == ["0", "4"], version + reviewed = json.loads(DEVIATIONS.read_text())["core_version"] + assert version.split(".")[:2] == reviewed.split(".")[:2], version capabilities = core.capabilities() assert capabilities["contract_version"] == 1 supported = capabilities["supported_entities"] @@ -174,12 +177,16 @@ def test_contextual_identifiers_do_not_activate_without_context(label, raw): assert not result.entities -def test_npi_retains_native_identity_but_legacy_overlap_prefers_phone(): - """4.x suppresses overlaps before selection; this is a reviewed difference.""" +def test_contextual_npi_survives_the_legacy_overlap_policy(): + """A leading-1 national number is not PHONE, so NPI is no longer suppressed. + + Core 1.2 accepts exchange codes starting with 1 but keeps the area-code + 2-9 check, so `1234567893` is NPI only. Under Core 0.4 the same span was + ambiguous and the legacy overlap policy dropped the NPI. + """ text = "😀 café NPI 1234567893" native = v5.scan(text) assert [(item.entity_type, item.matched_text) for item in native] == [ - ("PHONE", "1234567893"), ("NPI", "1234567893"), ] assert { @@ -187,10 +194,13 @@ def test_npi_retains_native_identity_but_legacy_overlap_prefers_phone(): } == {(11, 21)} legacy = datafog.scan(text, backend="rust") assert [(item.type, item.text) for item in legacy.entities] == [ - ("PHONE", "1234567893") + ("NPI", "1234567893") ] - assert not datafog.scan(text, entity_types=["NPI"], backend="rust").entities - assert datafog.redact(text, backend="rust").redacted_text == "😀 café NPI [PHONE_1]" + assert [ + item.type + for item in datafog.scan(text, entity_types=["NPI"], backend="rust").entities + ] == ["NPI"] + assert datafog.redact(text, backend="rust").redacted_text == "😀 café NPI [NPI_1]" # Native selection occurs before transformation overlap resolution, exposing # the NPI-specific route without changing the 4.x compatibility contract. result = v5.scan_and_transform( @@ -200,9 +210,13 @@ def test_npi_retains_native_identity_but_legacy_overlap_prefers_phone(): assert result.text == "😀 café NPI [NPI]" -def test_structured_only_person_rejected_by_text_adapter(): - with pytest.raises(ValueError, match="PERSON"): - datafog.scan("Jane Doe", entity_types=["PERSON"], backend="rust") +def test_structured_only_person_served_by_the_embedded_model(): + """`PERSON` is structured-only in the rule inventory but the embedded CNN + covers prose, so text selection succeeds instead of raising.""" + with pytest.raises(ValueError, match="Unsupported entity_types"): + datafog.scan("Jane Doe", entity_types=["PERSON"], backend="python") + result = datafog.scan("Jane Doe\n14 Penny Lane", entity_types=["PERSON", "ADDRESS"]) + assert [item.type for item in result.entities] == ["PERSON", "ADDRESS"] def test_unsupported_locale_rejected_explicitly(): @@ -210,6 +224,7 @@ def test_unsupported_locale_rejected_explicitly(): datafog.scan("a@example.com", locales=["unsupported"], backend="rust") -def test_python_default_backend_remains_unchanged(): - assert not datafog.scan(JWT).entities +def test_native_backend_is_the_default(): + assert [item.type for item in datafog.scan(JWT).entities] == ["JWT"] assert [item.type for item in datafog.scan(JWT, backend="rust").entities] == ["JWT"] + assert not datafog.scan(JWT, backend="python").entities diff --git a/tests/test_core15_integration.py b/tests/test_core15_integration.py new file mode 100644 index 00000000..ed9bcdf4 --- /dev/null +++ b/tests/test_core15_integration.py @@ -0,0 +1,58 @@ +"""Core 1.5 candidate seam regressions; exercised before the dependency floor moves.""" + +import importlib.metadata + +import pytest + +import datafog + +pytestmark = pytest.mark.skipif( + importlib.metadata.version("datafog-core").split(".")[:2] != ["1", "5"], + reason="qualifies the separately built Core 1.5 candidate before publication", +) + + +@pytest.mark.parametrize("value", ["::ffff:192.168.1.1", "64:ff9b::192.0.2.33"]) +def test_ipv6_embedded_ipv4_tail_is_fully_redacted(value): + result = datafog.redact(value) + assert result.redacted_text == "[IP_ADDRESS_1]" + assert result.mapping == {"[IP_ADDRESS_1]": value} + + +def test_contextual_email_does_not_swallow_neighboring_phone(): + text = "phone=415-555-2671&email=bob@example.com" + result = datafog.scan(text) + assert [(item.type, item.text) for item in result.entities] == [ + ("PHONE", "415-555-2671"), + ("EMAIL", "bob@example.com"), + ] + assert datafog.redact(text).redacted_text == "phone=[PHONE_1]&email=[EMAIL_1]" + + +def test_phone_followed_by_zip_does_not_become_credit_card(): + text = "555-123-4567 94105" + assert [(item.type, item.text) for item in datafog.scan(text).entities] == [ + ("PHONE", "555-123-4567"), + ("ZIP_CODE", "94105"), + ] + assert datafog.scan(text, entity_types=["PHONE"]).entities[0].text == "555-123-4567" + + +def test_adding_model_selection_does_not_drop_large_document_ssns(): + text = "ssn 123-45-6789 " * 3000 + result = datafog.scan(text, entity_types=["SSN", "PERSON"]) + assert len([item for item in result.entities if item.type == "SSN"]) == 3000 + assert all(text[item.start : item.end] == item.text for item in result.entities) + + +def test_international_email_and_person_punctuation(): + assert datafog.scan("mail jörg@müller.de").entities[0].text == "jörg@müller.de" + result = datafog.scan("Jane Doe,\n14 Penny Lane", entity_types=["PERSON"]) + assert result.entities and all( + item.text == item.text.rstrip(",;:.!?") for item in result.entities + ) + + +def test_german_context_prevents_generic_false_positive_labels(): + assert not datafog.scan("Steuer-ID 12345678901 liegt vor.").entities + assert not datafog.scan("IBAN DE44 5001 0517 5407 3249 31 ist gueltig.").entities diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py index 96901cf2..d36cc6af 100644 --- a/tests/test_de_pii_regex.py +++ b/tests/test_de_pii_regex.py @@ -134,12 +134,17 @@ def test_redaction_and_service_locale_support() -> None: def test_german_vat_redaction_suppresses_inner_generic_ssn_match( text: str, vat_text: str ) -> None: - # Default (no locale): v4.4.0 parity — the bare nine-digit run still - # matches the generic SSN pattern even when prefixed by a country code. - scan_result = scan(text, engine="regex") + # Default (no locale): the Python engine keeps v4.4.0 parity and still + # matches the inner nine-digit run. The native engine rejects an SSN that is + # embedded in an alphanumeric token, a reviewed detector difference. + scan_result = scan(text, engine="regex", backend="python") assert [(entity.type, entity.text) for entity in scan_result.entities] == [ ("SSN", "123456789") ] + native_expected = [] if "DE1" in text else [("SSN", "123456789")] + assert [ + (entity.type, entity.text) for entity in scan(text, engine="regex").entities + ] == native_expected # German locale: the longer DE_VAT_ID span wins via the engine's # span-overlap suppression, so the inner SSN match is dropped. @@ -148,8 +153,12 @@ def test_german_vat_redaction_suppresses_inner_generic_ssn_match( ("DE_VAT_ID", vat_text) ] - default_redaction = scan_and_redact(text, engine="regex") + default_redaction = scan_and_redact(text, engine="regex", backend="python") assert default_redaction.redacted_text == text.replace("123456789", "[SSN_1]") + native_redaction = scan_and_redact(text, engine="regex").redacted_text + assert native_redaction == ( + text if native_expected == [] else text.replace("123456789", "[SSN_1]") + ) redaction = scan_and_redact(text, engine="regex", locales=["de"]) assert redaction.redacted_text == text.replace(vat_text, "[DE_VAT_ID_1]") @@ -157,7 +166,10 @@ def test_german_vat_redaction_suppresses_inner_generic_ssn_match( def test_top_level_helpers_and_supported_entities_respect_locale() -> None: default_entities = get_supported_entities() - assert all(not entity.startswith("DE_") for entity in default_entities) + assert {"EMAIL", "DE_VAT_ID", "DE_IBAN", "JWT", "PERSON"} <= set(default_entities) + assert "JWT" not in get_supported_entities(backend="python") + # Supported inventory includes opt-in labels; scanning still requires locale activation. + assert "DE_TAX_ID" in default_entities german_entities = get_supported_entities(locales=["de"]) assert "DE_VAT_ID" in german_entities diff --git a/tests/test_detection_accuracy.py b/tests/test_detection_accuracy.py index 61dec7ba..8cff1e82 100644 --- a/tests/test_detection_accuracy.py +++ b/tests/test_detection_accuracy.py @@ -281,10 +281,13 @@ def _canon_type(entity_type: str) -> str: def _extract_entities( - text: str, engine: str, locales: list[str] | None = None + text: str, + engine: str, + locales: list[str] | None = None, + backend: str | None = None, ) -> list[dict[str, Any]]: try: - result = scan(text=text, engine=engine, locales=locales) + result = scan(text=text, engine=engine, locales=locales, backend=backend) except (ImportError, EngineNotAvailable) as exc: pytest.skip(f"{engine} engine unavailable in this environment: {exc}") @@ -336,6 +339,44 @@ def _required_expected( return expected_list +def _corpus_backend(engine: str | None = None) -> str: + """Backend under test for the frozen corpus, settable from the CLI. + + The default is ``python`` because the corpus expectations were authored + against that engine. Set ``DATAFOG_ACCURACY_BACKEND=rust`` to score the + native engine; its known divergences are then skipped through + ``tests/contracts/native-corpus.json``. + """ + import os + + value = os.environ.get("DATAFOG_ACCURACY_BACKEND", "python").strip().lower() + if engine is not None and value == "rust" and engine != "regex": + # The native path serves regex plus embedded-model findings; other + # engines are Python ML paths by definition. + value = "python" + if value not in {"python", "rust"}: + raise ValueError("DATAFOG_ACCURACY_BACKEND must be python or rust") + return "python" if value == "rust" and engine != "regex" else value + + +@lru_cache(maxsize=1) +def _native_deltas() -> dict[str, Any]: + path = Path("tests/contracts/native-corpus.json") + if not path.exists(): + return {"core_version": None, "cases": {}} + return json.loads(path.read_text(encoding="utf-8")) + + +def _native_missing(case_id: str) -> list[list[str]]: + """Findings the native engine is known to miss, asserted only as recorded.""" + return _native_deltas()["cases"].get(case_id, {}).get("missing", []) + + +def _native_offsets_shifted(case_id: str) -> bool: + """True when the native engine finds the value with different boundaries.""" + return bool(_native_deltas()["extra"].get(case_id, {}).get("offsets_shifted")) + + def _xfail_if_known_limitation( case: dict[str, Any], engine: str, corpus_kind: str ) -> None: @@ -346,11 +387,19 @@ def _xfail_if_known_limitation( def _assert_expected_found( - case: dict[str, Any], engine: str, corpus_kind: str + case: dict[str, Any], engine: str, corpus_kind: str, backend: str | None = None ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: text = case["input"] - actual = _extract_entities(text, engine, locales=case.get("locales")) + actual = _extract_entities( + text, engine, locales=case.get("locales"), backend=backend + ) expected = _required_expected(case["expected_entities"], engine, corpus_kind) + if (backend or _corpus_backend(engine)) == "rust": + expected = [ + entry + for entry in expected + if [entry["type"], entry["text"]] not in _native_missing(case["id"]) + ] for exp in expected: exp_type = _canon_type(exp["type"]) @@ -358,6 +407,15 @@ def _assert_expected_found( matches = [ ent for ent in actual if ent["type"] == exp_type and ent["text"] == exp_text ] + if _native_offsets_shifted(case["id"]): + # Same entity, different span: the native boundary rules treat the + # surrounding characters differently. Overlap is good enough. + matches = [ + ent + for ent in actual + if ent["type"] == exp_type + and not (ent["end"] <= exp["start"] or ent["start"] >= exp["end"]) + ] if not matches: matches = [ ent @@ -373,7 +431,9 @@ def _assert_expected_found( # If offsets are available from the engine output, validate exact position. with_offsets = [m for m in matches if m["start"] >= 0 and m["end"] >= 0] if with_offsets: - if engine == "regex" or exp_type in STRUCTURED_TYPES: + if ( + engine == "regex" or exp_type in STRUCTURED_TYPES + ) and not _native_offsets_shifted(case["id"]): assert any( m["start"] == exp["start"] and m["end"] == exp["end"] for m in with_offsets @@ -393,6 +453,33 @@ def _assert_expected_found( return actual, expected +def _unexpected_negative_entities( + case: dict[str, Any], actual: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Keep legacy negative PII checks while recognizing two reviewed URL spans. + + PY-M14 restores GLiNER's URL output. Only exact, source-aligned URL candidates + from that model are permitted; other labels and malformed offsets still fail. + These added label checks are outside the historical PII metric inventory. + """ + permitted = case.get("permitted_gliner_entities", []) + return [ + entity + for entity in actual + if not ( + entity.get("engine") == "gliner" + and any( + all( + entity.get(key) == span[key] + for key in ("type", "text", "start", "end") + ) + and case["input"][span["start"] : span["end"]] == span["text"] + for span in permitted + ) + ) + ] + + def _compute_metrics( engines: list[str], corpora: list[tuple[str, list[dict[str, Any]]]] ) -> dict[str, Any]: @@ -406,8 +493,13 @@ def _compute_metrics( for corpus_kind, cases in corpora: for case in cases: actual = _extract_entities( - case["input"], engine, locales=case.get("locales") + case["input"], + engine, + locales=case.get("locales"), + backend=_corpus_backend(engine), ) + if corpus_kind == "negative": + actual = _unexpected_negative_entities(case, actual) expected = _required_expected( case["expected_entities"], engine, corpus_kind ) @@ -475,7 +567,7 @@ def _prf(scores: dict[str, int]) -> dict[str, float]: @pytest.mark.parametrize("engine", FAST_ENGINES) def test_structured_pii_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "structured") - _assert_expected_found(case, engine, "structured") + _assert_expected_found(case, engine, "structured", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -485,7 +577,7 @@ def test_structured_pii_detection_fast(case: dict[str, Any], engine: str) -> Non @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_structured_pii_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "structured") - _assert_expected_found(case, engine, "structured") + _assert_expected_found(case, engine, "structured", backend=_corpus_backend(engine)) @pytest.mark.parametrize( @@ -494,7 +586,19 @@ def test_structured_pii_detection_slow(case: dict[str, Any], engine: str) -> Non @pytest.mark.parametrize("engine", FAST_ENGINES) def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "negative") - actual = _extract_entities(case["input"], engine, locales=case.get("locales")) + actual = _extract_entities( + case["input"], + engine, + locales=case.get("locales"), + backend=_corpus_backend(engine), + ) + if _corpus_backend(engine) == "rust": + added = { + tuple(pair) + for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) + } + actual = [e for e in actual if (e["type"], e["text"]) not in added] + actual = _unexpected_negative_entities(case, actual) assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -505,7 +609,19 @@ def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "negative") - actual = _extract_entities(case["input"], engine, locales=case.get("locales")) + actual = _extract_entities( + case["input"], + engine, + locales=case.get("locales"), + backend=_corpus_backend(engine), + ) + if _corpus_backend(engine) == "rust": + added = { + tuple(pair) + for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) + } + actual = [e for e in actual if (e["type"], e["text"]) not in added] + actual = _unexpected_negative_entities(case, actual) assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -515,7 +631,9 @@ def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", ["smart"]) def test_unstructured_pii_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "unstructured") - _assert_expected_found(case, engine, "unstructured") + _assert_expected_found( + case, engine, "unstructured", backend=_corpus_backend(engine) + ) @pytest.mark.slow @@ -525,14 +643,16 @@ def test_unstructured_pii_detection_fast(case: dict[str, Any], engine: str) -> N @pytest.mark.parametrize("engine", ["gliner", "spacy"]) def test_unstructured_pii_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "unstructured") - _assert_expected_found(case, engine, "unstructured") + _assert_expected_found( + case, engine, "unstructured", backend=_corpus_backend(engine) + ) @pytest.mark.parametrize("case", load_corpus("mixed_pii.json"), ids=lambda c: c["id"]) @pytest.mark.parametrize("engine", FAST_ENGINES) def test_mixed_pii_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "mixed") - _assert_expected_found(case, engine, "mixed") + _assert_expected_found(case, engine, "mixed", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -540,14 +660,14 @@ def test_mixed_pii_detection_fast(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_mixed_pii_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "mixed") - _assert_expected_found(case, engine, "mixed") + _assert_expected_found(case, engine, "mixed", backend=_corpus_backend(engine)) @pytest.mark.parametrize("case", load_corpus("edge_cases.json"), ids=lambda c: c["id"]) @pytest.mark.parametrize("engine", FAST_ENGINES) def test_edge_case_detection_fast(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "edge") - _assert_expected_found(case, engine, "edge") + _assert_expected_found(case, engine, "edge", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -555,7 +675,7 @@ def test_edge_case_detection_fast(case: dict[str, Any], engine: str) -> None: @pytest.mark.parametrize("engine", SLOW_ENGINES) def test_edge_case_detection_slow(case: dict[str, Any], engine: str) -> None: _xfail_if_known_limitation(case, engine, "edge") - _assert_expected_found(case, engine, "edge") + _assert_expected_found(case, engine, "edge", backend=_corpus_backend(engine)) @pytest.mark.slow @@ -577,3 +697,36 @@ def test_accuracy_metrics_snapshot() -> None: output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(json.dumps(metrics, indent=2), encoding="utf-8") assert "overall" in metrics and metrics["overall"] + + +def test_reviewed_url_candidates_require_exact_label_origin_and_offsets(): + case = next( + case + for case in load_corpus("negative_cases.json") + if case["id"] == "markdown-link" + ) + permitted = dict(case["permitted_gliner_entities"][0], engine="gliner") + assert _unexpected_negative_entities(case, [permitted]) == [] + for altered in [ + dict(permitted, type="EMAIL"), + dict(permitted, type="PHONE"), + dict(permitted, type="IP_ADDRESS"), + dict(permitted, start=14), + dict(permitted, end=41), + dict(permitted, engine="spacy"), + dict(permitted, text="example.com"), + ]: + assert _unexpected_negative_entities(case, [altered]) == [altered] + + +def test_reviewed_url_permission_is_confined_to_the_two_named_cases(): + permitted_cases = [ + case + for case in load_corpus("negative_cases.json") + if case.get("permitted_gliner_entities") + ] + assert {case["id"] for case in permitted_cases} == {"markdown-link", "url-with-at"} + for case in permitted_cases: + assert case["permitted_gliner_reason"].startswith("PY-M14") + assert len(case["permitted_gliner_entities"]) == 1 + assert case["permitted_gliner_entities"][0]["type"] == "URL" diff --git a/tests/test_donut_lazy_import.py b/tests/test_donut_lazy_import.py index 9b2a28f1..aab7bcfc 100644 --- a/tests/test_donut_lazy_import.py +++ b/tests/test_donut_lazy_import.py @@ -20,8 +20,7 @@ def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]: def test_no_torch_import_when_donut_disabled(): """Test that torch is not imported when use_donut is False""" - _run_isolated_python( - """ + _run_isolated_python(""" import sys from datafog.services.image_service import ImageService @@ -29,8 +28,7 @@ def test_no_torch_import_when_donut_disabled(): assert "torch" not in sys.modules assert "transformers" not in sys.modules -""" - ) +""") def test_lazy_import_mechanism(): @@ -39,8 +37,7 @@ def test_lazy_import_mechanism(): # to use lazy imports. We don't need to actually test the imports themselves, # just that the structure is correct. - _run_isolated_python( - """ + _run_isolated_python(""" import sys from datafog.processing.image_processing.donut_processor import DonutProcessor @@ -50,5 +47,4 @@ def test_lazy_import_mechanism(): assert "transformers" not in sys.modules assert hasattr(processor, "extract_text_from_image") assert not hasattr(processor, "ensure_installed") -""" - ) +""") diff --git a/tests/test_engine_api.py b/tests/test_engine_api.py index 2a43e147..35aebe2a 100644 --- a/tests/test_engine_api.py +++ b/tests/test_engine_api.py @@ -198,7 +198,7 @@ def test_redact_invalid_strategy_raises_value_error() -> None: redact("test", entities=[], strategy="invalid") -def test_redact_ignores_invalid_spans() -> None: +def test_redact_rejects_invalid_spans() -> None: text = "hello" entities = [ Entity( @@ -219,9 +219,8 @@ def test_redact_ignores_invalid_spans() -> None: ), ] - result = redact(text=text, entities=entities, strategy="token") - assert result.redacted_text == text - assert result.mapping == {} + with pytest.raises(ValueError, match="valid ranges"): + redact(text=text, entities=entities, strategy="token") def test_scan_and_redact_combines_operations() -> None: diff --git a/tests/test_gliner_annotator.py b/tests/test_gliner_annotator.py index fbf0bbdf..b9a82ad5 100644 --- a/tests/test_gliner_annotator.py +++ b/tests/test_gliner_annotator.py @@ -11,8 +11,8 @@ import pytest -@pytest.fixture(scope="session", autouse=True) -def mock_gliner_module(): +@pytest.fixture(autouse=True) +def mock_gliner_module(monkeypatch): """Mock the gliner module for all tests.""" # Create a mock gliner module mock_gliner = MagicMock() @@ -20,22 +20,42 @@ def mock_gliner_module(): # Configure the mock model behavior mock_model = MagicMock() - mock_model.predict_entities.return_value = [ - {"label": "person", "text": "John Doe", "start": 0, "end": 8}, - {"label": "email", "text": "john@example.com", "start": 20, "end": 36}, - {"label": "phone number", "text": "555-123-4567", "start": 50, "end": 62}, - ] + + def predict(text, labels): + findings = [] + for label, value in [ + ("person", "John Doe"), + ("email", "john@example.com"), + ("phone number", "555-123-4567"), + ]: + start = text.find(value) + if start >= 0 and label in labels: + findings.append( + { + "label": label, + "text": value, + "start": start, + "end": start + len(value), + } + ) + return findings + + mock_model.predict_entities.side_effect = predict mock_gliner_class.from_pretrained.return_value = mock_model mock_gliner.GLiNER = mock_gliner_class - # Add to sys.modules to make import work - sys.modules["gliner"] = mock_gliner + from datafog import engine + gliner_factory, spacy_factory = ( + engine._get_gliner_annotator, + engine._get_spacy_annotator, + ) + gliner_factory.cache_clear() + spacy_factory.cache_clear() + monkeypatch.setitem(sys.modules, "gliner", mock_gliner) yield mock_gliner_class, mock_model - - # Cleanup - if "gliner" in sys.modules: - del sys.modules["gliner"] + gliner_factory.cache_clear() + spacy_factory.cache_clear() class TestGLiNERAnnotatorWithDependencies: @@ -119,15 +139,9 @@ def test_gliner_annotate_long_text(self, mock_gliner_module): # Create text longer than MAXIMAL_STRING_SIZE long_text = "A" * 1000001 # Exceeds 1M character limit - with patch( - "datafog.processing.text_processing.gliner_annotator.logging" - ) as mock_logging: - result = annotator.annotate(long_text) - - # Should log a warning about truncation - mock_logging.warning.assert_called_once() - # Should still return a valid result - assert isinstance(result, dict) + with pytest.raises(ValueError, match="exceeds 1000000"): + annotator.annotate(long_text) + mock_model.predict_entities.assert_not_called() def test_gliner_download_model(self, mock_gliner_module): """Test GLiNER model download functionality.""" @@ -326,21 +340,19 @@ def test_text_service_gliner_engine_without_dependencies(self): ): TextService(engine="gliner") - def test_text_service_smart_engine_without_dependencies(self): - """Test smart engine degrades gracefully when GLiNER dependencies are missing.""" + def test_text_service_smart_engine_without_dependencies(self, monkeypatch): + """Missing optional engines preserve the structured native layer.""" + from datafog import engine from datafog.services.text_service import TextService - with patch.object(TextService, "_create_gliner_annotator", return_value=None): - with patch.object( - TextService, "_create_spacy_annotator", return_value=None - ): - service = TextService(engine="smart") - with pytest.warns(UserWarning, match="GLiNER not available"): - result = service.annotate_text_sync( - "John Doe from Acme Corporation needs follow up." - ) - assert "EMAIL" in result - assert result["EMAIL"] == [] + unavailable = engine._UnavailableAnnotator("optional model unavailable") + monkeypatch.setattr(engine, "_get_gliner_annotator", lambda: unavailable) + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: unavailable) + with pytest.warns(UserWarning, match="GLiNER unavailable"): + result = TextService(engine="smart").annotate_text_sync( + "mail john@example.com" + ) + assert result == {"EMAIL": ["john@example.com"]} def test_text_service_valid_engines(self): """Test that all valid engines are accepted.""" @@ -382,10 +394,10 @@ def test_text_service_valid_engines(self): assert service.engine == engine def test_text_service_invalid_engine(self): - """Test that invalid engines raise AssertionError.""" + """Test that invalid engines raise a configuration error.""" from datafog.services.text_service import TextService - with pytest.raises(AssertionError, match="Invalid engine"): + with pytest.raises(ValueError, match="engine must be one of"): TextService(engine="invalid_engine") @pytest.mark.parametrize( @@ -414,30 +426,14 @@ def test_cascade_should_stop_logic(self, engine, expected_count): result_above_threshold = {"TYPE1": ["entity1"] * (expected_count + 1)} assert service._cascade_should_stop(engine, result_above_threshold) - def test_smart_cascade_flow(self, mock_gliner_annotator): - """Test the smart cascading flow.""" + def test_smart_engine_combines_gliner_with_structured_detection(self): + """GLiNER adds exact NER spans without removing structured findings.""" from datafog.services.text_service import TextService - # Inject annotators directly to keep this cascade test deterministic - # across Python versions and import ordering. - mock_regex = Mock() - mock_regex.annotate.return_value = {"EMAIL": []} - - mock_spacy = Mock() - mock_spacy.annotate.return_value = {"PERSON": ["John Doe"]} - - service = TextService(engine="smart") - service._regex_annotator = mock_regex - service._gliner_annotator = mock_gliner_annotator - service._gliner_import_attempted = True - service._spacy_annotator = mock_spacy - service._spacy_import_attempted = True - - service.annotate_text_sync("John Doe works at john@example.com") - - # Should have tried regex first, then GLiNER. - mock_regex.annotate.assert_called_once() - mock_gliner_annotator.annotate.assert_called_once() + text = "John Doe owns 6011111111111117 and john@example.com" + result = TextService(engine="smart").annotate_text_sync(text, structured=True) + assert {item.label for item in result} == {"PERSON", "CREDIT_CARD", "EMAIL"} + assert all(text[item.start : item.end] == item.text for item in result) # Test CLI updates as well diff --git a/tests/test_image_service.py b/tests/test_image_service.py index da8e6deb..03d472b1 100644 --- a/tests/test_image_service.py +++ b/tests/test_image_service.py @@ -3,10 +3,11 @@ import asyncio import pytest -from PIL import Image from datafog.services.image_service import ImageService +Image = pytest.importorskip("PIL.Image", reason="Image tests require datafog[ocr]") + urls = [ "https://www.pdffiller.com/preview/101/35/101035394.png", "https://www.pdffiller.com/preview/435/972/435972694.png", diff --git a/tests/test_legacy_retirement.py b/tests/test_legacy_retirement.py index f5dd80e3..ed5910a0 100644 --- a/tests/test_legacy_retirement.py +++ b/tests/test_legacy_retirement.py @@ -29,7 +29,10 @@ def unexpected_import(*args): warnings.simplefilter("error", LegacySurfaceWarning) with pytest.raises( LegacySurfaceWarning, - match=f"{surface} support is deprecated in 4.9 and will be removed in 5.0", + match=( + f"{surface} support is deprecated in 4.9 and will be removed in 5.0. " + 'Last supported version is 4.12.0. Pin it with: pip install "datafog==4.12.0".' + ), ): (ImageService if surface == "OCR" else SparkService)() @@ -40,7 +43,7 @@ def unavailable(name): monkeypatch.setattr(importlib, "import_module", unavailable) with ( - pytest.warns(LegacySurfaceWarning, match="final 4.x release"), + pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.12.0"), pytest.raises(ImportError, match=r"datafog\[distributed\]"), ): SparkService() @@ -82,7 +85,7 @@ def test_donut_mock_output_is_preserved(monkeypatch): monkeypatch.setattr(donut_processor, "DONUT_TESTING_ENABLED", False) with pytest.warns(LegacySurfaceWarning): processor = DonutProcessor() - with pytest.warns(LegacySurfaceWarning, match="final 4.x release"): + with pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.12.0"): assert asyncio.run(processor.extract_text_from_image(object())) == ( '{"text": "Mock OCR text for testing"}' ) @@ -242,7 +245,7 @@ def test_image_cli_notice_visible_under_default_python_filters(image_argument): args.append("image.png") result = CliRunner().invoke(client.app, args) assert "deprecated in 4.9 and will be removed in 5.0" in result.stderr, result.output -assert "final 4.x release" in result.stderr, result.output +assert 'Last supported version is 4.12.0' in result.stderr, result.output if sys.argv[1] == "True": assert result.exit_code == 0, result.output assert "OCR Pipeline Results: ['OCR text']" in result.stdout diff --git a/tests/test_litellm_guardrail.py b/tests/test_litellm_guardrail.py index 1d7656a4..c4c1d697 100644 --- a/tests/test_litellm_guardrail.py +++ b/tests/test_litellm_guardrail.py @@ -152,13 +152,14 @@ async def test_post_call_fail_open_returns_unredacted_response(self, monkeypatch @pytest.mark.asyncio class TestEdgeShapes: - async def test_data_without_messages_passes_through(self): + async def test_embedding_input_is_redacted(self): guardrail = DataFogGuardrail(guardrail_name="datafog-pii") data = {"input": f"embed {EMAIL}"} result = await guardrail.async_pre_call_hook( user_api_key_dict=None, cache=None, data=data, call_type="aembedding" ) - assert result == data + assert EMAIL not in result["input"] + assert result["input"].startswith("embed [EMAIL_") async def test_message_without_content_key_passes_through(self): guardrail = DataFogGuardrail(guardrail_name="datafog-pii") diff --git a/tests/test_no_network_core.py b/tests/test_no_network_core.py index f06e8360..cbdf8c08 100644 --- a/tests/test_no_network_core.py +++ b/tests/test_no_network_core.py @@ -20,8 +20,7 @@ def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]: def test_import_scan_and_redact_do_not_open_network() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import socket import urllib.request @@ -39,8 +38,7 @@ def blocked(*_args, **_kwargs): redact_result = datafog.redact("Email jane@example.com or call 415-555-1212") assert "jane@example.com" not in redact_result.redacted_text assert "415-555-1212" not in redact_result.redacted_text -""" - ) +""") def test_core_defaults_do_not_initialize_optional_engines(monkeypatch) -> None: @@ -78,8 +76,7 @@ def fail_optional_engine_probe(): def test_import_probes_do_not_load_optional_models() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import sys import types @@ -105,13 +102,11 @@ def from_pretrained(*_args, **_kwargs): import datafog assert datafog.scan("Email jane@example.com").entities -""" - ) +""") def test_core_path_does_not_import_optional_dependency_modules() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import importlib.abc import sys @@ -144,5 +139,4 @@ def find_spec(self, fullname, path=None, target=None): assert datafog.scan_prompt("Email jane@example.com").entities assert datafog.filter_output("Email jane@example.com").redacted_text == "Email [EMAIL_1]" assert datafog.create_guardrail().filter("Email jane@example.com").redacted_text == "Email [EMAIL_1]" -""" - ) +""") diff --git a/tests/test_ocr_integration.py b/tests/test_ocr_integration.py index 9ceef4a2..c7333dbd 100644 --- a/tests/test_ocr_integration.py +++ b/tests/test_ocr_integration.py @@ -9,11 +9,12 @@ from unittest.mock import patch import pytest -from PIL import Image from datafog.processing.image_processing.donut_processor import DonutProcessor from datafog.services.image_service import ImageService +Image = pytest.importorskip("PIL.Image", reason="OCR tests require datafog[ocr]") + # Mark all tests in this file as integration tests pytestmark = pytest.mark.integration diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py new file mode 100644 index 00000000..5e44c461 --- /dev/null +++ b/tests/test_review_engine.py @@ -0,0 +1,360 @@ +"""Regression contracts for the 4.11 review's engine and annotation findings.""" + +import hashlib +import hmac +import random +from types import SimpleNamespace + +import pytest + +from datafog import engine +from datafog.processing.text_processing.gliner_annotator import GLiNERAnnotator +from datafog.processing.text_processing.regex_annotator.regex_annotator import ( + RegexAnnotator, +) +from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator +from datafog.services.text_service import TextService + + +def test_native_surrogate_normalization_preserves_source_and_offsets(): + text = "\ud800😀 ssn 123-45-6789 \udfff" + result = engine.scan(text, engine="regex", backend="rust") + assert result.text == text + assert any(entity.type == "SSN" for entity in result.entities) + assert all(text[item.start : item.end] == item.text for item in result.entities) + assert "123-45-6789" not in engine.redact(text, result.entities).redacted_text + + +def test_interval_selection_preserves_longest_and_tiebreak_contract(): + rng = random.Random(113) + for _ in range(30): + entities = [ + engine.Entity( + rng.choice(["PHONE", "SSN"]), + "", + start, + start + rng.randint(1, 30), + rng.random(), + "regex", + ) + for start in [rng.randint(0, 200) for _ in range(100)] + ] + expected = [] + for entity in sorted( + entities, + key=lambda item: ( + -engine._entity_length(item), + -engine.ENTITY_TYPE_PRIORITY.get(item.type, 0), + -item.confidence, + item.start, + item.end, + item.type, + ), + ): + if not any(engine._entities_overlap(entity, kept) for kept in expected): + expected.append(entity) + assert engine._suppress_overlapping_entities(entities) == sorted( + expected, key=lambda item: (item.start, item.end, item.type) + ) + + +@pytest.mark.parametrize("structured", [False, True]) +@pytest.mark.parametrize("value", ["john.doe@example.com", "123-45-6789"]) +def test_text_service_retains_findings_at_chunk_boundaries(value, structured): + text = "x" * 996 + " " + value + " tail" + result = TextService(text_chunk_length=1000).annotate_text_sync(text, structured) + if structured: + assert any( + item.text == value and text[item.start : item.end] == value + for item in result + ) + else: + assert any(value in items for items in result.values()) + + +def test_spacy_uses_original_offsets_and_does_not_turn_relative_dates_into_pii( + monkeypatch, +): + text = "Annual report prepared by Ann yesterday." + spans = [ + SimpleNamespace(label_="PERSON", text="Ann", start_char=26, end_char=29), + SimpleNamespace(label_="DATE", text="yesterday", start_char=30, end_char=39), + ] + annotator = SpacyPIIAnnotator(nlp=lambda _: SimpleNamespace(ents=spans)) + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: annotator) + result = engine.scan_and_redact(text, engine="spacy") + assert result.redacted_text == "Annual report prepared by [PERSON_1] yesterday." + + +def test_legacy_string_annotator_does_not_match_names_inside_words(): + result = engine._entities_from_dict( + "Annual report by Ann.", {"PERSON": ["Ann"]}, "spacy", 0.7 + ) + assert [(item.start, item.end) for item in result] == [(17, 20)] + + +@pytest.mark.parametrize("mode", ["smart", "spacy", "gliner"]) +def test_every_engine_keeps_native_structured_detection(monkeypatch, mode): + monkeypatch.setattr(engine, "_gliner_entities", lambda _: []) + monkeypatch.setattr(engine, "_spacy_entities", lambda _: []) + text = "card 6011111111111117 ip 2001:db8::1" + result = engine.scan_and_redact(text, engine=mode, backend="rust") + assert {item.type for item in result.entities} == {"CREDIT_CARD", "IP_ADDRESS"} + assert "6011111111111117" not in result.redacted_text + assert "2001:db8::1" not in result.redacted_text + + +@pytest.mark.parametrize("labels", ["EMAIL", ["BOGUS"], [1], [""]]) +def test_entity_type_mistakes_fail_explicitly(labels): + with pytest.raises(ValueError, match="entity_types"): + engine.scan("john@example.com", entity_types=labels) + + +@pytest.mark.parametrize("label", ["JWT", "PERSON", "ORGANIZATION", "UUID"]) +def test_unavailable_fallback_labels_raise(label): + with pytest.raises(ValueError, match="Unsupported entity_types"): + engine.scan( + "john@example.com", engine="regex", backend="python", entity_types=[label] + ) + + +@pytest.mark.parametrize("pattern", [r"(a|a)+$", r"((a+))+$"]) +def test_pathological_allowlist_cannot_suppress_a_finding(pattern): + value = "a" * 300 + "@b.com" + entity = engine.Entity("EMAIL", value, 0, len(value), 1.0, "regex") + assert engine._apply_allowlist([entity], None, [pattern]) == [entity] + + +@pytest.mark.parametrize( + "pattern,value", + [(r"(\d{3}){2}", "123123"), (r"(?:\d{3}-){2}\d{4}", "415-555-2671")], +) +def test_safe_quantified_allowlist_patterns_are_supported(pattern, value): + entity = engine.Entity("PHONE", value, 0, len(value), 1.0, "regex") + assert engine._apply_allowlist([entity], None, [pattern]) == [] + + +@pytest.mark.parametrize( + "annotator", + [ + SpacyPIIAnnotator( + nlp=lambda _: (_ for _ in ()).throw(RuntimeError("raw secret")) + ), + GLiNERAnnotator( + model=SimpleNamespace( + predict_entities=lambda *_: (_ for _ in ()).throw( + RuntimeError("raw secret") + ) + ), + entity_types=["person"], + model_name="test", + ), + ], +) +def test_ner_failures_and_oversize_inputs_are_explicit(annotator): + with pytest.raises(RuntimeError, match="annotation failed"): + annotator.annotate("secret") + with pytest.raises(ValueError, match="exceeds"): + annotator.annotate("x" * 1_000_001) + + +@pytest.mark.parametrize( + "label", ["passport number", "medical record number", "license number", "url"] +) +def test_gliner_extra_labels_and_original_offsets_are_preserved(monkeypatch, label): + text = "prefix SECRET suffix" + annotator = GLiNERAnnotator( + model=SimpleNamespace( + predict_entities=lambda *_: [ + {"label": label, "text": "SECRET", "start": 7, "end": 13} + ] + ), + entity_types=[label], + model_name="test", + ) + monkeypatch.setattr(engine, "_get_gliner_annotator", lambda: annotator) + result = engine.scan( + text, engine="gliner", entity_types=[label.upper().replace(" ", "_")] + ) + assert len(result.entities) == 1 + assert result.entities[0].text == "SECRET" + assert result.entities[0].start == 7 + + +def test_model_chunks_are_bounded_and_keep_document_offsets(): + text = "😀" * 9000 + " Jane Doe" + calls = [] + + def scan_model(chunk): + calls.append(chunk) + start = chunk.find("Jane Doe") + findings = ( + [] + if start < 0 + else [ + SimpleNamespace( + entity_type="PERSON", + codepoint_range=SimpleNamespace(start=start, end=start + 8), + matched_text="Jane Doe", + ) + ] + ) + return SimpleNamespace(findings=findings) + + findings = engine._cnn_findings( + SimpleNamespace(scan_model=scan_model), {"PERSON"}, text + ) + assert len(calls) >= 3 and max(len(chunk.encode()) for chunk in calls) <= 16384 + assert any( + offset + item.codepoint_range.start == text.index("Jane Doe") + for offset, item in findings + ) + + +def test_token_collision_and_repeated_value_mapping(): + text = "[EMAIL_1] a@b.com and a@b.com" + result = engine.scan_and_redact(text, engine="regex") + assert result.redacted_text == "[EMAIL_1] [EMAIL_2] and [EMAIL_2]" + assert result.mapping == {"[EMAIL_2]": "a@b.com"} + + +@pytest.mark.parametrize( + "entity", + [ + engine.Entity("EMAIL", "", 0, 1, 1.0, "regex"), + engine.Entity("EMAIL", "a", 0, 100, 1.0, "regex"), + engine.Entity("EMAIL", "wrong", 0, 1, 1.0, "regex"), + ], +) +def test_invalid_explicit_spans_are_rejected(entity): + with pytest.raises(ValueError, match="valid ranges"): + engine.redact("a", [entity]) + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256", "sha3_256"]) +def test_hash_algorithm_and_hmac_are_honored(algorithm): + text = "a@b.com" + entities = engine.scan(text, engine="regex").entities + plain = engine.redact(text, entities, strategy="hash", hash_type=algorithm) + assert hashlib.new(algorithm, text.encode()).hexdigest()[:12] in plain.redacted_text + key = bytes(range(32)) + keyed = engine.redact( + text, entities, strategy="hash", hash_type=algorithm, hash_key=key + ) + assert hmac.new(key, text.encode(), algorithm).hexdigest() in keyed.redacted_text + assert plain != keyed + + +def test_invalid_hash_parameters_fail(): + with pytest.raises(ValueError, match="hash_key"): + engine.redact("text", [], strategy="hash", hash_key=b"short") + with pytest.raises(ValueError, match="hash_type"): + engine.redact("text", [], strategy="hash", hash_type="bogus") + + +@pytest.mark.parametrize("locales", [[""], [1]]) +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_invalid_locales_are_consistently_rejected(locales, backend): + with pytest.raises(ValueError, match="locale"): + engine.scan("safe", engine="regex", locales=locales, backend=backend) + + +@pytest.mark.parametrize("locale", ["fr", "en-US"]) +def test_fallback_accepts_locale_tags_without_extra_detectors(locale): + assert engine.scan( + "a@b.com", engine="regex", locales=[locale], backend="python" + ).entities + + +def test_unicode_email_in_fallback(): + result = engine.scan("mail jörg@müller.de", engine="regex", backend="python") + assert result.entities[0].text == "jörg@müller.de" + + +@pytest.mark.parametrize( + "options", + [ + {"engine": "bogus"}, + {"text_chunk_length": 0}, + {"text_chunk_length": -1}, + {"text_chunk_length": True}, + ], +) +def test_text_service_argument_validation(options): + with pytest.raises(ValueError): + TextService(**options) + + +def test_fallback_adversarial_patterns_complete(): + annotator = RegexAnnotator(locales=["de"]) + for value in [ + "!" * 80000, + "#a" * 40000, + "Steuer-ID" + " " * 80000 + "x", + "Passport" + " " * 80000, + ]: + assert not any(annotator.annotate(value).values()) + + +def test_hash_marker_cannot_collide_with_literal_input(): + value = "a@b.com" + digest = hashlib.sha256(value.encode()).hexdigest()[:12] + marker = f"[EMAIL_{digest}]" + text = f"{marker} {value} {value}" + result = engine.scan_and_redact(text, engine="regex", strategy="hash") + assert result.mapping == {f"[EMAIL_{digest}_2]": value} + assert result.redacted_text == f"{marker} [EMAIL_{digest}_2] [EMAIL_{digest}_2]" + + +@pytest.mark.parametrize( + "text", + [ + "?" * 11 + "xiaoming@example.cn", + '{"note":"email=alice@example.com"}', + "email = 'dev@example.com'", + "phone=415-555-2671&email=bob@example.com", + ], +) +def test_fallback_email_context_is_trimmed_without_losing_mailboxes(text): + result = engine.scan(text, engine="regex", backend="python", entity_types=["EMAIL"]) + assert len(result.entities) == 1 + assert result.entities[0].text in { + "xiaoming@example.cn", + "alice@example.com", + "dev@example.com", + "bob@example.com", + } + + +def test_spacy_selection_rejects_gliner_only_labels(): + with pytest.raises(ValueError, match="PASSPORT_NUMBER"): + engine.scan("passport SECRET", engine="spacy", entity_types=["PASSPORT_NUMBER"]) + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256", "sha3_256"]) +def test_shared_hash_helper_retains_legacy_widths_and_full_hmac(algorithm): + value = "a@b.com" + digest = hashlib.new(algorithm, value.encode()).hexdigest() + assert engine._hash_value(value, algorithm, None) == digest[:12] + assert engine._hash_value(value, algorithm, None, plain_length=8) == digest[:8] + key = bytes(range(32)) + assert ( + engine._hash_value(value, algorithm, key, plain_length=8) + == hmac.new(key, value.encode(), algorithm).hexdigest() + ) + + +@pytest.mark.parametrize("mode", ["smart", "spacy", "gliner"]) +def test_optional_ner_cannot_remove_overlapping_structured_scan_findings( + monkeypatch, mode +): + text = "Ping 127.0.0.1" + candidates = [engine.Entity("PERSON", text, 0, len(text), 0.8, mode)] + monkeypatch.setattr(engine, "_spacy_entities", lambda _: candidates) + monkeypatch.setattr(engine, "_gliner_entities", lambda _: candidates) + result = engine.scan(text, engine=mode) + assert any( + item.type == "IP_ADDRESS" and item.text == "127.0.0.1" + for item in result.entities + ) + assert any(item.type == "PERSON" for item in result.entities) diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py new file mode 100644 index 00000000..519c9cf6 --- /dev/null +++ b/tests/test_review_entrypoints.py @@ -0,0 +1,270 @@ +"""Regression contracts for the November 2026 entrypoint review.""" + +import asyncio +import hashlib +import importlib.util +import logging +from pathlib import Path +from types import SimpleNamespace + +import pytest +from typer.testing import CliRunner + +import datafog +from datafog.agent import create_guardrail +from datafog.client import app +from datafog.config import OperationType, get_config +from datafog.main import DataFog, TextPIIAnnotator +from datafog.models.annotator import AnnotationResult, AnnotatorRequest +from datafog.models.anonymizer import Anonymizer, AnonymizerType, HashType +from datafog.models.common import EntityTypes + +EMAIL = "jane@" "example.com" + + +@pytest.mark.asyncio +async def test_decorator_awaits_and_filters(): + @create_guardrail() + async def answer(): + await asyncio.sleep(0) + return EMAIL + + assert EMAIL not in await answer() + + +@pytest.mark.parametrize("output", [[EMAIL], {"text": EMAIL}, None, 1]) +def test_guarded_non_strings_rejected(output): + @create_guardrail() + def answer(): + return output + + with pytest.raises(TypeError, match="must return a string"): + answer() + + +@pytest.mark.parametrize( + "operation", [OperationType.REDACT, OperationType.REPLACE, OperationType.HASH] +) +def test_anonymizing_operations_imply_scan(operation): + service = DataFog(operations=[operation]) + assert EMAIL not in service.run_text_pipeline_sync([EMAIL])[0] + assert EMAIL not in service.process_text(EMAIL) + + +@pytest.mark.parametrize("algorithm", list(HashType)) +def test_hash_algorithm_honored_across_api_and_cli(algorithm): + digest = hashlib.new(algorithm.value, EMAIL.encode()).hexdigest() + assert ( + f"[EMAIL_{digest[:12]}]" + == DataFog(hash_type=algorithm).process(EMAIL, anonymize=True, method="hash")[ + "anonymized" + ] + ) + assert ( + f"[EMAIL_{digest[:8]}]" + == datafog.process( + EMAIL, anonymize=True, method="hash", hash_type=algorithm.value + )["anonymized"] + ) + result = CliRunner().invoke( + app, ["hash-text", EMAIL, "--hash-type", algorithm.value] + ) + assert result.exit_code == 0, result.output + assert result.stdout.strip() == f"[EMAIL_{digest[:12]}]" + + +def test_existing_plain_hash_defaults_remain_compatible(): + assert ( + datafog.redact(EMAIL, strategy="hash").redacted_text + == f"[EMAIL_{hashlib.sha256(EMAIL.encode()).hexdigest()[:12]}]" + ) + assert ( + datafog.process(EMAIL, anonymize=True, method="hash")["anonymized"] + == f"[EMAIL_{hashlib.md5(EMAIL.encode()).hexdigest()[:8]}]" + ) + + +def test_keyed_hash_public_facades(): + import hmac + + key = b"k" * 32 + digest = hmac.new(key, EMAIL.encode(), "sha256").hexdigest() + assert digest in datafog.redact(EMAIL, strategy="hash", hash_key=key).redacted_text + assert digest in datafog.anonymize_text(EMAIL, method="hash", hash_key=key) + assert ( + digest + in datafog.process(EMAIL, anonymize=True, method="hash", hash_key=key)[ + "anonymized" + ] + ) + + +@pytest.mark.parametrize( + "factory", [lambda: datafog.process, lambda: DataFog().process] +) +def test_bad_method_rejected(factory): + with pytest.raises(ValueError, match="method"): + factory()(EMAIL, anonymize=True, method="bogus") + + +def test_legacy_detection_uses_core(): + text = "card 6011111111111117 ip 2001:db8::1" + assert {item["type"] for item in datafog.detect(text)} >= { + "CREDIT_CARD", + "IP_ADDRESS", + } + assert "6011111111111117" not in datafog.process(text, anonymize=True)["anonymized"] + assert "IP_ADDRESS" in TextPIIAnnotator().run(text) + assert "JWT" in datafog.get_supported_entities() + + +def test_annotation_defaults_and_labels(): + assert AnnotatorRequest(text="x", language="en").score_threshold is None + for label in ["SSN", "ORG", "GPE", "IP_ADDRESS"]: + assert ( + AnnotationResult(start=0, end=1, entity_type=label).entity_type != "UNKNOWN" + ) + + +def test_enum_filter_and_date_replacement(): + annotation = AnnotationResult( + start=0, end=len(EMAIL), entity_type=EntityTypes.EMAIL + ) + assert ( + Anonymizer(anonymizer_type="redact", entities=[EntityTypes.EMAIL]) + .anonymize(EMAIL, [annotation]) + .anonymized_text + == "[REDACTED]" + ) + date = AnnotationResult(start=0, end=10, entity_type="DATE") + assert ( + Anonymizer().anonymize("2024-01-02", [date]).anonymized_text + == "[REDACTED_DATE]" + ) + + +@pytest.mark.parametrize("strategy", list(AnonymizerType)) +def test_legacy_empty_entity_selection_keeps_protecting_all(strategy): + annotation = AnnotationResult(start=0, end=len(EMAIL), entity_type="EMAIL") + result = Anonymizer(anonymizer_type=strategy, entities=[]).anonymize( + EMAIL, [annotation] + ) + assert EMAIL not in result.anonymized_text + assert len(result.anonymized_entities) == 1 + + +@pytest.mark.parametrize("strategy", list(AnonymizerType)) +def test_legacy_overlaps_never_leave_partial_value(strategy): + text = "abcdefghij" + spans = [ + AnnotationResult(start=0, end=6, entity_type="EMAIL"), + AnnotationResult(start=4, end=10, entity_type="SSN"), + ] + result = Anonymizer(anonymizer_type=strategy).anonymize(text, spans) + assert "ghij" not in result.anonymized_text + assert len(result.anonymized_entities) == 1 + assert result.anonymized_entities[0]["original"] == text + + +def test_cli_hides_key_and_handles_bad_operation(): + config = get_config() + old_key = config.api_key + try: + config.api_key = "private-test-key" + result = CliRunner().invoke(app, ["show-config"]) + assert result.exit_code == 0 + assert "private-test-key" not in result.stdout + assert "REDACTED" in result.stdout + finally: + config.api_key = old_key + result = CliRunner().invoke(app, ["scan-text", EMAIL, "--operations", "bogus"]) + assert result.exit_code == 2 + assert "operations must use" in result.output + assert "Traceback" not in result.output + + +def test_import_and_cli_preserve_logger_level(): + logger = logging.getLogger("datafog_logger") + old = logger.level + try: + logger.setLevel(logging.ERROR) + import importlib + + import datafog.main + + importlib.reload(datafog.main) + assert logger.level == logging.ERROR + finally: + logger.setLevel(old) + + +def test_spark_real_spacy_labels_and_oversize(monkeypatch): + from datafog.processing.spark_processing import pyspark_udfs as spark + + monkeypatch.setattr(spark, "ensure_installed", lambda _: None) + + def nlp(text): + return SimpleNamespace( + ents=[ + SimpleNamespace(label_="PERSON", text="Ann"), + SimpleNamespace(label_="ORG", text="Acme"), + ] + ) + + with pytest.warns(FutureWarning): + result = spark.pii_annotator("Ann at Acme", SimpleNamespace(value=nlp)) + assert result[spark.PII_ANNOTATION_LABELS.index("PER")] == ["Ann"] + assert result[spark.PII_ANNOTATION_LABELS.index("ORG")] == ["Acme"] + with pytest.warns(FutureWarning), pytest.raises(ValueError, match="maximum"): + spark.pii_annotator( + "x" * (spark.MAXIMAL_STRING_SIZE + 1), SimpleNamespace(value=nlp) + ) + + +def _version_selector(): + path = Path(__file__).parents[1] / "scripts/release_version.py" + spec = importlib.util.spec_from_file_location("release_version", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.select_version + + +@pytest.mark.parametrize( + "current, kind, override", + [ + ("4.9.0a2", "alpha", None), + ("4.11.0", "beta", None), + ("4.12.0", "stable", "4.10.0"), + ], +) +def test_release_rejects_stale_bases(current, kind, override): + with pytest.raises(ValueError, match="newer than"): + _version_selector()(current, kind, [], "4.11.0", override) + + +def test_release_version_positive_and_duplicate_paths(): + select = _version_selector() + assert select("4.12.0", "alpha", ["v4.12.0a1"], "4.11.0") == "4.12.0a2" + assert select("4.12.0", "stable", [], "4.11.0") == "4.12.0" + with pytest.raises(ValueError, match="already exists"): + select("4.12.0", "stable", ["v4.12.0"], "4.11.0") + + +def test_release_skips_published_prereleases_without_tags(): + select = _version_selector() + assert select("4.12.0", "alpha", [], "4.11.0", published=["4.12.0a1"]) == "4.12.0a2" + + +def test_native_version_uses_import_error_when_metadata_is_missing(monkeypatch): + import importlib.metadata + + import datafog.native + + assert datafog.native.version() + + def missing(_): + raise importlib.metadata.PackageNotFoundError("datafog-core") + + monkeypatch.setattr(importlib.metadata, "version", missing) + with pytest.raises(ImportError, match="metadata is missing"): + datafog.native.version() diff --git a/tests/test_review_guardrails.py b/tests/test_review_guardrails.py new file mode 100644 index 00000000..ea6c4cbe --- /dev/null +++ b/tests/test_review_guardrails.py @@ -0,0 +1,434 @@ +"""Review regressions for request/response coverage and safe stream failures.""" + +# ruff: noqa: E402 +# Optional proxy dependencies must be present before importing the adapter. +import copy +import json + +import pytest + +pytest.importorskip("litellm") +pytest.importorskip("fastapi") +from fastapi import HTTPException # noqa: E402 +from litellm import ModelResponseStream # noqa: E402 +from litellm.types.utils import Delta, StreamingChoices # noqa: E402 + +import datafog # noqa: E402 +from datafog.integrations import claude_code as hook # noqa: E402 +from datafog.integrations import litellm_guardrail as gateway # noqa: E402 + +EMAIL_A = "alice@" "example.com" +EMAIL_B = "bob@" "example.com" +SSN = "123-45-" "6789" + + +def guard(**kwargs): + return gateway.DataFogGuardrail(guardrail_name="review", **kwargs) + + +async def pre(adapter, data): + return await adapter.async_pre_call_hook(None, None, data, "completion") + + +async def source(chunks): + for chunk in chunks: + yield chunk + + +async def streamed(adapter, chunks): + return [ + chunk + async for chunk in adapter.async_post_call_streaming_iterator_hook( + None, source(chunks), {} + ) + ] + + +@pytest.mark.asyncio +async def test_request_shares_tokens_without_existing_token_collisions(): + original = { + "messages": [ + {"role": "user", "content": f"[EMAIL_1] {EMAIL_A}"}, + {"role": "user", "content": f"other {EMAIL_B}; same {EMAIL_A}"}, + ], + "prompt": EMAIL_B, + "input": [{"content": EMAIL_A}], + } + before = copy.deepcopy(original) + result = await pre(guard(), original) + assert result["messages"][0]["content"] == "[EMAIL_1] [EMAIL_2]" + assert result["messages"][1]["content"] == "other [EMAIL_3]; same [EMAIL_2]" + assert ( + result["prompt"] == "[EMAIL_3]" and result["input"][0]["content"] == "[EMAIL_2]" + ) + assert original == before + another = await pre(guard(), {"prompt": EMAIL_A}) + assert another["prompt"] == "[EMAIL_1]" + + +@pytest.mark.asyncio +async def test_request_scans_tool_arguments_and_structured_input_values(): + data = { + "messages": [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "save", + "arguments": json.dumps({"email": EMAIL_A, "ssn": SSN}), + }, + } + ], + } + ], + "input": {"name": EMAIL_B, "id": SSN, "url": "send " + EMAIL_A}, + } + result = await pre(guard(), data) + assert ( + EMAIL_A not in result["messages"][0]["tool_calls"][0]["function"]["arguments"] + ) + assert SSN not in result["messages"][0]["tool_calls"][0]["function"]["arguments"] + assert result["input"] == { + "name": "[EMAIL_2]", + "id": "[SSN_1]", + "url": "send [EMAIL_1]", + } + assert result["messages"][0]["tool_calls"][0]["id"] == "call_1" + + +@pytest.mark.asyncio +async def test_multimodal_media_remains_intact_and_text_is_scanned(): + image = { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,AA4242424242424242", + "detail": "high", + }, + } + audio = { + "type": "input_audio", + "input_audio": {"data": "4242424242424242", "format": "wav"}, + } + result = await pre( + guard(), + { + "messages": [ + { + "role": "user", + "content": [image, audio, {"type": "text", "text": EMAIL_A}], + } + ] + }, + ) + assert result["messages"][0]["content"][:2] == [image, audio] + assert result["messages"][0]["content"][2]["text"] == "[EMAIL_1]" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["messages", "prompt", "input"]) +async def test_surrogate_cannot_disable_blocking(field): + value = f"ssn {SSN} \ud83d" + data = ( + {field: [{"role": "user", "content": value}]} + if field == "messages" + else {field: value} + ) + with pytest.raises(HTTPException) as error: + await pre(guard(action="block"), data) + assert error.value.status_code == 400 and SSN not in str(error.value.detail) + + +@pytest.mark.asyncio +async def test_response_reserves_tokens_in_tools_and_covers_legacy_and_text(): + response = { + "choices": [ + { + "index": 0, + "message": { + "content": EMAIL_A, + "tool_calls": [ + { + "function": { + "name": "save", + "arguments": f"[EMAIL_1] {EMAIL_B}", + } + } + ], + }, + }, + { + "index": 1, + "message": { + "content": None, + "function_call": {"name": "old", "arguments": EMAIL_A}, + }, + }, + {"index": 2, "text": EMAIL_B}, + ] + } + result = await guard().async_post_call_success_hook({}, None, response) + assert result is response + assert result["choices"][0]["message"]["content"] == "[EMAIL_2]" + assert ( + result["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] + == "[EMAIL_1] [EMAIL_3]" + ) + assert result["choices"][1]["message"]["function_call"]["arguments"] == "[EMAIL_2]" + assert result["choices"][2]["text"] == "[EMAIL_3]" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_response_scanning_is_atomic_and_respects_fail_policy( + monkeypatch, policy +): + actual = gateway._redact_text + + def failing(text, *args, **kwargs): + if text == "second": + raise RuntimeError(EMAIL_A) + return actual(text, *args, **kwargs) + + monkeypatch.setattr(gateway, "_redact_text", failing) + response = { + "choices": [ + {"message": {"content": EMAIL_A}}, + {"message": {"content": "second"}}, + ] + } + before = copy.deepcopy(response) + if policy == "closed": + with pytest.raises(RuntimeError) as error: + await guard(fail_policy=policy).async_post_call_success_hook( + {}, None, response + ) + assert EMAIL_A not in str(error.value) + else: + assert ( + await guard(fail_policy=policy).async_post_call_success_hook( + {}, None, response + ) + is response + ) + assert response == before + + +@pytest.mark.asyncio +async def test_stream_scans_all_choice_and_tool_argument_fragments(): + chunks = [ + { + "id": "first", + "choices": [ + { + "index": 0, + "delta": { + "content": "[EMAIL_1] alice@", + "tool_calls": [ + { + "index": 0, + "id": "call_a", + "function": { + "name": "save", + "arguments": '{"ssn":"123-', + }, + }, + { + "index": 1, + "id": "call_b", + "function": {"name": "save", "arguments": "bob@"}, + }, + ], + }, + }, + { + "index": 1, + "delta": { + "content": "bob@", + "function_call": {"name": "old", "arguments": "alice@"}, + }, + }, + {"index": 2, "text": "bob@"}, + ], + }, + { + "id": "last", + "choices": [ + { + "index": 0, + "delta": { + "content": "example.com", + "tool_calls": [ + {"index": 0, "function": {"arguments": '45-6789"}'}}, + {"index": 1, "function": {"arguments": "example.com"}}, + ], + }, + "finish_reason": "tool_calls", + }, + { + "index": 1, + "delta": { + "content": "example.com", + "function_call": {"arguments": "example.com"}, + }, + "finish_reason": "stop", + }, + {"index": 2, "text": "example.com", "finish_reason": "stop"}, + ], + }, + ] + original = copy.deepcopy(chunks) + output = await streamed(guard(), chunks) + first, last = output + assert first["choices"][0]["delta"]["content"] == "[EMAIL_1] [EMAIL_2]" + calls = first["choices"][0]["delta"]["tool_calls"] + assert calls[0]["function"]["arguments"] == '{"ssn":"[SSN_1]"}' + assert calls[1]["function"]["arguments"] == "[EMAIL_3]" + assert first["choices"][1]["delta"]["content"] == "[EMAIL_3]" + assert first["choices"][1]["delta"]["function_call"]["arguments"] == "[EMAIL_2]" + assert first["choices"][2]["text"] == "[EMAIL_3]" + assert last["choices"][0]["delta"]["content"] == "" + assert last["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] == "" + assert last["choices"][0]["finish_reason"] == "tool_calls" + assert chunks == original + assert first["id"] == "first" and last["id"] == "last" + + +@pytest.mark.asyncio +async def test_stream_handles_real_litellm_native_chunks(): + chunks = [ + ModelResponseStream( + id="same", choices=[StreamingChoices(index=0, delta=Delta(content=part))] + ) + for part in ("alice@", "example.com") + ] + output = await streamed(guard(), chunks) + assert output[0].choices[0].delta.content == "[EMAIL_1]" + assert output[1].choices[0].delta.content == "" + assert chunks[0].choices[0].delta.content == "alice@" + + +@pytest.mark.asyncio +async def test_stream_snapshots_reused_mutable_chunks(): + async def reused(): + chunk = {"choices": [{"index": 0, "delta": {"content": "alice@"}}]} + yield chunk + chunk["choices"][0]["delta"]["content"] = "example.com" + yield chunk + + output = [ + chunk + async for chunk in guard().async_post_call_streaming_iterator_hook( + None, reused(), {} + ) + ] + assert output[0]["choices"][0]["delta"]["content"] == "[EMAIL_1]" + assert output[1]["choices"][0]["delta"]["content"] == "" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_stream_source_errors_emit_nothing_and_hide_provider_text(policy): + async def broken(): + yield {"choices": [{"delta": {"content": EMAIL_A}}]} + raise RuntimeError("provider failed on " + EMAIL_A) + + emitted = [] + with pytest.raises(RuntimeError) as error: + async for chunk in guard( + fail_policy=policy + ).async_post_call_streaming_iterator_hook(None, broken(), {}): + emitted.append(chunk) + assert not emitted and EMAIL_A not in str(error.value) + assert error.value.__cause__ is None and error.value.__suppress_context__ + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit", ["MAX_STREAM_CHARS", "MAX_STREAM_CHUNKS"]) +async def test_stream_limits_reject_without_emission(monkeypatch, limit): + monkeypatch.setattr(gateway, limit, 1) + chunks = [{"choices": [{"delta": {"content": "ab"}}]}] * 2 + emitted = [] + with pytest.raises(RuntimeError, match="scan budget"): + async for chunk in guard().async_post_call_streaming_iterator_hook( + None, source(chunks), {} + ): + emitted.append(chunk) + assert not emitted + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_stream_engine_failures_follow_documented_policy(monkeypatch, policy): + def failing(*args, **kwargs): + raise RuntimeError(EMAIL_A) + + monkeypatch.setattr(gateway, "_redact_text", failing) + chunks = [{"choices": [{"delta": {"content": EMAIL_A}}]}] + if policy == "open": + assert await streamed(guard(fail_policy=policy), chunks) == chunks + else: + with pytest.raises(RuntimeError, match="fail_policy is 'closed'") as error: + await streamed(guard(fail_policy=policy), chunks) + assert EMAIL_A not in str(error.value) + + +@pytest.mark.parametrize("action", ["ask", "deny"]) +def test_hook_oversize_never_silently_allows_unscanned_tail(action): + payload = { + "hook_event_name": "PreToolUse", + "tool_name": "Write", + "tool_input": {"content": "x" * (hook.MAX_SCAN_CHARS + 1) + " " + SSN}, + } + code, output = hook.run(payload, {"DATAFOG_HOOK_ACTION": action}) + decision = json.loads(output)["hookSpecificOutput"] + assert code == 0 and decision["permissionDecision"] == action + assert ( + "not fully scanned" in decision["permissionDecisionReason"] + and SSN not in output + ) + + +def test_hook_surrogate_and_total_budget_cannot_disable_pii_gate(monkeypatch): + payload = { + "hook_event_name": "PreToolUse", + "tool_input": {"command": "echo " + SSN, "description": "\udc00"}, + } + code, output = hook.run(payload, {}) + assert code == 0 and "SSN x1" in output + monkeypatch.setattr(hook, "TOTAL_SCAN_CHARS", 20) + code, output = hook.run( + {"hook_event_name": "PreToolUse", "tool_input": ["a" * 15, "b" * 15]}, {} + ) + assert ( + code == 0 + and json.loads(output)["hookSpecificOutput"]["permissionDecision"] == "ask" + ) + + +@pytest.mark.asyncio +async def test_decorator_awaits_and_rejects_non_string_returns(): + adapter = datafog.create_guardrail() + + @adapter + async def generated(): + return EMAIL_A + + assert await generated() == "[EMAIL_1]" + + @adapter + async def structured(): + return [EMAIL_A] + + with pytest.raises(TypeError, match="must return a string"): + await structured() + + @adapter + def sync_structured(): + return {"text": EMAIL_A} + + with pytest.raises(TypeError, match="must return a string"): + sync_structured() diff --git a/tests/test_runtime_dependency_safety.py b/tests/test_runtime_dependency_safety.py index d34ceb8a..b311fff4 100644 --- a/tests/test_runtime_dependency_safety.py +++ b/tests/test_runtime_dependency_safety.py @@ -42,8 +42,7 @@ def test_runtime_code_does_not_install_packages() -> None: def test_ocr_and_spark_public_services_do_not_require_optional_imports() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import importlib.abc import sys @@ -74,8 +73,7 @@ def find_spec(self, fullname, path=None, target=None): assert TextService is not None assert datafog.ImageService is ImageService assert datafog.SparkService is SparkService -""" - ) +""") def test_spacy_pii_missing_model_requires_explicit_download( diff --git a/tests/test_rust_backend.py b/tests/test_rust_backend.py index 74922c8f..3cbdd607 100644 --- a/tests/test_rust_backend.py +++ b/tests/test_rust_backend.py @@ -66,15 +66,11 @@ def test_selection_alias_and_empty_selection(native, selection): assert len(result.entities) == 1 -def test_unknown_selection_preserves_legacy_empty_result(native): +def test_unknown_selection_is_rejected(native): findings, _ = native findings.append(finding("EMAIL", "a@example.com", 0, 13)) - assert ( - engine.scan( - "a@example.com", "regex", entity_types=["UNKNOWN"], backend="rust" - ).entities - == [] - ) + with pytest.raises(ValueError, match="Unsupported entity_types"): + engine.scan("a@example.com", "regex", entity_types=["UNKNOWN"], backend="rust") def test_unknown_native_label_preserved_without_silently_dropping_pii(native): @@ -84,7 +80,15 @@ def test_unknown_native_label_preserved_without_silently_dropping_pii(native): assert [item.type for item in result.entities] == ["FUTURE_LABEL"] -def test_python_overlap_priority_precedes_selection(native): +def test_python_overlap_priority_precedes_selection(native, monkeypatch): + metadata = capability_fixture() + metadata["supported_entities"].extend(["PHONE", "CREDIT_CARD"]) + for label in ("PHONE", "CREDIT_CARD"): + metadata["entities"][label] = { + "scopes": ["text"], + "activation": {"kind": "default"}, + } + monkeypatch.setattr(sys.modules["datafog_core"], "capabilities", lambda: metadata) findings, _ = native text = "1234567890123456" findings.extend( @@ -116,7 +120,7 @@ def test_python_allowlist_semantics(native, kwargs, count): ) -@pytest.mark.parametrize("pattern", ["[", "(a+)+", "x" * 513]) +@pytest.mark.parametrize("pattern", ["[", "x" * 513]) def test_allowlist_validation_before_native(native, pattern): _, calls = native with pytest.raises(ValueError): @@ -146,16 +150,20 @@ def test_unknown_locale_validation_preserved(native): @pytest.mark.parametrize("name", ["smart", "spacy", "gliner"]) -def test_nonregex_engine_rejected(native, name): +def test_nonregex_engines_use_native_structured_stage(native, name, monkeypatch): _, calls = native - with pytest.raises(ValueError, match="only engine='regex'"): - engine.scan("", name, backend="rust") - assert not calls + monkeypatch.setattr(engine, "_spacy_entities", lambda _: []) + monkeypatch.setattr(engine, "_gliner_entities", lambda _: []) + assert not engine.scan("", name, backend="rust").entities + assert calls == [""] -def test_bad_backend_rejected(): +def test_bad_backend_rejected(monkeypatch): + with pytest.raises(ValueError, match="backend must be one of"): + engine.scan("", "regex", backend="unknown") + monkeypatch.setenv(engine.BACKEND_ENV, "unknown") with pytest.raises(ValueError, match="backend must be one of"): - engine.scan("", "regex", backend="auto") + engine.scan("", "regex") def test_no_native_import_on_python_path_and_actionable_missing_error(monkeypatch): @@ -198,7 +206,9 @@ def test_all_legacy_transformations_use_native_findings(native, strategy): finding("EMAIL", "a@example.com", 14, 27), ] ) - expected = engine.scan_and_redact(text, "regex", strategy=strategy) + expected = engine.scan_and_redact( + text, "regex", strategy=strategy, backend="python" + ) actual = engine.scan_and_redact(text, "regex", strategy=strategy, backend="rust") assert actual == expected assert calls == [text] @@ -208,7 +218,9 @@ def test_all_legacy_transformations_use_native_findings(native, strategy): def test_real_native_unicode_and_transformations(strategy): pytest.importorskip("datafog_core") text = "😀 München a@example.com and a@example.com" - python_result = engine.scan_and_redact(text, "regex", strategy=strategy) + python_result = engine.scan_and_redact( + text, "regex", strategy=strategy, backend="python" + ) native_result = engine.scan_and_redact( text, "regex", strategy=strategy, backend="rust" ) diff --git a/tests/test_rust_contract.py b/tests/test_rust_contract.py index 9bea3641..73909bf4 100644 --- a/tests/test_rust_contract.py +++ b/tests/test_rust_contract.py @@ -6,7 +6,12 @@ import pytest from tests.contract_481 import FIXTURE -from tests.rust_contract import DEVIATIONS, applicable, native_observation +from tests.rust_contract import ( + DEVIATIONS, + applicable, + expected_observation, + native_observation, +) pytest.importorskip("datafog_core") CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) @@ -15,7 +20,9 @@ def test_native_version_and_review_inventory(): - assert importlib.metadata.version("datafog-core").startswith("0.4.") + installed = importlib.metadata.version("datafog-core") + major, minor = REVIEWED["core_version"].split(".")[:2] + assert installed.split(".")[:2] == [major, minor] assert set(REVIEWED["cases"]) <= {case["id"] for case in CASES} for item in REVIEWED["cases"].values(): assert item["classification"] in { @@ -28,6 +35,4 @@ def test_native_version_and_review_inventory(): @pytest.mark.parametrize("case", CASES, ids=lambda case: case["id"]) def test_native_contract(case): - deviation = REVIEWED["cases"].get(case["id"]) - expected = deviation["expected"] if deviation else case["expected"] - assert native_observation(case) == expected, case["id"] + assert native_observation(case) == expected_observation(case), case["id"] diff --git a/tests/test_spacy_span_contract.py b/tests/test_spacy_span_contract.py new file mode 100644 index 00000000..7029b391 --- /dev/null +++ b/tests/test_spacy_span_contract.py @@ -0,0 +1,56 @@ +"""Optional real-spaCy offset integration without downloading model weights.""" + +import pytest + +from datafog import engine +from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator +from datafog.services.text_service import TextService + +spacy = pytest.importorskip("spacy") + + +@pytest.fixture +def real_annotator(monkeypatch): + nlp = spacy.blank("en") + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns( + [ + {"label": "PERSON", "pattern": "Ann"}, + {"label": "ORG", "pattern": "Acme Corp"}, + {"label": "DATE", "pattern": "yesterday"}, + ] + ) + annotator = SpacyPIIAnnotator(nlp=nlp, model_name="local-test-ruler") + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: annotator) + return annotator + + +def test_spacy_ranges_redact_actual_name_without_hitting_prefix(real_annotator): + text = "Annual report by Ann yesterday: 6011111111111117" + result = engine.scan_and_redact(text, engine="spacy") + assert ( + result.redacted_text == "Annual report by [PERSON_1] yesterday: [CREDIT_CARD_1]" + ) + assert all(text[item.start : item.end] == item.text for item in result.entities) + + +def test_spacy_service_combines_ner_and_native_detection(real_annotator): + text = "Ann at Acme Corp: 6011111111111117 ip 2001:db8::1" + service = TextService(engine="spacy") + result = service.annotate_text_sync(text) + assert result == { + "PERSON": ["Ann"], + "ORGANIZATION": ["Acme Corp"], + "CREDIT_CARD": ["6011111111111117"], + "IP_ADDRESS": ["2001:db8::1"], + } + assert all( + text[item.start : item.end] == item.text + for item in service.annotate_text_sync(text, structured=True) + ) + + +def test_spacy_repeated_names_keep_each_original_span(real_annotator): + text = "Ann wrote an Annual report for Ann." + result = engine.scan(text, engine="spacy") + assert [(item.start, item.end) for item in result.entities] == [(0, 3), (31, 34)] diff --git a/tests/test_text_service.py b/tests/test_text_service.py index 74156082..3d5973ad 100644 --- a/tests/test_text_service.py +++ b/tests/test_text_service.py @@ -1,366 +1,124 @@ -from unittest.mock import Mock, patch - -import pytest - -# Legacy coverage for the historical shadow TextService. New behavior belongs -# in datafog.services.text_service. -from datafog.services.text_service_original import TextService +"""Public TextService behavior using the shipped service and shared engine.""" +import asyncio +import threading +from types import SimpleNamespace -@pytest.fixture -def mock_annotator(): - mock = Mock() - mock.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - return mock - +import pytest -@pytest.fixture -def mock_regex_annotator(): - mock = Mock() - mock.annotate.return_value = { - "EMAIL": ["john@example.com"], - "PHONE": ["555-555-5555"], - } +from datafog import engine +from datafog.services.text_service import TextService - # Add mock for annotate_with_spans method - from datafog.processing.text_processing.regex_annotator import ( - AnnotationResult, - Span, - ) - spans = [ - Span(label="EMAIL", start=0, end=15, text="john@example.com"), - Span(label="PHONE", start=20, end=32, text="555-555-5555"), - ] - mock.annotate_with_spans.return_value = ( - {"EMAIL": ["john@example.com"], "PHONE": ["555-555-5555"]}, - AnnotationResult(text="test", spans=spans), - ) - return mock +@pytest.mark.parametrize( + "options", + [{"engine": "invalid"}, {"text_chunk_length": 0}, {"text_chunk_length": -1}], +) +def test_invalid_configuration_is_explicit(options): + with pytest.raises(ValueError): + TextService(**options) -@pytest.fixture -def text_service(mock_annotator, mock_regex_annotator): - # Configure regex annotator to return empty results so auto mode falls back to spaCy - # This ensures backward compatibility with existing tests while using 'auto' mode - mock_regex_annotator.annotate.return_value = { - key: [] - for key in ["EMAIL", "PHONE", "SSN", "CREDIT_CARD", "IP_ADDRESS", "DOB", "ZIP"] +def test_default_service_detects_native_only_entities(): + result = TextService().annotate_text_sync("card 6011111111111117 ip 2001:db8::1") + assert result == { + "CREDIT_CARD": ["6011111111111117"], + "IP_ADDRESS": ["2001:db8::1"], } - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - return_value=mock_annotator, - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - return_value=mock_regex_annotator, - ): - # Use 'auto' engine to match production default, but regex will find nothing - # so it will always fall back to spaCy, maintaining test compatibility - return TextService(text_chunk_length=10, engine="auto") - - -@pytest.fixture -def text_service_with_engine(mock_annotator, mock_regex_annotator): - def _create_service(engine="auto"): - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - return_value=mock_annotator, - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - return_value=mock_regex_annotator, - ): - return TextService(text_chunk_length=10, engine=engine) - return _create_service +def test_long_document_seams_preserve_complete_entities(): + value = "john@example.com" + text = "x" * 995 + " " + value + " " + "y" * 1010 + " ssn 123-45-6789" + result = TextService(text_chunk_length=1000).annotate_text_sync(text) + assert result == {"EMAIL": [value], "SSN": ["123-45-6789"]} -def test_init(text_service): - assert text_service.text_chunk_length == 10 - - -def test_init_with_default_engine(text_service): - assert text_service.text_chunk_length == 10 - # We're using 'auto' in our fixture to match production default - assert text_service.engine == "auto" - - -def test_init_with_custom_engine(text_service_with_engine): - service = text_service_with_engine(engine="regex") - assert service.engine == "regex" - - service = text_service_with_engine(engine="spacy") - assert service.engine == "spacy" - - service = text_service_with_engine(engine="auto") - assert service.engine == "auto" - - -def test_init_with_invalid_engine(): - with pytest.raises(AssertionError, match="Invalid engine"): - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - ): - TextService(engine="invalid") - - -def test_chunk_text(text_service): - text = "This is a test sentence for chunking." - chunks = text_service._chunk_text(text) - assert len(chunks) == 4 - assert chunks == ["This is a ", "test sente", "nce for ch", "unking."] +def test_structured_output_preserves_unicode_source_ranges(): + text = "😀 café john@example.com and john@example.com" + result = TextService(text_chunk_length=10).annotate_text_sync(text, structured=True) + assert len(result) == 2 + assert all( + item.label == "EMAIL" and text[item.start : item.end] == item.text + for item in result + ) + assert result[0].start < result[1].start -def test_combine_annotations(text_service): - annotations = [ - {"PER": ["John"], "ORG": ["Acme"]}, - {"PER": ["Doe"], "LOC": ["New York"]}, +def test_batch_results_preserve_input_order_and_duplicate_documents(): + texts = ["a@example.com", "no pii", "a@example.com"] + assert TextService().batch_annotate_text_sync(texts) == [ + {"EMAIL": ["a@example.com"]}, + {}, + {"EMAIL": ["a@example.com"]}, ] - combined = text_service._combine_annotations(annotations) - assert combined == {"PER": ["John", "Doe"], "ORG": ["Acme"], "LOC": ["New York"]} - - -def test_annotate_text_sync(text_service): - result = text_service.annotate_text_sync("John Doe works at Acme Inc") - assert result == { - "PER": ["John Doe", "John Doe", "John Doe"], - "ORG": ["Acme Inc", "Acme Inc", "Acme Inc"], - } - - -def test_batch_annotate_text_sync(text_service): - texts = ["John Doe", "Acme Inc"] - result = text_service.batch_annotate_text_sync(texts) - assert result == { - "John Doe": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - "Acme Inc": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - } @pytest.mark.asyncio -async def test_annotate_text_async(text_service): - result = await text_service.annotate_text_async("John Doe works at Acme Inc") - assert result == { - "PER": ["John Doe", "John Doe", "John Doe"], - "ORG": ["Acme Inc", "Acme Inc", "Acme Inc"], - } +async def test_async_results_match_synchronous_service(): + service = TextService() + texts = ["a@example.com", "ssn 123-45-6789", "no pii"] + assert await service.batch_annotate_text_async( + texts + ) == service.batch_annotate_text_sync(texts) + assert await service.annotate_text_async( + texts[0], structured=True + ) == service.annotate_text_sync(texts[0], structured=True) @pytest.mark.asyncio -async def test_batch_annotate_text_async(text_service): - texts = ["John Doe", "Acme Inc"] - result = await text_service.batch_annotate_text_async(texts) - assert result == { - "John Doe": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - "Acme Inc": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - } - - -def test_long_text_chunking(text_service): - long_text = "John Doe works at Acme Inc. Jane Smith is from New York City." - result = text_service.annotate_text_sync(long_text) - expected_count = len(text_service._chunk_text(long_text)) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -@pytest.mark.asyncio -async def test_long_text_chunking_async(text_service): - long_text = "John Doe works at Acme Inc. Jane Smith is from New York City." - result = await text_service.annotate_text_async(long_text) - expected_count = len(text_service._chunk_text(long_text)) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -def test_empty_string(text_service): - result = text_service.annotate_text_sync("") - assert result == {} - - -def test_short_string(text_service): - result = text_service.annotate_text_sync("Short") - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_special_characters(text_service): - result = text_service.annotate_text_sync("John@Doe.com works at Acme-Inc!!!") - expected_count = len(text_service._chunk_text("John@Doe.com works at Acme-Inc!!!")) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -def test_regex_engine(text_service_with_engine, mock_regex_annotator): - service = text_service_with_engine(engine="regex") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("john@example.com") - - # Should only call the regex annotator - assert mock_regex_annotator.annotate.called - assert not service.spacy_annotator.annotate.called - assert result == {"EMAIL": ["john@example.com"], "PHONE": ["555-555-5555"]} - - -def test_spacy_engine(text_service_with_engine, mock_annotator): - service = text_service_with_engine(engine="spacy") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("John Doe works at Acme Inc") - - # Should only call the spaCy annotator - assert mock_annotator.annotate.called - assert not service.regex_annotator.annotate.called - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_auto_engine_with_regex_results( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - # Configure regex annotator to return results - mock_regex_annotator.annotate.return_value = {"EMAIL": ["john@example.com"]} - - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("john@example.com") - - # Should call regex annotator but not spaCy - assert mock_regex_annotator.annotate.called - assert not mock_annotator.annotate.called - - assert result == {"EMAIL": ["john@example.com"]} - - -def test_auto_engine_with_fallback( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - # Configure regex annotator to return empty results - mock_regex_annotator.annotate.return_value = {"EMAIL": [], "PHONE": []} - - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("John Doe works at Acme Inc") - - # Should call both annotators - assert mock_regex_annotator.annotate.called - assert mock_annotator.annotate.called - - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_structured_output_regex_engine(text_service_with_engine, mock_regex_annotator): - """Test structured output mode with regex engine.""" - # Set up the mock to return spans that match the input text - from datafog.processing.text_processing.regex_annotator import ( - AnnotationResult, - Span, - ) - - # Create spans that will be returned by the mock - test_text = "john@example.com" - spans = [ - # Make sure the end position matches the actual length of the text - Span(label="EMAIL", start=0, end=len(test_text), text=test_text), - ] - - # Update the mock to return spans that match the input text - mock_regex_annotator.annotate_with_spans.return_value = ( - {"EMAIL": [test_text]}, - AnnotationResult(text=test_text, spans=spans), +async def test_async_detection_runs_outside_event_loop_thread(monkeypatch): + loop_thread = threading.get_ident() + worker_threads = [] + + def detect(text, **_): + worker_threads.append(threading.get_ident()) + return SimpleNamespace(entities=[]) + + monkeypatch.setattr(engine, "scan", detect) + await asyncio.wait_for(TextService().annotate_text_async("text"), timeout=2) + assert worker_threads and all(thread != loop_thread for thread in worker_threads) + + +@pytest.mark.parametrize("mode", ["auto", "smart"]) +def test_optional_ner_adds_to_structured_findings(monkeypatch, mode): + text = "Ann owns 6011111111111117" + monkeypatch.setattr( + engine, + "_gliner_entities", + lambda _: [engine.Entity("PERSON", "Ann", 0, 3, 0.8, "gliner")], ) + result = TextService(engine=mode).annotate_text_sync(text) + assert result == {"PERSON": ["Ann"], "CREDIT_CARD": ["6011111111111117"]} - service = text_service_with_engine(engine="regex") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync(test_text, structured=True) - - # Should call regex annotator's annotate_with_spans method - assert mock_regex_annotator.annotate_with_spans.called - - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 1 # Only one span should be returned (EMAIL) - - # Verify the span has the correct properties - assert result[0].label == "EMAIL" - assert result[0].text == test_text - assert result[0].start == 0 - assert result[0].end == len(test_text) - - -def test_structured_output_spacy_engine(text_service_with_engine, mock_annotator): - """Test structured output mode with spaCy engine.""" - service = text_service_with_engine(engine="spacy") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - # Set up mock to return entities that can be found in the test text - test_text = "John Doe works at Acme Inc" - mock_annotator.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} +def test_custom_gliner_model_is_forwarded(monkeypatch): + models = [] - result = service.annotate_text_sync(test_text, structured=True) + def detect(_, model_name): + models.append(model_name) + return [] - # Should call spaCy annotator - assert mock_annotator.annotate.called - - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 2 - - # Check that spans were created correctly - per_spans = [span for span in result if span.label == "PER"] - org_spans = [span for span in result if span.label == "ORG"] - - assert len(per_spans) == 1 - assert per_spans[0].text == "John Doe" - assert per_spans[0].start == test_text.find("John Doe") - assert per_spans[0].end == test_text.find("John Doe") + len("John Doe") - - assert len(org_spans) == 1 - assert org_spans[0].text == "Acme Inc" - assert org_spans[0].start == test_text.find("Acme Inc") - assert org_spans[0].end == test_text.find("Acme Inc") + len("Acme Inc") - - -def test_structured_output_auto_engine( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - """Test structured output mode with auto engine.""" - # Configure regex annotator to return empty spans - from datafog.processing.text_processing.regex_annotator import AnnotationResult - - mock_regex_annotator.annotate_with_spans.return_value = ( - {"EMAIL": [], "PHONE": []}, - AnnotationResult(text="test", spans=[]), + monkeypatch.setattr(engine, "_gliner_entities", detect) + assert ( + TextService(engine="smart", gliner_model="local/custom").annotate_text_sync( + "no pii" + ) + == {} ) + assert models == ["local/custom"] - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - - # Set up mock to return entities that can be found in the test text - test_text = "John Doe works at Acme Inc" - mock_annotator.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - result = service.annotate_text_sync(test_text, structured=True) +def test_locale_activates_additional_shared_detectors(): + text = "Aufenthaltstitel AT1234567 gueltig." + assert TextService(locales=["de"]).annotate_text_sync(text) == { + "DE_RESIDENCE_PERMIT_NUMBER": ["AT1234567"] + } - # Should call both annotators - assert mock_regex_annotator.annotate_with_spans.called - assert mock_annotator.annotate.called - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 2 +def test_empty_and_invalid_text(): + service = TextService() + assert service.annotate_text_sync("") == {} + assert service.annotate_text_sync("", structured=True) == [] + with pytest.raises(TypeError, match="text must be a string"): + service.annotate_text_sync(None) diff --git a/tests/test_text_service_integration.py b/tests/test_text_service_integration.py index d3363f5f..538f63c8 100644 --- a/tests/test_text_service_integration.py +++ b/tests/test_text_service_integration.py @@ -5,6 +5,15 @@ from datafog.services.text_service import TextService +@pytest.fixture +def require_spacy_model(): + spacy = pytest.importorskip("spacy") + try: + spacy.load("en_core_web_lg") + except OSError: + pytest.skip("en_core_web_lg must be provisioned for trained-model integration") + + @pytest.fixture def real_text_service(): """Create a real TextService instance for integration testing.""" @@ -32,7 +41,7 @@ def test_engine_regex_detects_simple_entities(): @pytest.mark.integration -def test_engine_auto_fallbacks_to_spacy(): +def test_engine_auto_fallbacks_to_spacy(require_spacy_model): """Test that auto mode works correctly with entity detection.""" # We need to test the auto mode in a more controlled way # Create a text that contains only named entities (no emails, phones, etc.) @@ -45,7 +54,7 @@ def test_engine_auto_fallbacks_to_spacy(): # Verify spaCy finds named entities assert "PERSON" in spacy_result and spacy_result["PERSON"] - assert "ORG" in spacy_result and spacy_result["ORG"] + assert "ORGANIZATION" in spacy_result and spacy_result["ORGANIZATION"] # Now create a special text that contains both regex-detectable and spaCy-detectable entities mixed_text = "John Smith's email is john.smith@example.com" @@ -61,7 +70,7 @@ def test_engine_auto_fallbacks_to_spacy(): @pytest.mark.integration -def test_engine_spacy_only(): +def test_engine_spacy_adds_to_structured_pii(require_spacy_model): """Test that spaCy engine is always used regardless of regex potential hits.""" # Sample text with both regex-detectable and spaCy-detectable entities text = """John Smith's email is john.smith@example.com. @@ -79,11 +88,10 @@ def test_engine_spacy_only(): # Verify spaCy detected named entities assert "PERSON" in spacy_result and spacy_result["PERSON"] - assert "ORG" in spacy_result and spacy_result["ORG"] + assert "ORGANIZATION" in spacy_result and spacy_result["ORGANIZATION"] - # Verify spaCy did NOT detect the email (which confirms it's using spaCy only) - # This is because spaCy doesn't have a built-in EMAIL entity type - assert "EMAIL" not in spacy_result or not spacy_result["EMAIL"] + # NER augments the shared structured detector. + assert "john.smith@example.com" in spacy_result["EMAIL"] @pytest.mark.integration @@ -130,7 +138,7 @@ def test_structured_annotation_output(): @pytest.mark.integration -def test_debug_entity_types(): +def test_spacy_entity_labels_are_canonical(require_spacy_model): """Debug test to print the actual entity types returned by spaCy.""" # Sample text with named entities text = """John Smith works at Microsoft Corporation in Seattle. @@ -146,8 +154,8 @@ def test_debug_entity_types(): if values: # Only print non-empty lists print(f" {entity_type}: {values}") - # No assertion needed, this is just for debugging - assert True + assert "ORGANIZATION" in spacy_result + assert "ORG" not in spacy_result @pytest.mark.skip(reason="Performance benchmarking requires more setup") diff --git a/tox.ini b/tox.ini index 83d3f793..a7a1565c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py310,py311,py312 +envlist = py310,py311,py312,py313,py314 isolated_build = True [testenv:integration]