From 352cafeeda6398ea7e4dc5aeec673e0863455bb7 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:53:31 -0800 Subject: [PATCH 01/20] fix(engine): preserve detection across backends, spans and large inputs --- datafog/engine.py | 378 +++++++++++++----- .../text_processing/gliner_annotator.py | 78 ++-- .../regex_annotator/regex_annotator.py | 29 +- .../text_processing/spacy_pii_annotator.py | 47 ++- datafog/services/text_service.py | 46 ++- tests/test_allowlist.py | 16 +- tests/test_backend_resolution.py | 4 +- tests/test_core04_integration.py | 5 +- tests/test_engine_api.py | 7 +- tests/test_review_engine.py | 306 ++++++++++++++ 10 files changed, 707 insertions(+), 209 deletions(-) create mode 100644 tests/test_review_engine.py diff --git a/datafog/engine.py b/datafog/engine.py index 495c2d18..ec33081b 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import hmac import os import re import warnings @@ -10,6 +11,8 @@ from functools import lru_cache from typing import Optional +import regex as bounded_re + from .exceptions import CoreUnavailableWarning, EngineNotAvailable from .processing.text_processing.regex_annotator import RegexAnnotator @@ -66,6 +69,10 @@ "US_ROUTING_NUMBER", "NPI", "UUID", + "PASSPORT_NUMBER", + "MEDICAL_RECORD_NUMBER", + "LICENSE_NUMBER", + "URL", } #: Labels the native rule engine reports as structured-only, served for text by @@ -78,7 +85,16 @@ _core_warned = False _cnn_cache: tuple[object, object] | None = None -NER_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} +NER_ENTITY_TYPES = { + "PERSON", + "ORGANIZATION", + "LOCATION", + "ADDRESS", + "PASSPORT_NUMBER", + "MEDICAL_RECORD_NUMBER", + "LICENSE_NUMBER", + "URL", +} ENTITY_TYPE_PRIORITY = { "DE_IBAN": 100, @@ -155,9 +171,13 @@ def _find_all_occurrences(text: str, needle: str) -> list[tuple[int, int]]: def _entities_from_dict( text: str, payload: dict[str, list[str]], engine: str, confidence: float ) -> list[Entity]: + """Compatibility for third-party annotators without offset support. + + Whole-word matching avoids interpreting "Ann" inside "Annual" as a name. + First-party annotators use their original character ranges instead. + """ entities: list[Entity] = [] value_offsets: dict[str, int] = {} - for raw_type, values in payload.items(): canonical_type = _canonical_type(raw_type) if canonical_type not in ALL_ENTITY_TYPES: @@ -165,25 +185,47 @@ def _entities_from_dict( for value in values: if not isinstance(value, str) or not value.strip(): continue - search_start = value_offsets.get(value, 0) - idx = text.find(value, search_start) - if idx < 0: - idx = text.find(value) - end = idx + len(value) if idx >= 0 else -1 - value_offsets[value] = end if end >= 0 else search_start + 1 + pattern = re.compile(r"(? list[Entity]: + method = getattr(annotator, "annotate_with_spans", None) + if not callable(method): + return _entities_from_dict(text, annotator.annotate(text), engine, confidence) + _, result = method(text) + entities = [] + for span in result.spans: + label = _canonical_type(span.label) + if label not in ALL_ENTITY_TYPES: + continue + if ( + not 0 <= span.start < span.end <= len(text) + or text[span.start : span.end] != span.text + ): + raise ValueError(f"{engine} returned an invalid entity span") + entities.append( + Entity(label, span.text, span.start, span.end, confidence, engine) + ) + return entities + + def _entity_length(entity: Entity) -> int: return max(entity.end - entity.start, 0) @@ -195,7 +237,31 @@ def _entities_overlap(left: Entity, right: Entity) -> bool: def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]: - selected: list[Entity] = [] + """Preserve longest-span precedence with O(n log n) interval queries.""" + coordinates = { + value: index + 1 + for index, value in enumerate( + sorted( + {value for entity in entities for value in (entity.start, entity.end)} + ) + ) + } + starts = [0] * (len(coordinates) + 1) + ends = [0] * (len(coordinates) + 1) + + def add(tree: list[int], index: int) -> None: + while index < len(tree): + tree[index] += 1 + index += index & -index + + def count(tree: list[int], index: int) -> int: + total = 0 + while index > 0: + total += tree[index] + index -= index & -index + return total + + selected = [] for entity in sorted( entities, key=lambda item: ( @@ -207,9 +273,12 @@ def _suppress_overlapping_entities(entities: list[Entity]) -> list[Entity]: item.type, ), ): - if any(_entities_overlap(entity, kept) for kept in selected): + start, end = coordinates[entity.start], coordinates[entity.end] + if count(starts, end - 1) > count(ends, start): continue selected.append(entity) + add(starts, start) + add(ends, end) return sorted(selected, key=lambda item: (item.start, item.end, item.type)) @@ -289,15 +358,28 @@ def _cnn_detector_or_none() -> object | None: def _cnn_findings( detector: object, requested: set[str], text: str -) -> list[tuple[dict, object]]: - """Serve prose names and address blocks from the embedded model.""" - try: - result = detector.scan_model(text) - except Exception as exc: # relative-path/limit errors surface to the caller - raise _model_error(exc) from exc - return [ - ({}, finding) for finding in result.findings if finding.entity_type in requested - ] +) -> list[tuple[int, object]]: + """Bound model work while retaining document character coordinates. + + 4096 characters fit the native byte/token caps even for four-byte Unicode. + Overlap gives names and address blocks context on either side of a seam. + """ + chunk_length, overlap = 4096, 256 + findings = [] + for offset in range(0, max(len(text), 1), chunk_length - overlap): + chunk = text[offset : offset + chunk_length] + try: + result = detector.scan_model(chunk) + except Exception as exc: + raise _model_error(exc) from exc + findings.extend( + (offset, finding) + for finding in result.findings + if finding.entity_type in requested + ) + if offset + chunk_length >= len(text): + break + return findings def _model_error(exc: Exception) -> Exception: @@ -337,17 +419,18 @@ def _rust_entities( configs = scan_configs(datafog_core, requested - MODEL_ENTITY_TYPES, locales) entities: list[Entity] = [] seen: set[tuple[str, int, int]] = set() + native_text = re.sub(r"[\ud800-\udfff]", "\ufffd", text) findings = [ - (config, finding) + (0, finding) for config in configs - for finding in datafog_core.scan(text, config=config) + for finding in datafog_core.scan(native_text, config=config) ] if detector is not None: - findings.extend(_cnn_findings(detector, model_requested, text)) - for config, finding in findings: + findings.extend(_cnn_findings(detector, model_requested, native_text)) + for offset, finding in findings: canonical_type = _canonical_type(finding.entity_type) - start = finding.codepoint_range.start - end = finding.codepoint_range.end + start = offset + finding.codepoint_range.start + end = offset + finding.codepoint_range.end key = (canonical_type, start, end) if key in seen or not finding.matched_text.strip(): continue @@ -355,7 +438,7 @@ def _rust_entities( entities.append( Entity( type=canonical_type, - text=finding.matched_text, + text=text[start:end], start=start, end=end, confidence=1.0, @@ -369,16 +452,22 @@ def _spacy_entities(text: str) -> list[Entity]: annotator = _get_spacy_annotator() if isinstance(annotator, _UnavailableAnnotator): raise EngineNotAvailable(annotator.message) - payload = annotator.annotate(text) - return _entities_from_dict(text, payload, engine="spacy", confidence=0.7) + return [ + entity + for entity in _annotator_entities(text, annotator, "spacy", 0.7) + if entity.type in NER_ENTITY_TYPES + ] -def _gliner_entities(text: str) -> list[Entity]: - annotator = _get_gliner_annotator() +def _gliner_entities(text: str, model_name: str | None = None) -> list[Entity]: + annotator = ( + _get_gliner_annotator() + if model_name is None + else _get_gliner_annotator(model_name) + ) if isinstance(annotator, _UnavailableAnnotator): raise EngineNotAvailable(annotator.message) - payload = annotator.annotate(text) - return _entities_from_dict(text, payload, engine="gliner", confidence=0.8) + return _annotator_entities(text, annotator, "gliner", 0.8) @lru_cache(maxsize=1) @@ -398,15 +487,15 @@ def _get_spacy_annotator(): ) -@lru_cache(maxsize=1) -def _get_gliner_annotator(): +@lru_cache(maxsize=4) +def _get_gliner_annotator(model_name: str = "urchade/gliner_multi_pii-v1"): try: from .processing.text_processing.gliner_annotator import GLiNERAnnotator except ImportError as exc: return _UnavailableAnnotator(str(exc)) try: - annotator = GLiNERAnnotator.create() + annotator = GLiNERAnnotator.create(model_name=model_name) except ImportError as exc: return _UnavailableAnnotator(str(exc)) except Exception as exc: @@ -438,44 +527,39 @@ def _filter_entity_types( return [entity for entity in entities if entity.type in allowed] -# Python's re module backtracks; a quantified group containing another -# quantifier (e.g. ``(a+)+``) can take exponential time on adversarial -# input, and entity text can be attacker-influenced (LLM messages, tool -# output). Reject that construct outright rather than matching under it. -_NESTED_QUANTIFIER = re.compile( - r"\((?:[^()\\]|\\.)*(? list["re.Pattern[str]"]: +def _compile_allowlist_patterns(allowlist_patterns: Optional[list[str]]) -> list: compiled = [] + if isinstance(allowlist_patterns, str): + raise ValueError("allowlist_patterns must be a list of patterns") for raw in allowlist_patterns or []: - if len(raw) > MAX_ALLOWLIST_PATTERN_LENGTH: + if not isinstance(raw, str) or len(raw) > MAX_ALLOWLIST_PATTERN_LENGTH: raise ValueError( - "allowlist_patterns entries must be at most " - f"{MAX_ALLOWLIST_PATTERN_LENGTH} characters" - ) - if _NESTED_QUANTIFIER.search(raw): - raise ValueError( - "allowlist_patterns contains a quantified group with a nested " - f"quantifier ({raw!r}), which risks catastrophic backtracking; " - "rewrite the pattern without nesting quantifiers" + "allowlist_patterns entries must be strings of at most 512 characters" ) try: - compiled.append(re.compile(raw)) - except re.error as exc: + compiled.append(bounded_re.compile(raw)) + except bounded_re.error as exc: raise ValueError( f"allowlist_patterns contains an invalid regex: {raw!r} ({exc})" ) from None return compiled +def _allowlist_matches(pattern: object, value: str) -> bool: + if len(value) > MAX_PATTERN_SUBJECT_LENGTH: + return False + try: + return pattern.fullmatch(value, timeout=ALLOWLIST_MATCH_TIMEOUT) is not None + except TimeoutError: + # A slow exemption must never suppress a sensitive finding. + return False + + def _apply_allowlist( entities: list[Entity], allowlist: Optional[list[str]], @@ -498,11 +582,7 @@ def _apply_allowlist( entity for entity in entities if entity.text not in exact - and not any( - pattern.fullmatch(entity.text) - for pattern in patterns - if len(entity.text) <= MAX_PATTERN_SUBJECT_LENGTH - ) + and not any(_allowlist_matches(pattern, entity.text) for pattern in patterns) ] @@ -532,8 +612,7 @@ def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: An explicit ``backend`` wins over ``DATAFOG_BACKEND``. ``auto`` uses the native engine whenever Core is importable and degrades to Python otherwise. - Non-``regex`` engines always take the Python path, because the native - backend serves ``regex`` and model findings only. + Every engine uses this backend for structured PII; NER adds spans. """ requested = backend or os.environ.get(BACKEND_ENV) or "auto" normalized = requested.strip().lower() @@ -543,8 +622,6 @@ def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: "backend argument)" ) if normalized == "auto": - if engine != "regex": - return "python" return "rust" if core_available() else "python" return normalized @@ -552,8 +629,33 @@ def resolve_backend(backend: str | None = None, engine: str = "regex") -> str: def _validate_backend(backend: str, engine: str) -> None: if backend not in {"python", "rust"}: raise ValueError("backend must be one of: python, rust") - if backend == "rust" and engine != "regex": - raise ValueError("backend='rust' supports only engine='regex'") + + +def _validate_entity_types( + entity_types: Optional[list[str]], backend: str, engine: str +) -> Optional[list[str]]: + if entity_types is None: + return None + if not isinstance(entity_types, (list, tuple)) or any( + not isinstance(value, str) or not value.strip() for value in entity_types + ): + raise ValueError("entity_types must be a list of entity labels") + requested = [_canonical_type(value) for value in entity_types] + if backend == "rust": + from ._core_capabilities import _read_capabilities + + supported, _, _ = _read_capabilities(_require_core().capabilities) + supported = supported | MODEL_ENTITY_TYPES + else: + supported = {_canonical_type(label) for label in RegexAnnotator.LABELS} + if engine != "regex": + supported |= NER_ENTITY_TYPES | {"DATE"} + unsupported = set(requested) - supported + if unsupported: + raise ValueError( + f"Unsupported entity_types for {backend}/{engine}: {', '.join(sorted(unsupported))}" + ) + return requested def scan( @@ -565,13 +667,15 @@ def scan( allowlist_patterns: Optional[list[str]] = None, *, backend: Optional[str] = None, + gliner_model: Optional[str] = None, ) -> ScanResult: """Scan text for PII entities. ``backend`` defaults to ``auto``: the native engine when ``datafog-core`` is importable, otherwise the pure-Python engine with a one-time warning. Set ``DATAFOG_BACKEND`` or pass ``backend="python"`` to force the fallback. The - native path serves ``engine="regex"``; non-regex engines always use Python. + selected backend supplies structured PII for every engine mode; optional + NER engines add entity spans. Probe names and address blocks come from the embedded model on the native path, so ``PERSON`` and ``ADDRESS`` work without spaCy or a model download. @@ -593,6 +697,7 @@ def scan( ) backend = resolve_backend(backend, engine) _validate_backend(backend, engine) + entity_types = _validate_entity_types(entity_types, backend, engine) if backend == "python" and requested_backend == "auto" and not core_available(): _warn_core_missing() @@ -634,7 +739,11 @@ def scan( if engine == "gliner" and _needs_ner(entity_types): try: - gliner_entities = _gliner_entities(text) + gliner_entities = ( + _gliner_entities(text) + if gliner_model is None + else _gliner_entities(text, gliner_model) + ) combined.extend(gliner_entities) engines_used.add("gliner") except EngineNotAvailable: @@ -649,13 +758,16 @@ def scan( if engine == "smart" and _needs_ner(entity_types): try: - gliner_entities = _gliner_entities(text) + gliner_entities = ( + _gliner_entities(text) + if gliner_model is None + else _gliner_entities(text, gliner_model) + ) combined.extend(gliner_entities) engines_used.add("gliner") - except EngineNotAvailable: + except EngineNotAvailable as exc: warnings.warn( - "GLiNER not available, smart scan falling back to spaCy. " - "Install with: pip install datafog[nlp-advanced]", + f"GLiNER unavailable; smart scan falling back to spaCy. {exc}", UserWarning, stacklevel=2, ) @@ -673,7 +785,7 @@ def scan( filtered = _filter_entity_types(combined, entity_types) filtered = _apply_allowlist(filtered, allowlist, allowlist_patterns) - deduped = _dedupe_entities(filtered) + deduped = _suppress_overlapping_entities(_dedupe_entities(filtered)) return ScanResult( entities=deduped, text=text, @@ -685,6 +797,9 @@ def redact( text: str, entities: list[Entity], strategy: str = "token", + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Redact PII entities from text. @@ -703,19 +818,36 @@ def redact( if strategy not in {"token", "mask", "hash", "pseudonymize"}: raise ValueError("strategy must be one of: token, mask, hash, pseudonymize") - redacted_text = text + if hash_key is not None and (not isinstance(hash_key, bytes) or len(hash_key) < 32): + raise ValueError("hash_key must contain at least 32 bytes") + if strategy == "hash": + try: + hashlib.new(hash_type).hexdigest() + except (ValueError, TypeError): + raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None + mapping: dict[str, str] = {} counters: dict[str, int] = {} pseudonym_by_value: dict[tuple[str, str], str] = {} - valid_entities = [ - entity - for entity in entities - if 0 <= entity.start < entity.end <= len(text) and entity.text - ] + valid_entities = [] + for entity in entities: + if ( + not 0 <= entity.start < entity.end <= len(text) + or not entity.text + or text[entity.start : entity.end] != entity.text + ): + raise ValueError( + "entities must contain valid ranges whose text matches the source" + ) + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", entity.type): + raise ValueError("entity types must be uppercase labels") + valid_entities.append(entity) # Returns non-overlapping spans sorted by document position. valid_entities = _suppress_overlapping_entities(valid_entities) + occupied_tokens = set(re.findall(r"\[[A-Z][A-Za-z0-9_]*\]", text)) + # Assign replacements in document order so the per-type counters used by # the token/pseudonymize strategies number the first occurrence _1; spans # are applied right-to-left below so earlier offsets stay valid. @@ -727,30 +859,62 @@ def redact( replacement = "*" * max(len(original), 1) # Distinct same-length values share a mask, so the mask string # cannot key the mapping without collisions. - counters[entity.type] = counters.get(entity.type, 0) + 1 - mapping_key = f"[{entity.type}_MASK_{counters[entity.type]}]" + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + mapping_key = f"[{entity.type}_MASK_{counters[entity.type]}]" + if mapping_key not in occupied_tokens: + break + occupied_tokens.add(mapping_key) elif strategy == "hash": - digest = hashlib.sha256(original.encode("utf-8")).hexdigest()[:12] - replacement = f"[{entity.type}_{digest}]" + encoded = original.encode("utf-8", "surrogatepass") + digest = ( + hmac.new(hash_key, encoded, hash_type).hexdigest() + if hash_key is not None + else hashlib.new(hash_type, encoded).hexdigest() + ) + key = (entity.type, original) + if key not in pseudonym_by_value: + candidate = f"[{entity.type}_{digest}]" + suffix = 1 + while candidate in occupied_tokens: + suffix += 1 + candidate = f"[{entity.type}_{digest}_{suffix}]" + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) + replacement = pseudonym_by_value[key] elif strategy == "pseudonymize": key = (entity.type, original) if key not in pseudonym_by_value: - counters[entity.type] = counters.get(entity.type, 0) + 1 - pseudonym_by_value[key] = ( - f"[{entity.type}_PSEUDO_{counters[entity.type]}]" - ) + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + candidate = f"[{entity.type}_PSEUDO_{counters[entity.type]}]" + if candidate not in occupied_tokens: + break + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) replacement = pseudonym_by_value[key] else: # token - counters[entity.type] = counters.get(entity.type, 0) + 1 - replacement = f"[{entity.type}_{counters[entity.type]}]" + key = (entity.type, original) + if key not in pseudonym_by_value: + while True: + counters[entity.type] = counters.get(entity.type, 0) + 1 + candidate = f"[{entity.type}_{counters[entity.type]}]" + if candidate not in occupied_tokens: + break + pseudonym_by_value[key] = candidate + occupied_tokens.add(candidate) + replacement = pseudonym_by_value[key] replacements.append((entity, replacement)) mapping[mapping_key if mapping_key is not None else replacement] = original - for entity, replacement in reversed(replacements): - redacted_text = ( - redacted_text[: entity.start] + replacement + redacted_text[entity.end :] - ) + parts = [] + offset = 0 + for entity, replacement in replacements: + parts.extend((text[offset : entity.start], replacement)) + offset = entity.end + parts.append(text[offset:]) + redacted_text = "".join(parts) return RedactResult( redacted_text=redacted_text, @@ -769,6 +933,8 @@ def scan_and_redact( allowlist_patterns: Optional[list[str]] = None, *, backend: Optional[str] = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Scan with the selected backend, then apply legacy Python redaction. @@ -783,4 +949,10 @@ def scan_and_redact( allowlist_patterns=allowlist_patterns, backend=backend, ) - return redact(text=text, entities=scan_result.entities, strategy=strategy) + return redact( + text=text, + entities=scan_result.entities, + strategy=strategy, + hash_type=hash_type, + hash_key=hash_key, + ) diff --git a/datafog/processing/text_processing/gliner_annotator.py b/datafog/processing/text_processing/gliner_annotator.py index 5a195f89..b871cf63 100644 --- a/datafog/processing/text_processing/gliner_annotator.py +++ b/datafog/processing/text_processing/gliner_annotator.py @@ -10,6 +10,8 @@ from pydantic import BaseModel, ConfigDict +from .regex_annotator.regex_annotator import AnnotationResult, Span + # Default entity types for PII detection using GLiNER # These can be customized based on specific use cases DEFAULT_PII_ENTITIES = [ @@ -92,55 +94,37 @@ def create( f"datafog download-model {model_name} --engine gliner" ) from e - def annotate(self, text: str) -> Dict[str, List[str]]: - """ - Annotate text for PII entities using GLiNER. - - Args: - text: Text to analyze for PII entities - - Returns: - Dictionary mapping entity types to lists of detected entities - """ - try: - if not text: - return { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } - - if len(text) > MAXIMAL_STRING_SIZE: - text = text[:MAXIMAL_STRING_SIZE] - logging.warning(f"Text truncated to {MAXIMAL_STRING_SIZE} characters") - - # Predict entities using GLiNER - entities = self.model.predict_entities(text, self.entity_types) - - # Organize results by entity type - classified_entities: Dict[str, List[str]] = { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } - + def annotate_with_spans(self, text: str): + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text) > MAXIMAL_STRING_SIZE: + raise ValueError( + f"GLiNER input exceeds {MAXIMAL_STRING_SIZE} characters; split the document before scanning" + ) + classified_entities = { + label.upper().replace(" ", "_"): [] for label in self.entity_types + } + spans = [] + if text: + try: + entities = self.model.predict_entities(text, self.entity_types) + except Exception as exc: + raise RuntimeError("GLiNER annotation failed") from exc for entity in entities: - entity_label = entity["label"].upper().replace(" ", "_") - entity_text = entity["text"] + label = entity["label"].upper().replace(" ", "_") + classified_entities.setdefault(label, []).append(entity["text"]) + spans.append( + Span( + label=label, + start=entity["start"], + end=entity["end"], + text=entity["text"], + ) + ) + return classified_entities, AnnotationResult(text=text, spans=spans) - if entity_label in classified_entities: - classified_entities[entity_label].append(entity_text) - else: - # Handle cases where GLiNER returns entity types not in our list - classified_entities[entity_label] = [entity_text] - - return classified_entities - - except Exception as e: - logging.error(f"Error processing text with GLiNER: {str(e)}") - # Return empty annotations in case of error - return { - entity_type.upper().replace(" ", "_"): [] - for entity_type in self.entity_types - } + def annotate(self, text: str) -> Dict[str, List[str]]: + return self.annotate_with_spans(text)[0] def set_entity_types(self, entity_types: List[str]) -> None: """ diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py index 44dfa2e0..2c132ac7 100644 --- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py +++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py @@ -40,11 +40,14 @@ class RegexAnnotator: ] LABELS = BASE_LABELS + GERMAN_LABELS DEFAULT_LABELS = BASE_LABELS - SUPPORTED_LOCALES = {"de", "de-de", "de_de"} + SUPPORTED_LOCALES = {"de", "de-de", "de_de", "en", "en-us", "fr"} LOCALE_LABELS = { "de": GERMAN_LABELS, "de-de": GERMAN_LABELS, "de_de": GERMAN_LABELS, + "en": [], + "en-us": [], + "fr": [], } def __init__( @@ -63,14 +66,14 @@ def __init__( # Note: This is broader than the spec to catch more potential emails "EMAIL": re.compile( r""" - (? (? (? (? (? tuple[str, . locales = [locales] normalized = [] for locale in locales: + if not isinstance(locale, str) or not locale.strip(): + raise ValueError("locales must contain nonempty locale identifiers") value = locale.strip().lower() - if not value: - continue if value not in cls.SUPPORTED_LOCALES: allowed = ", ".join(sorted(cls.SUPPORTED_LOCALES)) raise ValueError(f"locale must be one of: {allowed}") diff --git a/datafog/processing/text_processing/spacy_pii_annotator.py b/datafog/processing/text_processing/spacy_pii_annotator.py index d2d1d765..1cb2cbd2 100644 --- a/datafog/processing/text_processing/spacy_pii_annotator.py +++ b/datafog/processing/text_processing/spacy_pii_annotator.py @@ -1,8 +1,9 @@ -import logging from typing import Any, Dict, List from pydantic import BaseModel, ConfigDict +from .regex_annotator.regex_annotator import AnnotationResult, Span + PII_ANNOTATION_LABELS = [ "CARDINAL", "DATE", @@ -28,7 +29,7 @@ class SpacyPIIAnnotator(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) + model_config = ConfigDict(arbitrary_types_allowed=True, protected_namespaces=()) nlp: Any model_name: str = DEFAULT_SPACY_MODEL @@ -53,22 +54,32 @@ def create(cls, model_name: str = DEFAULT_SPACY_MODEL) -> "SpacyPIIAnnotator": return cls(nlp=nlp, model_name=model_name) - def annotate(self, text: str) -> Dict[str, List[str]]: - try: - if not text: - return {label: [] for label in PII_ANNOTATION_LABELS} - if len(text) > MAXIMAL_STRING_SIZE: - text = text[:MAXIMAL_STRING_SIZE] - doc = self.nlp(text) - classified_entities: Dict[str, List[str]] = { - label: [] for label in PII_ANNOTATION_LABELS - } + def annotate_with_spans(self, text: str): + if not isinstance(text, str): + raise TypeError("text must be a string") + if len(text) > MAXIMAL_STRING_SIZE: + raise ValueError( + f"spaCy input exceeds {MAXIMAL_STRING_SIZE} characters; split the document before scanning" + ) + classified_entities = {label: [] for label in PII_ANNOTATION_LABELS} + spans = [] + if text: + try: + doc = self.nlp(text) + except Exception as exc: + raise RuntimeError("spaCy annotation failed") from exc for ent in doc.ents: if ent.label_ in classified_entities: classified_entities[ent.label_].append(ent.text) - return classified_entities - except Exception as e: - logging.error(f"Error processing text for PII annotations: {str(e)}") - return { - label: [] for label in PII_ANNOTATION_LABELS - } # Return empty annotations in case of error + spans.append( + Span( + label=ent.label_, + start=ent.start_char, + end=ent.end_char, + text=ent.text, + ) + ) + return classified_entities, AnnotationResult(text=text, spans=spans) + + def annotate(self, text: str) -> Dict[str, List[str]]: + return self.annotate_with_spans(text)[0] diff --git a/datafog/services/text_service.py b/datafog/services/text_service.py index 6b80b26d..eee7ca90 100644 --- a/datafog/services/text_service.py +++ b/datafog/services/text_service.py @@ -61,10 +61,17 @@ def __init__( German structured identifiers. Raises: - AssertionError: If an invalid engine type is provided + ValueError: If the engine or text_chunk_length is invalid ImportError: If spacy/gliner engine is requested but corresponding extra is not installed """ - assert engine in {"regex", "spacy", "gliner", "auto", "smart"}, "Invalid engine" + if engine not in {"regex", "spacy", "gliner", "auto", "smart"}: + raise ValueError("engine must be one of: regex, spacy, gliner, auto, smart") + if ( + not isinstance(text_chunk_length, int) + or isinstance(text_chunk_length, bool) + or text_chunk_length <= 0 + ): + raise ValueError("text_chunk_length must be a positive integer") self.engine = engine self.text_chunk_length = text_chunk_length self.gliner_model = gliner_model @@ -379,17 +386,32 @@ def annotate_text_sync( _start = _time.monotonic() - if len(text) <= self.text_chunk_length: - # Single chunk processing - result = self._annotate_single_chunk(text, structured) + from datafog.engine import scan + + # Structured PII is scanned as a complete document. Fixed-size chunks + # lose entities at their boundaries; native model work chunks separately. + options = { + "engine": "smart" if self.engine == "auto" else self.engine, + "locales": self.locales, + } + if self.gliner_model != "urchade/gliner_multi_pii-v1": + options["gliner_model"] = self.gliner_model + entities = scan(text, **options).entities + if structured: + SpanClass = _get_span_class() + result = [ + SpanClass( + label=entity.type, + start=entity.start, + end=entity.end, + text=entity.text, + ) + for entity in entities + ] else: - # Multi-chunk processing - chunks = self._chunk_text(text) - - if structured: - result = self._annotate_multiple_chunks_structured(chunks) - else: - result = self._annotate_multiple_chunks_dict(chunks) + result = {} + for entity in entities: + result.setdefault(entity.type, []).append(entity.text) try: from datafog.telemetry import ( diff --git a/tests/test_allowlist.py b/tests/test_allowlist.py index f737a9f2..fc4b2865 100644 --- a/tests/test_allowlist.py +++ b/tests/test_allowlist.py @@ -93,13 +93,15 @@ def test_patterns_and_values_combine(self): class TestReDoSGuards: - def test_catastrophic_pattern_rejected(self): - with pytest.raises(ValueError, match="catastrophic backtracking"): - datafog.scan("text", engine="regex", allowlist_patterns=[r"(a+)+$"]) - - def test_nested_star_rejected(self): - with pytest.raises(ValueError, match="catastrophic backtracking"): - datafog.scan("text", engine="regex", allowlist_patterns=[r"(.*)*"]) + def test_nested_quantifier_uses_bounded_matching(self): + assert datafog.scan( + "a@b.com", engine="regex", allowlist_patterns=[r"(a+)+$"] + ).entities + + def test_safe_nested_star_is_supported(self): + assert not datafog.scan( + "a@b.com", engine="regex", allowlist_patterns=[r"(.*)*"] + ).entities def test_overlong_pattern_rejected(self): with pytest.raises(ValueError, match="at most"): diff --git a/tests/test_backend_resolution.py b/tests/test_backend_resolution.py index c6cd6d60..55abb06d 100644 --- a/tests/test_backend_resolution.py +++ b/tests/test_backend_resolution.py @@ -33,10 +33,10 @@ def test_auto_resolves_to_python_without_core(without_core): assert engine.resolve_backend(None, "regex") == "python" -def test_auto_keeps_non_regex_engines_on_python(): +def test_auto_uses_native_structured_layer_for_every_engine(): pytest.importorskip("datafog_core") for engine_name in ("smart", "spacy", "gliner"): - assert engine.resolve_backend(None, engine_name) == "python" + assert engine.resolve_backend(None, engine_name) == "rust" def test_python_fallback_warns_once_then_stays_quiet(without_core): diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index 8cd218a9..b0e2ae9f 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -210,9 +210,8 @@ def test_contextual_npi_survives_the_legacy_overlap_policy(): def test_structured_only_person_served_by_the_embedded_model(): """`PERSON` is structured-only in the rule inventory but the embedded CNN covers prose, so text selection succeeds instead of raising.""" - assert not datafog.scan( - "Jane Doe", entity_types=["PERSON"], backend="python" - ).entities + with pytest.raises(ValueError, match="Unsupported entity_types"): + datafog.scan("Jane Doe", entity_types=["PERSON"], backend="python") result = datafog.scan("Jane Doe\n14 Penny Lane", entity_types=["PERSON", "ADDRESS"]) assert [item.type for item in result.entities] == ["PERSON", "ADDRESS"] diff --git a/tests/test_engine_api.py b/tests/test_engine_api.py index 2a43e147..35aebe2a 100644 --- a/tests/test_engine_api.py +++ b/tests/test_engine_api.py @@ -198,7 +198,7 @@ def test_redact_invalid_strategy_raises_value_error() -> None: redact("test", entities=[], strategy="invalid") -def test_redact_ignores_invalid_spans() -> None: +def test_redact_rejects_invalid_spans() -> None: text = "hello" entities = [ Entity( @@ -219,9 +219,8 @@ def test_redact_ignores_invalid_spans() -> None: ), ] - result = redact(text=text, entities=entities, strategy="token") - assert result.redacted_text == text - assert result.mapping == {} + with pytest.raises(ValueError, match="valid ranges"): + redact(text=text, entities=entities, strategy="token") def test_scan_and_redact_combines_operations() -> None: diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py new file mode 100644 index 00000000..b10e7d33 --- /dev/null +++ b/tests/test_review_engine.py @@ -0,0 +1,306 @@ +"""Regression contracts for the 4.11 review's engine and annotation findings.""" + +import hashlib +import hmac +import random +from types import SimpleNamespace + +import pytest + +from datafog import engine +from datafog.processing.text_processing.gliner_annotator import GLiNERAnnotator +from datafog.processing.text_processing.regex_annotator.regex_annotator import ( + RegexAnnotator, +) +from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator +from datafog.services.text_service import TextService + + +def test_native_surrogate_normalization_preserves_source_and_offsets(): + text = "\ud800😀 ssn 123-45-6789 \udfff" + result = engine.scan(text, engine="regex", backend="rust") + assert result.text == text + assert any(entity.type == "SSN" for entity in result.entities) + assert all(text[item.start : item.end] == item.text for item in result.entities) + assert "123-45-6789" not in engine.redact(text, result.entities).redacted_text + + +def test_interval_selection_preserves_longest_and_tiebreak_contract(): + rng = random.Random(113) + for _ in range(30): + entities = [ + engine.Entity( + rng.choice(["PHONE", "SSN"]), + "", + start, + start + rng.randint(1, 30), + rng.random(), + "regex", + ) + for start in [rng.randint(0, 200) for _ in range(100)] + ] + expected = [] + for entity in sorted( + entities, + key=lambda item: ( + -engine._entity_length(item), + -engine.ENTITY_TYPE_PRIORITY.get(item.type, 0), + -item.confidence, + item.start, + item.end, + item.type, + ), + ): + if not any(engine._entities_overlap(entity, kept) for kept in expected): + expected.append(entity) + assert engine._suppress_overlapping_entities(entities) == sorted( + expected, key=lambda item: (item.start, item.end, item.type) + ) + + +@pytest.mark.parametrize("structured", [False, True]) +@pytest.mark.parametrize("value", ["john.doe@example.com", "123-45-6789"]) +def test_text_service_retains_findings_at_chunk_boundaries(value, structured): + text = "x" * 996 + " " + value + " tail" + result = TextService(text_chunk_length=1000).annotate_text_sync(text, structured) + if structured: + assert any( + item.text == value and text[item.start : item.end] == value + for item in result + ) + else: + assert any(value in items for items in result.values()) + + +def test_spacy_uses_original_offsets_and_does_not_turn_relative_dates_into_pii( + monkeypatch, +): + text = "Annual report prepared by Ann yesterday." + spans = [ + SimpleNamespace(label_="PERSON", text="Ann", start_char=26, end_char=29), + SimpleNamespace(label_="DATE", text="yesterday", start_char=30, end_char=39), + ] + annotator = SpacyPIIAnnotator(nlp=lambda _: SimpleNamespace(ents=spans)) + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: annotator) + result = engine.scan_and_redact(text, engine="spacy") + assert result.redacted_text == "Annual report prepared by [PERSON_1] yesterday." + + +def test_legacy_string_annotator_does_not_match_names_inside_words(): + result = engine._entities_from_dict( + "Annual report by Ann.", {"PERSON": ["Ann"]}, "spacy", 0.7 + ) + assert [(item.start, item.end) for item in result] == [(17, 20)] + + +@pytest.mark.parametrize("mode", ["smart", "spacy", "gliner"]) +def test_every_engine_keeps_native_structured_detection(monkeypatch, mode): + monkeypatch.setattr(engine, "_gliner_entities", lambda _: []) + monkeypatch.setattr(engine, "_spacy_entities", lambda _: []) + text = "card 6011111111111117 ip 2001:db8::1" + result = engine.scan_and_redact(text, engine=mode, backend="rust") + assert {item.type for item in result.entities} == {"CREDIT_CARD", "IP_ADDRESS"} + assert "6011111111111117" not in result.redacted_text + assert "2001:db8::1" not in result.redacted_text + + +@pytest.mark.parametrize("labels", ["EMAIL", ["BOGUS"], [1], [""]]) +def test_entity_type_mistakes_fail_explicitly(labels): + with pytest.raises(ValueError, match="entity_types"): + engine.scan("john@example.com", entity_types=labels) + + +@pytest.mark.parametrize("label", ["JWT", "PERSON", "ORGANIZATION", "UUID"]) +def test_unavailable_fallback_labels_raise(label): + with pytest.raises(ValueError, match="Unsupported entity_types"): + engine.scan( + "john@example.com", engine="regex", backend="python", entity_types=[label] + ) + + +@pytest.mark.parametrize("pattern", [r"(a|a)+$", r"((a+))+$"]) +def test_pathological_allowlist_cannot_suppress_a_finding(pattern): + value = "a" * 300 + "@b.com" + entity = engine.Entity("EMAIL", value, 0, len(value), 1.0, "regex") + assert engine._apply_allowlist([entity], None, [pattern]) == [entity] + + +@pytest.mark.parametrize( + "pattern,value", + [(r"(\d{3}){2}", "123123"), (r"(?:\d{3}-){2}\d{4}", "415-555-2671")], +) +def test_safe_quantified_allowlist_patterns_are_supported(pattern, value): + entity = engine.Entity("PHONE", value, 0, len(value), 1.0, "regex") + assert engine._apply_allowlist([entity], None, [pattern]) == [] + + +@pytest.mark.parametrize( + "annotator", + [ + SpacyPIIAnnotator( + nlp=lambda _: (_ for _ in ()).throw(RuntimeError("raw secret")) + ), + GLiNERAnnotator( + model=SimpleNamespace( + predict_entities=lambda *_: (_ for _ in ()).throw( + RuntimeError("raw secret") + ) + ), + entity_types=["person"], + model_name="test", + ), + ], +) +def test_ner_failures_and_oversize_inputs_are_explicit(annotator): + with pytest.raises(RuntimeError, match="annotation failed"): + annotator.annotate("secret") + with pytest.raises(ValueError, match="exceeds"): + annotator.annotate("x" * 1_000_001) + + +@pytest.mark.parametrize( + "label", ["passport number", "medical record number", "license number", "url"] +) +def test_gliner_extra_labels_and_original_offsets_are_preserved(monkeypatch, label): + text = "prefix SECRET suffix" + annotator = GLiNERAnnotator( + model=SimpleNamespace( + predict_entities=lambda *_: [ + {"label": label, "text": "SECRET", "start": 7, "end": 13} + ] + ), + entity_types=[label], + model_name="test", + ) + monkeypatch.setattr(engine, "_get_gliner_annotator", lambda: annotator) + result = engine.scan( + text, engine="gliner", entity_types=[label.upper().replace(" ", "_")] + ) + assert len(result.entities) == 1 + assert result.entities[0].text == "SECRET" + assert result.entities[0].start == 7 + + +def test_model_chunks_are_bounded_and_keep_document_offsets(): + text = "😀" * 9000 + " Jane Doe" + calls = [] + + def scan_model(chunk): + calls.append(chunk) + start = chunk.find("Jane Doe") + findings = ( + [] + if start < 0 + else [ + SimpleNamespace( + entity_type="PERSON", + codepoint_range=SimpleNamespace(start=start, end=start + 8), + matched_text="Jane Doe", + ) + ] + ) + return SimpleNamespace(findings=findings) + + findings = engine._cnn_findings( + SimpleNamespace(scan_model=scan_model), {"PERSON"}, text + ) + assert len(calls) >= 3 and max(len(chunk.encode()) for chunk in calls) <= 16384 + assert any( + offset + item.codepoint_range.start == text.index("Jane Doe") + for offset, item in findings + ) + + +def test_token_collision_and_repeated_value_mapping(): + text = "[EMAIL_1] a@b.com and a@b.com" + result = engine.scan_and_redact(text, engine="regex") + assert result.redacted_text == "[EMAIL_1] [EMAIL_2] and [EMAIL_2]" + assert result.mapping == {"[EMAIL_2]": "a@b.com"} + + +@pytest.mark.parametrize( + "entity", + [ + engine.Entity("EMAIL", "", 0, 1, 1.0, "regex"), + engine.Entity("EMAIL", "a", 0, 100, 1.0, "regex"), + engine.Entity("EMAIL", "wrong", 0, 1, 1.0, "regex"), + ], +) +def test_invalid_explicit_spans_are_rejected(entity): + with pytest.raises(ValueError, match="valid ranges"): + engine.redact("a", [entity]) + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256", "sha3_256"]) +def test_hash_algorithm_and_hmac_are_honored(algorithm): + text = "a@b.com" + entities = engine.scan(text, engine="regex").entities + plain = engine.redact(text, entities, strategy="hash", hash_type=algorithm) + assert hashlib.new(algorithm, text.encode()).hexdigest() in plain.redacted_text + key = bytes(range(32)) + keyed = engine.redact( + text, entities, strategy="hash", hash_type=algorithm, hash_key=key + ) + assert hmac.new(key, text.encode(), algorithm).hexdigest() in keyed.redacted_text + assert plain != keyed + + +def test_invalid_hash_parameters_fail(): + with pytest.raises(ValueError, match="hash_key"): + engine.redact("text", [], strategy="hash", hash_key=b"short") + with pytest.raises(ValueError, match="hash_type"): + engine.redact("text", [], strategy="hash", hash_type="bogus") + + +@pytest.mark.parametrize("locales", [[""], [1]]) +@pytest.mark.parametrize("backend", ["python", "rust"]) +def test_invalid_locales_are_consistently_rejected(locales, backend): + with pytest.raises(ValueError, match="locale"): + engine.scan("safe", engine="regex", locales=locales, backend=backend) + + +@pytest.mark.parametrize("locale", ["fr", "en-US"]) +def test_fallback_accepts_locale_tags_without_extra_detectors(locale): + assert engine.scan( + "a@b.com", engine="regex", locales=[locale], backend="python" + ).entities + + +def test_unicode_email_in_fallback(): + result = engine.scan("mail jörg@müller.de", engine="regex", backend="python") + assert result.entities[0].text == "jörg@müller.de" + + +@pytest.mark.parametrize( + "options", + [ + {"engine": "bogus"}, + {"text_chunk_length": 0}, + {"text_chunk_length": -1}, + {"text_chunk_length": True}, + ], +) +def test_text_service_argument_validation(options): + with pytest.raises(ValueError): + TextService(**options) + + +def test_fallback_adversarial_patterns_complete(): + annotator = RegexAnnotator(locales=["de"]) + for value in [ + "!" * 80000, + "#a" * 40000, + "Steuer-ID" + " " * 80000 + "x", + "Passport" + " " * 80000, + ]: + assert not any(annotator.annotate(value).values()) + + +def test_hash_marker_cannot_collide_with_literal_input(): + value = "a@b.com" + digest = hashlib.sha256(value.encode()).hexdigest() + marker = f"[EMAIL_{digest}]" + text = f"{marker} {value} {value}" + result = engine.scan_and_redact(text, engine="regex", strategy="hash") + assert result.mapping == {f"[EMAIL_{digest}_2]": value} + assert result.redacted_text == f"{marker} [EMAIL_{digest}_2] [EMAIL_{digest}_2]" From f8700e3997461da0f353d7831bb06c62e9a81b8b Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:56:06 -0800 Subject: [PATCH 02/20] fix(regex): keep email context boundaries linear and complete --- .../regex_annotator/regex_annotator.py | 23 ++++++++++---- tests/test_review_engine.py | 20 ++++++++++++ tests/test_rust_backend.py | 31 +++++++++++-------- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/datafog/processing/text_processing/regex_annotator/regex_annotator.py b/datafog/processing/text_processing/regex_annotator/regex_annotator.py index 2c132ac7..03641e82 100644 --- a/datafog/processing/text_processing/regex_annotator/regex_annotator.py +++ b/datafog/processing/text_processing/regex_annotator/regex_annotator.py @@ -67,8 +67,7 @@ def __init__( "EMAIL": re.compile( r""" (? str: - return match.groupdict().get("value") or match.group() + @classmethod + def _match_text(cls, match: Match[str]) -> str: + start, end = cls._match_span(match) + return match.string[start:end] @staticmethod def _match_span(match: Match[str]) -> tuple[int, int]: if "value" in match.groupdict() and match.group("value") is not None: return match.start("value"), match.end("value") - return match.start(), match.end() + start, end = match.start(), match.end() + value = match.group() + if "@" in value and any(char.isalnum() for char in value[: value.rfind("@")]): + local = value[: value.rfind("@")] + # Query/assignment separators and prose wrappers are context, + # not part of the mailbox. Consume the whole lexical run before + # trimming so failed candidates never retry every suffix. + boundary = max((local.rfind(char) for char in "&?=|/"), default=-1) + start += boundary + 1 + while start < end and match.string[start] in ".!?'*{}`": + start += 1 + return start, end @classmethod def create(cls, **kwargs) -> "RegexAnnotator": diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py index b10e7d33..020f0935 100644 --- a/tests/test_review_engine.py +++ b/tests/test_review_engine.py @@ -304,3 +304,23 @@ def test_hash_marker_cannot_collide_with_literal_input(): result = engine.scan_and_redact(text, engine="regex", strategy="hash") assert result.mapping == {f"[EMAIL_{digest}_2]": value} assert result.redacted_text == f"{marker} [EMAIL_{digest}_2] [EMAIL_{digest}_2]" + + +@pytest.mark.parametrize( + "text", + [ + "?" * 11 + "xiaoming@example.cn", + '{"note":"email=alice@example.com"}', + "email = 'dev@example.com'", + "phone=415-555-2671&email=bob@example.com", + ], +) +def test_fallback_email_context_is_trimmed_without_losing_mailboxes(text): + result = engine.scan(text, engine="regex", backend="python", entity_types=["EMAIL"]) + assert len(result.entities) == 1 + assert result.entities[0].text in { + "xiaoming@example.cn", + "alice@example.com", + "dev@example.com", + "bob@example.com", + } diff --git a/tests/test_rust_backend.py b/tests/test_rust_backend.py index 34da6a21..3cbdd607 100644 --- a/tests/test_rust_backend.py +++ b/tests/test_rust_backend.py @@ -66,15 +66,11 @@ def test_selection_alias_and_empty_selection(native, selection): assert len(result.entities) == 1 -def test_unknown_selection_preserves_legacy_empty_result(native): +def test_unknown_selection_is_rejected(native): findings, _ = native findings.append(finding("EMAIL", "a@example.com", 0, 13)) - assert ( - engine.scan( - "a@example.com", "regex", entity_types=["UNKNOWN"], backend="rust" - ).entities - == [] - ) + with pytest.raises(ValueError, match="Unsupported entity_types"): + engine.scan("a@example.com", "regex", entity_types=["UNKNOWN"], backend="rust") def test_unknown_native_label_preserved_without_silently_dropping_pii(native): @@ -84,7 +80,15 @@ def test_unknown_native_label_preserved_without_silently_dropping_pii(native): assert [item.type for item in result.entities] == ["FUTURE_LABEL"] -def test_python_overlap_priority_precedes_selection(native): +def test_python_overlap_priority_precedes_selection(native, monkeypatch): + metadata = capability_fixture() + metadata["supported_entities"].extend(["PHONE", "CREDIT_CARD"]) + for label in ("PHONE", "CREDIT_CARD"): + metadata["entities"][label] = { + "scopes": ["text"], + "activation": {"kind": "default"}, + } + monkeypatch.setattr(sys.modules["datafog_core"], "capabilities", lambda: metadata) findings, _ = native text = "1234567890123456" findings.extend( @@ -116,7 +120,7 @@ def test_python_allowlist_semantics(native, kwargs, count): ) -@pytest.mark.parametrize("pattern", ["[", "(a+)+", "x" * 513]) +@pytest.mark.parametrize("pattern", ["[", "x" * 513]) def test_allowlist_validation_before_native(native, pattern): _, calls = native with pytest.raises(ValueError): @@ -146,11 +150,12 @@ def test_unknown_locale_validation_preserved(native): @pytest.mark.parametrize("name", ["smart", "spacy", "gliner"]) -def test_nonregex_engine_rejected(native, name): +def test_nonregex_engines_use_native_structured_stage(native, name, monkeypatch): _, calls = native - with pytest.raises(ValueError, match="only engine='regex'"): - engine.scan("", name, backend="rust") - assert not calls + monkeypatch.setattr(engine, "_spacy_entities", lambda _: []) + monkeypatch.setattr(engine, "_gliner_entities", lambda _: []) + assert not engine.scan("", name, backend="rust").entities + assert calls == [""] def test_bad_backend_rejected(monkeypatch): From 67730c9662ff4b39cbaa0ad4add6a6a373dbedfd Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:00:22 -0800 Subject: [PATCH 03/20] fix(guardrails): scan complete requests and buffered streaming output --- datafog/agent.py | 24 +- datafog/integrations/claude_code.py | 35 +- datafog/integrations/litellm_guardrail.py | 306 +++++++++++---- tests/test_review_guardrails.py | 434 ++++++++++++++++++++++ 4 files changed, 717 insertions(+), 82 deletions(-) create mode 100644 tests/test_review_guardrails.py diff --git a/datafog/agent.py b/datafog/agent.py index e3cd7bdb..d8bfe56e 100644 --- a/datafog/agent.py +++ b/datafog/agent.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect import warnings from contextlib import contextmanager from dataclasses import dataclass @@ -101,14 +102,27 @@ def filter(self, text: str) -> RedactResult: return result def __call__(self, fn: F) -> F: - """Decorator that applies guardrail filtering to string return values.""" + """Filter string results, awaiting coroutine functions before scanning. + + Other result types raise TypeError rather than escaping unscanned. + """ + + def filtered(output: Any) -> str: + if not isinstance(output, str): + raise TypeError("A guarded function must return a string") + return self.filter(output).redacted_text + + if inspect.iscoroutinefunction(fn): + + @wraps(fn) + async def async_wrapped(*args: Any, **kwargs: Any) -> str: + return filtered(await fn(*args, **kwargs)) + + return async_wrapped # type: ignore[return-value] @wraps(fn) def wrapped(*args: Any, **kwargs: Any) -> Any: - output = fn(*args, **kwargs) - if isinstance(output, str): - return self.filter(output).redacted_text - return output + return filtered(fn(*args, **kwargs)) return wrapped # type: ignore[return-value] diff --git a/datafog/integrations/claude_code.py b/datafog/integrations/claude_code.py index 28c4e787..5eb72fb3 100644 --- a/datafog/integrations/claude_code.py +++ b/datafog/integrations/claude_code.py @@ -22,8 +22,9 @@ whose full text matches are exempt (note: a pattern containing a comma cannot be expressed here). -Failure policy: fail open. A hook bug must never brick a Claude Code -session, so any unexpected error exits non-blocking with no output. +Oversized inputs receive the configured ask/deny decision rather than proceeding +with an unscanned tail. Unexpected hook errors fail open with a non-blocking +error, so a hook bug does not prevent using the Claude Code session. """ import json @@ -50,6 +51,10 @@ _EXIT_FAIL_OPEN = 1 +class ScanLimitExceeded(ValueError): + """The payload cannot be fully inspected within the configured budget.""" + + def _entity_types(env: Mapping[str, str]) -> list[str]: raw = env.get("DATAFOG_HOOK_ENTITIES", "") parsed = [t.strip().upper() for t in raw.split(",") if t.strip()] @@ -82,6 +87,7 @@ def _iter_strings(value: Any) -> Iterator[str]: yield current elif isinstance(current, dict): stack.extend(current.values()) + stack.extend(current.keys()) elif isinstance(current, (list, tuple)): stack.extend(current) @@ -98,9 +104,9 @@ def _scan_findings( counts: dict[str, int] = {} total_budget = TOTAL_SCAN_CHARS for text in _iter_strings(value): - if total_budget <= 0: - break - chunk = text[: min(MAX_SCAN_CHARS, total_budget)] + if len(text) > MAX_SCAN_CHARS or len(text) > total_budget: + raise ScanLimitExceeded("Input exceeds the scan budget") + chunk = text total_budget -= len(chunk) result = datafog.scan( chunk, @@ -191,8 +197,20 @@ def run(payload: dict, env: Mapping[str, str]) -> tuple[int, str]: if handler is None: return _EXIT_OK, "" return _EXIT_OK, handler(payload, env) + except ScanLimitExceeded: + event = payload.get("hook_event_name") + reason = "DataFog PII firewall: input exceeds the scan budget and was not fully scanned. Split the input before proceeding." + if event == "PreToolUse": + return _EXIT_OK, _emit( + event, + { + "permissionDecision": _action(env), + "permissionDecisionReason": reason, + }, + ) + return _EXIT_OK, _emit(event, {"additionalContext": reason}) except Exception as exc: # noqa: BLE001 — fail open by design - print(f"datafog-hook error (fail-open): {exc}", file=sys.stderr) + print(f"datafog-hook error (fail-open): {type(exc).__name__}", file=sys.stderr) return _EXIT_FAIL_OPEN, "" @@ -206,7 +224,10 @@ def main() -> None: if not isinstance(payload, dict): payload = {} except Exception as exc: # noqa: BLE001 — fail open by design - print(f"datafog-hook: invalid hook payload (fail-open): {exc}", file=sys.stderr) + print( + f"datafog-hook: invalid hook payload (fail-open): {type(exc).__name__}", + file=sys.stderr, + ) sys.exit(_EXIT_FAIL_OPEN) code, stdout = run(payload, os.environ) diff --git a/datafog/integrations/litellm_guardrail.py b/datafog/integrations/litellm_guardrail.py index 7e33ed55..5611bec9 100644 --- a/datafog/integrations/litellm_guardrail.py +++ b/datafog/integrations/litellm_guardrail.py @@ -30,7 +30,9 @@ ``datafog`` core; the LiteLLM proxy, where this runs, always ships fastapi). """ +import copy import logging +import re from typing import Any, Optional from fastapi import HTTPException @@ -42,15 +44,74 @@ VALID_ACTIONS = {"redact", "block"} VALID_FAIL_POLICIES = {"open", "closed"} +MAX_STREAM_CHARS = 8_000_000 +MAX_STREAM_CHUNKS = 100_000 logger = logging.getLogger(__name__) +class _RedactionSession: + """Token assignments belong to one request or response, never the adapter.""" + + def __init__(self, strings=()): + self.tokens: dict[tuple[str, str], str] = {} + self.counters: dict[str, int] = {} + self.reserved = { + token for text in strings for token in re.findall(r"\[[A-Z_]+_\d+\]", text) + } + + def token(self, entity): + key = (entity.type, entity.text) + if key not in self.tokens: + number = self.counters.get(entity.type, 0) + 1 + while f"[{entity.type}_{number}]" in self.reserved: + number += 1 + token = f"[{entity.type}_{number}]" + self.counters[entity.type] = number + self.tokens[key] = token + self.reserved.add(token) + return self.tokens[key] + + +def _strings(value): + stack = [value] + visited = set() + while stack: + item = stack.pop() + if isinstance(item, (dict, list)): + if id(item) in visited: + continue + visited.add(id(item)) + if isinstance(item, str): + yield item + elif isinstance(item, dict): + stack.extend(item.values()) + stack.extend(item.keys()) + elif isinstance(item, list): + stack.extend(item) + + +def _get(value, name, default=None): + return ( + value.get(name, default) + if isinstance(value, dict) + else getattr(value, name, default) + ) + + +def _set(value, name, item): + if isinstance(value, dict): + value[name] = item + else: + setattr(value, name, item) + + def _redact_text( text: str, entity_types: list[str], allowlist: list[str] | None = None, allowlist_patterns: list[str] | None = None, + session: _RedactionSession | None = None, ) -> tuple[str, dict[str, int]]: """Redact ``text``; return (redacted_text, counts per entity type).""" import datafog @@ -65,7 +126,14 @@ def _redact_text( counts: dict[str, int] = {} for entity in result.entities: counts[entity.type] = counts.get(entity.type, 0) + 1 - return result.redacted_text, counts + session = session or _RedactionSession([text]) + parts = [] + cursor = 0 + for entity in sorted(result.entities, key=lambda entity: entity.start): + parts.extend((text[cursor : entity.start], session.token(entity))) + cursor = entity.end + parts.append(text[cursor:]) + return "".join(parts), counts def _summary(counts: dict[str, int]) -> str: @@ -97,39 +165,51 @@ def __init__( self.allowlist_patterns = allowlist_patterns super().__init__(**kwargs) - def _process_content(self, content: Any) -> tuple[Any, dict[str, int]]: - """Redact a message content value (str or list of content parts).""" + def _process_content( + self, content: Any, session=None + ) -> tuple[Any, dict[str, int]]: + """Walk textual input, including nested tool arguments and content parts.""" + session = session or _RedactionSession(_strings(content)) counts: dict[str, int] = {} if isinstance(content, str): redacted, counts = _redact_text( - content, self.entity_types, self.allowlist, self.allowlist_patterns + content, + self.entity_types, + self.allowlist, + self.allowlist_patterns, + session, ) return redacted, counts - if isinstance(content, list): - new_parts = [] - skipped_parts = 0 - for part in content: - if isinstance(part, dict) and isinstance(part.get("text"), str): - redacted, part_counts = _redact_text( - part["text"], - self.entity_types, - self.allowlist, - self.allowlist_patterns, - ) - new_parts.append({**part, "text": redacted}) - for etype, n in part_counts.items(): - counts[etype] = counts.get(etype, 0) + n + if isinstance(content, (list, dict)): + items = enumerate(content) if isinstance(content, list) else content.items() + output = [] if isinstance(content, list) else {} + for key, value in items: + # Media payloads are opaque; preserve them while scanning any + # textual sibling. name/id/url in ordinary user data are text. + media_type = content.get("type") if isinstance(content, dict) else None + if not isinstance(media_type, str): + media_type = None + opaque = { + "image_url": {"image_url"}, + "input_image": {"image_url", "file_id"}, + "image": {"source"}, + "input_audio": {"input_audio"}, + "audio": {"source", "data"}, + "video_url": {"video_url"}, + "file": {"file"}, + "input_file": {"file_id", "file_data"}, + } + if key in opaque.get(media_type, set()): + new_value, found = value, {} else: - # Images and other non-text parts are not scanned — - # count them so the blind spot is auditable. - new_parts.append(part) - skipped_parts += 1 - if skipped_parts: - logger.debug( - "DataFog guardrail: %d non-text content parts not scanned", - skipped_parts, - ) - return new_parts, counts + new_value, found = self._process_content(value, session) + if isinstance(output, list): + output.append(new_value) + else: + output[key] = new_value + for label, count in found.items(): + counts[label] = counts.get(label, 0) + count + return output, counts return content, counts def _handle_engine_error(self, exc: Exception) -> None: @@ -158,21 +238,20 @@ async def async_pre_call_hook( data: dict, call_type: str, ) -> dict: - messages = data.get("messages") - if not isinstance(messages, list): - return data - total_counts: dict[str, int] = {} - new_messages = [] + new_data = dict(data) + fields = ("messages", "prompt", "input") + session = _RedactionSession( + text for field in fields for text in _strings(data.get(field)) + ) try: - for message in messages: - if isinstance(message, dict) and "content" in message: - new_content, counts = self._process_content(message["content"]) - new_messages.append({**message, "content": new_content}) - for etype, n in counts.items(): - total_counts[etype] = total_counts.get(etype, 0) + n - else: - new_messages.append(message) + for field in fields: + if field in data: + new_data[field], counts = self._process_content( + data[field], session + ) + for label, count in counts.items(): + total_counts[label] = total_counts.get(label, 0) + count except Exception as exc: # noqa: BLE001 — fail policy decides self._handle_engine_error(exc) return data @@ -197,7 +276,6 @@ async def async_pre_call_hook( }, ) - new_data = {**data, "messages": new_messages} self._record_guardrail_logging(new_data, total_counts) return new_data @@ -215,42 +293,130 @@ def _record_guardrail_logging( except Exception: # noqa: BLE001 — observability must never break traffic logger.debug("could not record guardrail logging information") + @staticmethod + def _response_fields(response): + """Yield only generated text fields, preserving response metadata.""" + for choice in _get(response, "choices", []) or []: + if isinstance(_get(choice, "text"), str): + yield choice, "text" + message = _get(choice, "message") + if message is None: + continue + yield message, "content" + for call in _get(message, "tool_calls", []) or []: + function = _get(call, "function") + if function is not None: + yield function, "arguments" + function = _get(message, "function_call") + if function is not None: + yield function, "arguments" + async def async_post_call_success_hook( self, data: dict, user_api_key_dict: Any, response: Any, ) -> Any: - """Redact PII from model responses. + """Redact generated text in place after every field has been scanned.""" + try: + fields = list(self._response_fields(response)) + session = _RedactionSession( + text + for target, field in fields + for text in _strings(_get(target, field)) + ) + changes = [] + for target, field in fields: + value, _ = self._process_content(_get(target, field), session) + changes.append((target, field, value)) + for target, field, value in changes: + _set(target, field, value) + except Exception as exc: # noqa: BLE001 — fail policy decides + self._handle_engine_error(exc) + return response - Mutates ``response`` in place — deliberate: litellm post_call - guardrails share the response object rather than cloning it, and - an unredacted clone escaping through another callback would defeat - the purpose. + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict, response, request_data + ): + """Buffer to completion before emitting, protecting split entities. + + Native text and tool argument deltas are joined per choice/tool. The + first corresponding delta carries the redacted text and later deltas + are empty; metadata and finish events retain their original order. + Incomplete or oversized streams are rejected without emitting chunks. """ - choices = getattr(response, "choices", None) - if not choices: - return response + chunks = [] + groups = {} + size = 0 try: - skipped_parts = 0 - for choice in choices: - message = getattr(choice, "message", None) - if message is not None and isinstance(message.content, str): - redacted, counts = _redact_text( - message.content, - self.entity_types, - self.allowlist, - self.allowlist_patterns, - ) - if counts: - message.content = redacted - elif message is not None and message.content is not None: - skipped_parts += 1 - if skipped_parts: - logger.debug( - "DataFog guardrail: %d non-text response parts not scanned", - skipped_parts, + async for original in response: + # Iterators may reuse a mutable chunk object. Keep each emitted + # delta's value and metadata as they existed when received. + chunk = copy.deepcopy(original) + chunks.append(chunk) + for position, choice in enumerate(_get(chunk, "choices", []) or []): + index = _get(choice, "index", position) + fields = [((index, "text"), choice, "text")] + delta = _get(choice, "delta") + if delta is not None: + fields.append(((index, "content"), delta, "content")) + for call_position, call in enumerate( + _get(delta, "tool_calls", []) or [] + ): + function = _get(call, "function") + if function is not None: + fields.append( + ( + ( + index, + "tool", + _get(call, "index", call_position), + ), + function, + "arguments", + ) + ) + function = _get(delta, "function_call") + if function is not None: + fields.append( + ((index, "function_call"), function, "arguments") + ) + for key, target, field in fields: + value = _get(target, field) + if isinstance(value, str): + size += len(value) + groups.setdefault(key, []).append((target, field, value)) + if size > MAX_STREAM_CHARS or len(chunks) > MAX_STREAM_CHUNKS: + raise _StreamLimitExceeded() + except _StreamLimitExceeded: + raise RuntimeError( + "DataFog streaming response exceeded the scan budget; rejecting unscanned output" + ) from None + except Exception as exc: + # A failed upstream stream cannot be safely completed or replayed. + # Provider exceptions can contain private prompt/response text. + raise RuntimeError( + f"DataFog streaming response did not complete ({type(exc).__name__}); rejecting incomplete output" + ) from None + try: + texts = [ + "".join(value for _, _, value in group) for group in groups.values() + ] + session = _RedactionSession(texts) + changes = [] + for group, text in zip(groups.values(), texts): + redacted, _ = self._process_content(text, session) + changes.extend( + (target, field, redacted if i == 0 else "") + for i, (target, field, _) in enumerate(group) ) - except Exception as exc: # noqa: BLE001 — fail policy decides + for target, field, value in changes: + _set(target, field, value) + except Exception as exc: self._handle_engine_error(exc) - return response + for chunk in chunks: + yield chunk + + +class _StreamLimitExceeded(Exception): + """Internal marker; stream-budget failures always reject output.""" diff --git a/tests/test_review_guardrails.py b/tests/test_review_guardrails.py new file mode 100644 index 00000000..ea6c4cbe --- /dev/null +++ b/tests/test_review_guardrails.py @@ -0,0 +1,434 @@ +"""Review regressions for request/response coverage and safe stream failures.""" + +# ruff: noqa: E402 +# Optional proxy dependencies must be present before importing the adapter. +import copy +import json + +import pytest + +pytest.importorskip("litellm") +pytest.importorskip("fastapi") +from fastapi import HTTPException # noqa: E402 +from litellm import ModelResponseStream # noqa: E402 +from litellm.types.utils import Delta, StreamingChoices # noqa: E402 + +import datafog # noqa: E402 +from datafog.integrations import claude_code as hook # noqa: E402 +from datafog.integrations import litellm_guardrail as gateway # noqa: E402 + +EMAIL_A = "alice@" "example.com" +EMAIL_B = "bob@" "example.com" +SSN = "123-45-" "6789" + + +def guard(**kwargs): + return gateway.DataFogGuardrail(guardrail_name="review", **kwargs) + + +async def pre(adapter, data): + return await adapter.async_pre_call_hook(None, None, data, "completion") + + +async def source(chunks): + for chunk in chunks: + yield chunk + + +async def streamed(adapter, chunks): + return [ + chunk + async for chunk in adapter.async_post_call_streaming_iterator_hook( + None, source(chunks), {} + ) + ] + + +@pytest.mark.asyncio +async def test_request_shares_tokens_without_existing_token_collisions(): + original = { + "messages": [ + {"role": "user", "content": f"[EMAIL_1] {EMAIL_A}"}, + {"role": "user", "content": f"other {EMAIL_B}; same {EMAIL_A}"}, + ], + "prompt": EMAIL_B, + "input": [{"content": EMAIL_A}], + } + before = copy.deepcopy(original) + result = await pre(guard(), original) + assert result["messages"][0]["content"] == "[EMAIL_1] [EMAIL_2]" + assert result["messages"][1]["content"] == "other [EMAIL_3]; same [EMAIL_2]" + assert ( + result["prompt"] == "[EMAIL_3]" and result["input"][0]["content"] == "[EMAIL_2]" + ) + assert original == before + another = await pre(guard(), {"prompt": EMAIL_A}) + assert another["prompt"] == "[EMAIL_1]" + + +@pytest.mark.asyncio +async def test_request_scans_tool_arguments_and_structured_input_values(): + data = { + "messages": [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "save", + "arguments": json.dumps({"email": EMAIL_A, "ssn": SSN}), + }, + } + ], + } + ], + "input": {"name": EMAIL_B, "id": SSN, "url": "send " + EMAIL_A}, + } + result = await pre(guard(), data) + assert ( + EMAIL_A not in result["messages"][0]["tool_calls"][0]["function"]["arguments"] + ) + assert SSN not in result["messages"][0]["tool_calls"][0]["function"]["arguments"] + assert result["input"] == { + "name": "[EMAIL_2]", + "id": "[SSN_1]", + "url": "send [EMAIL_1]", + } + assert result["messages"][0]["tool_calls"][0]["id"] == "call_1" + + +@pytest.mark.asyncio +async def test_multimodal_media_remains_intact_and_text_is_scanned(): + image = { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,AA4242424242424242", + "detail": "high", + }, + } + audio = { + "type": "input_audio", + "input_audio": {"data": "4242424242424242", "format": "wav"}, + } + result = await pre( + guard(), + { + "messages": [ + { + "role": "user", + "content": [image, audio, {"type": "text", "text": EMAIL_A}], + } + ] + }, + ) + assert result["messages"][0]["content"][:2] == [image, audio] + assert result["messages"][0]["content"][2]["text"] == "[EMAIL_1]" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["messages", "prompt", "input"]) +async def test_surrogate_cannot_disable_blocking(field): + value = f"ssn {SSN} \ud83d" + data = ( + {field: [{"role": "user", "content": value}]} + if field == "messages" + else {field: value} + ) + with pytest.raises(HTTPException) as error: + await pre(guard(action="block"), data) + assert error.value.status_code == 400 and SSN not in str(error.value.detail) + + +@pytest.mark.asyncio +async def test_response_reserves_tokens_in_tools_and_covers_legacy_and_text(): + response = { + "choices": [ + { + "index": 0, + "message": { + "content": EMAIL_A, + "tool_calls": [ + { + "function": { + "name": "save", + "arguments": f"[EMAIL_1] {EMAIL_B}", + } + } + ], + }, + }, + { + "index": 1, + "message": { + "content": None, + "function_call": {"name": "old", "arguments": EMAIL_A}, + }, + }, + {"index": 2, "text": EMAIL_B}, + ] + } + result = await guard().async_post_call_success_hook({}, None, response) + assert result is response + assert result["choices"][0]["message"]["content"] == "[EMAIL_2]" + assert ( + result["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] + == "[EMAIL_1] [EMAIL_3]" + ) + assert result["choices"][1]["message"]["function_call"]["arguments"] == "[EMAIL_2]" + assert result["choices"][2]["text"] == "[EMAIL_3]" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_response_scanning_is_atomic_and_respects_fail_policy( + monkeypatch, policy +): + actual = gateway._redact_text + + def failing(text, *args, **kwargs): + if text == "second": + raise RuntimeError(EMAIL_A) + return actual(text, *args, **kwargs) + + monkeypatch.setattr(gateway, "_redact_text", failing) + response = { + "choices": [ + {"message": {"content": EMAIL_A}}, + {"message": {"content": "second"}}, + ] + } + before = copy.deepcopy(response) + if policy == "closed": + with pytest.raises(RuntimeError) as error: + await guard(fail_policy=policy).async_post_call_success_hook( + {}, None, response + ) + assert EMAIL_A not in str(error.value) + else: + assert ( + await guard(fail_policy=policy).async_post_call_success_hook( + {}, None, response + ) + is response + ) + assert response == before + + +@pytest.mark.asyncio +async def test_stream_scans_all_choice_and_tool_argument_fragments(): + chunks = [ + { + "id": "first", + "choices": [ + { + "index": 0, + "delta": { + "content": "[EMAIL_1] alice@", + "tool_calls": [ + { + "index": 0, + "id": "call_a", + "function": { + "name": "save", + "arguments": '{"ssn":"123-', + }, + }, + { + "index": 1, + "id": "call_b", + "function": {"name": "save", "arguments": "bob@"}, + }, + ], + }, + }, + { + "index": 1, + "delta": { + "content": "bob@", + "function_call": {"name": "old", "arguments": "alice@"}, + }, + }, + {"index": 2, "text": "bob@"}, + ], + }, + { + "id": "last", + "choices": [ + { + "index": 0, + "delta": { + "content": "example.com", + "tool_calls": [ + {"index": 0, "function": {"arguments": '45-6789"}'}}, + {"index": 1, "function": {"arguments": "example.com"}}, + ], + }, + "finish_reason": "tool_calls", + }, + { + "index": 1, + "delta": { + "content": "example.com", + "function_call": {"arguments": "example.com"}, + }, + "finish_reason": "stop", + }, + {"index": 2, "text": "example.com", "finish_reason": "stop"}, + ], + }, + ] + original = copy.deepcopy(chunks) + output = await streamed(guard(), chunks) + first, last = output + assert first["choices"][0]["delta"]["content"] == "[EMAIL_1] [EMAIL_2]" + calls = first["choices"][0]["delta"]["tool_calls"] + assert calls[0]["function"]["arguments"] == '{"ssn":"[SSN_1]"}' + assert calls[1]["function"]["arguments"] == "[EMAIL_3]" + assert first["choices"][1]["delta"]["content"] == "[EMAIL_3]" + assert first["choices"][1]["delta"]["function_call"]["arguments"] == "[EMAIL_2]" + assert first["choices"][2]["text"] == "[EMAIL_3]" + assert last["choices"][0]["delta"]["content"] == "" + assert last["choices"][0]["delta"]["tool_calls"][0]["function"]["arguments"] == "" + assert last["choices"][0]["finish_reason"] == "tool_calls" + assert chunks == original + assert first["id"] == "first" and last["id"] == "last" + + +@pytest.mark.asyncio +async def test_stream_handles_real_litellm_native_chunks(): + chunks = [ + ModelResponseStream( + id="same", choices=[StreamingChoices(index=0, delta=Delta(content=part))] + ) + for part in ("alice@", "example.com") + ] + output = await streamed(guard(), chunks) + assert output[0].choices[0].delta.content == "[EMAIL_1]" + assert output[1].choices[0].delta.content == "" + assert chunks[0].choices[0].delta.content == "alice@" + + +@pytest.mark.asyncio +async def test_stream_snapshots_reused_mutable_chunks(): + async def reused(): + chunk = {"choices": [{"index": 0, "delta": {"content": "alice@"}}]} + yield chunk + chunk["choices"][0]["delta"]["content"] = "example.com" + yield chunk + + output = [ + chunk + async for chunk in guard().async_post_call_streaming_iterator_hook( + None, reused(), {} + ) + ] + assert output[0]["choices"][0]["delta"]["content"] == "[EMAIL_1]" + assert output[1]["choices"][0]["delta"]["content"] == "" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_stream_source_errors_emit_nothing_and_hide_provider_text(policy): + async def broken(): + yield {"choices": [{"delta": {"content": EMAIL_A}}]} + raise RuntimeError("provider failed on " + EMAIL_A) + + emitted = [] + with pytest.raises(RuntimeError) as error: + async for chunk in guard( + fail_policy=policy + ).async_post_call_streaming_iterator_hook(None, broken(), {}): + emitted.append(chunk) + assert not emitted and EMAIL_A not in str(error.value) + assert error.value.__cause__ is None and error.value.__suppress_context__ + + +@pytest.mark.asyncio +@pytest.mark.parametrize("limit", ["MAX_STREAM_CHARS", "MAX_STREAM_CHUNKS"]) +async def test_stream_limits_reject_without_emission(monkeypatch, limit): + monkeypatch.setattr(gateway, limit, 1) + chunks = [{"choices": [{"delta": {"content": "ab"}}]}] * 2 + emitted = [] + with pytest.raises(RuntimeError, match="scan budget"): + async for chunk in guard().async_post_call_streaming_iterator_hook( + None, source(chunks), {} + ): + emitted.append(chunk) + assert not emitted + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["open", "closed"]) +async def test_stream_engine_failures_follow_documented_policy(monkeypatch, policy): + def failing(*args, **kwargs): + raise RuntimeError(EMAIL_A) + + monkeypatch.setattr(gateway, "_redact_text", failing) + chunks = [{"choices": [{"delta": {"content": EMAIL_A}}]}] + if policy == "open": + assert await streamed(guard(fail_policy=policy), chunks) == chunks + else: + with pytest.raises(RuntimeError, match="fail_policy is 'closed'") as error: + await streamed(guard(fail_policy=policy), chunks) + assert EMAIL_A not in str(error.value) + + +@pytest.mark.parametrize("action", ["ask", "deny"]) +def test_hook_oversize_never_silently_allows_unscanned_tail(action): + payload = { + "hook_event_name": "PreToolUse", + "tool_name": "Write", + "tool_input": {"content": "x" * (hook.MAX_SCAN_CHARS + 1) + " " + SSN}, + } + code, output = hook.run(payload, {"DATAFOG_HOOK_ACTION": action}) + decision = json.loads(output)["hookSpecificOutput"] + assert code == 0 and decision["permissionDecision"] == action + assert ( + "not fully scanned" in decision["permissionDecisionReason"] + and SSN not in output + ) + + +def test_hook_surrogate_and_total_budget_cannot_disable_pii_gate(monkeypatch): + payload = { + "hook_event_name": "PreToolUse", + "tool_input": {"command": "echo " + SSN, "description": "\udc00"}, + } + code, output = hook.run(payload, {}) + assert code == 0 and "SSN x1" in output + monkeypatch.setattr(hook, "TOTAL_SCAN_CHARS", 20) + code, output = hook.run( + {"hook_event_name": "PreToolUse", "tool_input": ["a" * 15, "b" * 15]}, {} + ) + assert ( + code == 0 + and json.loads(output)["hookSpecificOutput"]["permissionDecision"] == "ask" + ) + + +@pytest.mark.asyncio +async def test_decorator_awaits_and_rejects_non_string_returns(): + adapter = datafog.create_guardrail() + + @adapter + async def generated(): + return EMAIL_A + + assert await generated() == "[EMAIL_1]" + + @adapter + async def structured(): + return [EMAIL_A] + + with pytest.raises(TypeError, match="must return a string"): + await structured() + + @adapter + def sync_structured(): + return {"text": EMAIL_A} + + with pytest.raises(TypeError, match="must return a string"): + sync_structured() From a9c19fc0b927f50bd5b98639d41feaa8f033251d Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:00:22 -0800 Subject: [PATCH 04/20] test: review wrapper contract changes and qualify shared detection --- datafog/engine.py | 59 +- tests/contracts/rust-2.0.0.json | 77 ++ tests/contracts/wrapper-4.12.0.json | 1149 ++++++++++++++++++++++++ tests/rust_contract.py | 23 +- tests/test_contract_481.py | 67 +- tests/test_core04_integration.py | 7 +- tests/test_core20_integration.py | 58 ++ tests/test_de_pii_regex.py | 5 +- tests/test_rust_contract.py | 11 +- tests/test_spacy_span_contract.py | 56 ++ tests/test_text_service.py | 428 ++------- tests/test_text_service_integration.py | 28 +- 12 files changed, 1539 insertions(+), 429 deletions(-) create mode 100644 tests/contracts/rust-2.0.0.json create mode 100644 tests/contracts/wrapper-4.12.0.json create mode 100644 tests/test_core20_integration.py create mode 100644 tests/test_spacy_span_contract.py diff --git a/datafog/engine.py b/datafog/engine.py index ec33081b..8ce61a5f 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -793,6 +793,35 @@ def scan( ) +def _validate_hash_parameters( + strategy: str, hash_type: str, hash_key: bytes | None +) -> None: + if hash_key is not None and (not isinstance(hash_key, bytes) or len(hash_key) < 32): + raise ValueError("hash_key must contain at least 32 bytes") + if strategy == "hash": + try: + hashlib.new(hash_type).hexdigest() + except (ValueError, TypeError): + raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None + + +def _validated_redaction_entities(text: str, entities: list[Entity]) -> list[Entity]: + valid_entities = [] + for entity in entities: + if ( + not 0 <= entity.start < entity.end <= len(text) + or not entity.text + or text[entity.start : entity.end] != entity.text + ): + raise ValueError( + "entities must contain valid ranges whose text matches the source" + ) + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", entity.type): + raise ValueError("entity types must be uppercase labels") + valid_entities.append(entity) + return _suppress_overlapping_entities(valid_entities) + + def redact( text: str, entities: list[Entity], @@ -809,6 +838,8 @@ def redact( type priority and then confidence. Suppressed spans are omitted from ``RedactResult.entities`` and ``mapping``. + Invalid ranges, mismatched source text, and malformed labels raise ValueError. + ``mapping`` for the ``mask`` strategy is keyed by per-type indexed keys (``[EMAIL_MASK_1]``) rather than the mask string, because distinct same-length values produce identical masks and would collide. @@ -818,39 +849,19 @@ def redact( if strategy not in {"token", "mask", "hash", "pseudonymize"}: raise ValueError("strategy must be one of: token, mask, hash, pseudonymize") - if hash_key is not None and (not isinstance(hash_key, bytes) or len(hash_key) < 32): - raise ValueError("hash_key must contain at least 32 bytes") - if strategy == "hash": - try: - hashlib.new(hash_type).hexdigest() - except (ValueError, TypeError): - raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None + _validate_hash_parameters(strategy, hash_type, hash_key) mapping: dict[str, str] = {} counters: dict[str, int] = {} pseudonym_by_value: dict[tuple[str, str], str] = {} - valid_entities = [] - for entity in entities: - if ( - not 0 <= entity.start < entity.end <= len(text) - or not entity.text - or text[entity.start : entity.end] != entity.text - ): - raise ValueError( - "entities must contain valid ranges whose text matches the source" - ) - if not re.fullmatch(r"[A-Z][A-Z0-9_]*", entity.type): - raise ValueError("entity types must be uppercase labels") - valid_entities.append(entity) - # Returns non-overlapping spans sorted by document position. - valid_entities = _suppress_overlapping_entities(valid_entities) + valid_entities = _validated_redaction_entities(text, entities) occupied_tokens = set(re.findall(r"\[[A-Z][A-Za-z0-9_]*\]", text)) # Assign replacements in document order so the per-type counters used by - # the token/pseudonymize strategies number the first occurrence _1; spans - # are applied right-to-left below so earlier offsets stay valid. + # the token/pseudonymize strategies number the first occurrence _1. + # Assemble the result in one pass without repeated document copies. replacements: list[tuple[Entity, str]] = [] for entity in valid_entities: original = text[entity.start : entity.end] diff --git a/tests/contracts/rust-2.0.0.json b/tests/contracts/rust-2.0.0.json new file mode 100644 index 00000000..324f9cfa --- /dev/null +++ b/tests/contracts/rust-2.0.0.json @@ -0,0 +1,77 @@ +{ + "core_version": "2.0.0", + "cases": { + "scan-observation-invalid-card": { + "classification": "detector-difference", + "reason": "Core validates card checksums; Python 4.8.1 accepts this card-shaped value. Experimental precision difference, not a claim of full parity.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "4111 1111 1111 1112", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "scan-observation-boundary": { + "classification": "detector-difference", + "reason": "Core rejects the SSN embedded in an ASCII alphanumeric token; Python 4.8.1 detects the numeric substring.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "x123-45-6789y", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "locale-unknown": { + "classification": "validation-difference", + "reason": "The capability adapter rejects the unadvertised locale with a backend-specific diagnostic instead of the Python regex locale inventory. The exception remains ValueError.", + "expected": { + "error": { + "type": "ValueError", + "message": "Unsupported locale for the Rust backend: 'xx'" + }, + "warnings": [] + } + }, + "scan-observation-ambiguous-number": { + "classification": "detector-difference", + "reason": "Core's North American area-code check requires a leading digit of 2-9, so a bare 10-digit run starting with 1 stays undetected. This is the deliberate Unix-timestamp protection; Python 4.8.1 matches it.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Invoice 1234567890", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "locale-default-DE_TAX_ID": { + "classification": "detector-difference", + "reason": "CORE-17/PY-L2 suppress a German tax identifier from the generic PHONE detector even when German detectors are not activated. Explicit locale selection still detects DE_TAX_ID.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "Steuer-ID 12345678901 liegt vor.", + "engine_used": "regex" + } + }, + "warnings": [] + } + } + }, + "review": "Core 2.0 candidate qualifications retain the four reviewed historical differences below. Wrapper 4.12 fixes are layered separately in wrapper-4.12.0.json." +} diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json new file mode 100644 index 00000000..e4c0835b --- /dev/null +++ b/tests/contracts/wrapper-4.12.0.json @@ -0,0 +1,1149 @@ +{ + "wrapper_version": "4.12.0", + "core_versions": [ + "1.4.0", + "2.0.0" + ], + "source_oracle": "4.8.1.json (unchanged)", + "review": "2026-11-15 PDF remediation: explicit reviewed wrapper changes, not a regenerated historical oracle", + "cases": { + "selection-unknown": { + "classification": "validation-difference", + "reason": "PY-M6 rejects unknown entity labels instead of silently disabling detection.", + "expected": { + "error": { + "type": "ValueError", + "message": "Unsupported entity_types for rust/regex: NOT_A_TYPE" + }, + "warnings": [] + } + }, + "redact-token": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses one collision-safe token for repeated values.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_2] [EMAIL_1] [SSN_1]", + "mapping": { + "[EMAIL_1]": "a@example.com", + "[EMAIL_2]": "b@example.com", + "[SSN_1]": "123-45-6789" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "SSN", + "text": "123-45-6789", + "start": 42, + "end": 53, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "engine-redact-token": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses one collision-safe token for repeated values.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_2] [EMAIL_1]", + "mapping": { + "[EMAIL_1]": "a@example.com", + "[EMAIL_2]": "b@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "redact-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c] [EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [SSN_01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0]", + "mapping": { + "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]": "a@example.com", + "[EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c]": "b@example.com", + "[SSN_01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0]": "123-45-6789" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "SSN", + "text": "123-45-6789", + "start": 42, + "end": 53, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "engine-redact-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c] [EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]", + "mapping": { + "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]": "a@example.com", + "[EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c]": "b@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 0, + "end": 13, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "b@example.com", + "start": 14, + "end": 27, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "a@example.com", + "start": 28, + "end": 41, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "preset-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]", + "mapping": { + "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]": "alice@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 0, + "end": 17, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "explicit-invalid-bounds": { + "classification": "validation-difference", + "reason": "PY-L7 rejects malformed explicit spans rather than silently skipping them.", + "expected": { + "error": { + "type": "ValueError", + "message": "entities must contain valid ranges whose text matches the source" + }, + "warnings": [] + } + }, + "explicit-mismatched-text": { + "classification": "validation-difference", + "reason": "PY-L7 rejects explicit span text that differs from the source.", + "expected": { + "error": { + "type": "ValueError", + "message": "entities must contain valid ranges whose text matches the source" + }, + "warnings": [] + } + }, + "error-invalid-pattern": { + "classification": "validation-difference", + "reason": "PY-M9 uses a timeout-capable matcher, whose syntax diagnostic counts positions differently.", + "expected": { + "error": { + "type": "ValueError", + "message": "allowlist_patterns contains an invalid regex: '[' (unterminated character set at position 1)" + }, + "warnings": [] + } + }, + "error-unsafe-pattern": { + "classification": "behavioral-fix", + "reason": "PY-M9 allows nested groups under a hard matching timeout instead of a bypassable syntax heuristic.", + "expected": { + "value": { + "result_type": "ScanResult", + "fields": { + "entities": [], + "text": "", + "engine_used": "regex" + } + }, + "warnings": [] + } + }, + "error-oversized-pattern": { + "classification": "validation-difference", + "reason": "PY-M9 validates both pattern type and length with an explicit diagnostic.", + "expected": { + "error": { + "type": "ValueError", + "message": "allowlist_patterns entries must be strings of at most 512 characters" + }, + "warnings": [] + } + }, + "facade-sanitize": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": "[EMAIL_1] [EMAIL_1]", + "warnings": [] + } + }, + "facade-filter_output": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": { + "result_type": "RedactResult", + "fields": { + "redacted_text": "[EMAIL_1] [EMAIL_1]", + "mapping": { + "[EMAIL_1]": "alice@example.com" + }, + "entities": [ + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 0, + "end": 17, + "confidence": 1.0, + "engine": "regex" + } + }, + { + "result_type": "Entity", + "fields": { + "type": "EMAIL", + "text": "alice@example.com", + "start": 18, + "end": 35, + "confidence": 1.0, + "engine": "regex" + } + } + ] + } + }, + "warnings": [] + } + }, + "facade-detect": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + }, + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 18, + "end": 35 + } + ], + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.detect() is deprecated and will be removed in 5.0. Use datafog.scan() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "facade-anonymize_text": { + "classification": "behavioral-fix", + "reason": "PY-L7 reuses the same token when the same entity value repeats.", + "expected": { + "value": "[EMAIL_1] [EMAIL_1]", + "warnings": [] + } + }, + "facade-process": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + }, + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 18, + "end": 35 + } + ] + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-redact": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_REDACTED]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-replace": { + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release that still provides the shim.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_XXXXX]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 replaces legacy truncated MD5 with the complete selected digest; PY-L3 updates the retirement notice.", + "expected": { + "value": { + "original": "alice@example.com", + "findings": [ + { + "type": "EMAIL", + "value": "alice@example.com", + "start": 0, + "end": 17 + } + ], + "anonymized": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "process-invalid": { + "classification": "validation-difference", + "reason": "PY-L5 rejects an invalid anonymization method instead of choosing an undocumented fallback; PY-L3 updates the notice.", + "expected": { + "error": { + "type": "ValueError", + "message": "method must be 'redact', 'replace', or 'hash'" + }, + "warnings": [ + { + "type": "FutureWarning", + "message": "datafog.process() is deprecated and will be removed in 5.0. Use datafog.scan() or datafog.redact() instead. Last supported version is 4.12.0. Pin it with: pip install \"datafog==4.12.0\". The earlier promise to retain this shim through 5.x has been revised. See docs/retirement.rst." + } + ] + } + }, + "service-default": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route TextService through shared native detection and return populated canonical labels.", + "expected": { + "value": { + "EMAIL": [ + "alice@example.com" + ] + }, + "warnings": [] + } + }, + "service-german": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route TextService through shared native detection with overlap resolution and populated canonical labels.", + "expected": { + "value": { + "DE_TAX_ID": [ + "12345678901" + ] + }, + "warnings": [] + } + }, + "service-batch": { + "classification": "behavioral-fix", + "reason": "PY-H3/PY-L6 route every TextService batch member through shared native detection, returning populated canonical labels.", + "expected": { + "value": [ + { + "EMAIL": [ + "alice@example.com" + ] + }, + {} + ], + "warnings": [] + } + }, + "public-signatures": { + "classification": "signature-addition", + "reason": "PY-M7 adds keyword-only hash_type/hash_key; TextService custom models use the engine gliner_model keyword; backend remains keyword-only.", + "expected": { + "value": { + "datafog:scan": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "preset", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:protect": { + "parameters": [ + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "on_detect", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + } + ] + }, + "datafog:detect": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:process": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "anonymize", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": false + }, + { + "name": "method", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog:sanitize": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:scan_prompt": { + "parameters": [ + { + "name": "prompt", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:filter_output": { + "parameters": [ + { + "name": "output", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "required": true, + "default": null + } + ] + }, + "datafog:create_guardrail": { + "parameters": [ + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "regex" + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "on_detect", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "redact" + } + ] + }, + "datafog.engine:Entity": { + "parameters": [ + { + "name": "type", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "start", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "end", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "confidence", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": [ + "type", + "text", + "start", + "end", + "confidence", + "engine" + ] + }, + "datafog.engine:ScanResult": { + "parameters": [ + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine_used", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": [ + "entities", + "text", + "engine_used" + ] + }, + "datafog.engine:RedactResult": { + "parameters": [ + { + "name": "redacted_text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "mapping", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + } + ], + "fields": [ + "redacted_text", + "mapping", + "entities" + ] + }, + "datafog.engine:scan": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "smart" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "gliner_model", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog.engine:redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "entities", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + }, + "datafog.engine:scan_and_redact": { + "parameters": [ + { + "name": "text", + "kind": "POSITIONAL_OR_KEYWORD", + "required": true, + "default": null + }, + { + "name": "engine", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "smart" + }, + { + "name": "entity_types", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "strategy", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": "token" + }, + { + "name": "locales", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "allowlist_patterns", + "kind": "POSITIONAL_OR_KEYWORD", + "required": false, + "default": null + }, + { + "name": "backend", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + }, + { + "name": "hash_type", + "kind": "KEYWORD_ONLY", + "required": false, + "default": "sha256" + }, + { + "name": "hash_key", + "kind": "KEYWORD_ONLY", + "required": false, + "default": null + } + ] + } + }, + "warnings": [] + } + }, + "anonymize-text-hash": { + "classification": "behavioral-fix", + "reason": "PY-M7 emits the complete chosen digest rather than a truncated unsalted token.", + "expected": { + "value": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]", + "warnings": [] + } + } + } +} diff --git a/tests/rust_contract.py b/tests/rust_contract.py index 2b618007..9e673091 100644 --- a/tests/rust_contract.py +++ b/tests/rust_contract.py @@ -9,7 +9,11 @@ from tests.contract_481 import FIXTURE, observe -DEVIATIONS = Path(__file__).parent / "contracts" / "rust-1.4.0.json" +CORE_VERSION = importlib.metadata.version("datafog-core") +CORE_LINE = ".".join(CORE_VERSION.split(".")[:2]) +DEVIATIONS = Path(__file__).parent / "contracts" / f"rust-{CORE_LINE}.0.json" +WRAPPER_DEVIATIONS = Path(__file__).parent / "contracts" / "wrapper-4.12.0.json" + TARGETS = { "datafog:scan", "datafog:redact", @@ -20,6 +24,18 @@ } +def reviewed_deviation(case): + """Layer explicit wrapper fixes on the unchanged historical wheel oracle.""" + wrapper = json.loads(WRAPPER_DEVIATIONS.read_text(encoding="utf-8")) + native = json.loads(DEVIATIONS.read_text(encoding="utf-8")) + return native["cases"].get(case["id"]) or wrapper["cases"].get(case["id"]) + + +def expected_observation(case): + deviation = reviewed_deviation(case) + return copy.deepcopy(deviation["expected"] if deviation else case["expected"]) + + def applicable(case): return ( case.get("operation", "call") == "call" @@ -39,15 +55,14 @@ def main(): parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() contract = json.loads(FIXTURE.read_text(encoding="utf-8")) - deviations = json.loads(DEVIATIONS.read_text(encoding="utf-8")) results = [] for case in contract["cases"]: if not applicable(case): results.append({"id": case["id"], "status": "outside-backend-scope"}) continue actual = native_observation(case) - deviation = deviations["cases"].get(case["id"]) - expected = deviation["expected"] if deviation else case["expected"] + deviation = reviewed_deviation(case) + expected = expected_observation(case) status = "regression" if actual != expected else "match" if status == "match" and deviation: status = deviation["classification"] diff --git a/tests/test_contract_481.py b/tests/test_contract_481.py index 43863dfd..b54ecf1e 100644 --- a/tests/test_contract_481.py +++ b/tests/test_contract_481.py @@ -1,64 +1,33 @@ """Frozen observations from the published 4.8.1 wheel; never regenerate in CI.""" -import copy import json import pytest from tests.contract_481 import FIXTURE, observe -from tests.rust_contract import DEVIATIONS +from tests.rust_contract import WRAPPER_DEVIATIONS, expected_observation CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) -# The default backend is native from 4.10.0, so the reviewed native -# differences are the published 4.10 observations for those cases. -REVIEWED = json.loads(DEVIATIONS.read_text(encoding="utf-8")) +REVIEWED_WRAPPER = json.loads(WRAPPER_DEVIATIONS.read_text(encoding="utf-8")) def expected_49(case): - """Allow only the added backend parameter, revised shim notices and the - reviewed native differences.""" - - deviation = REVIEWED["cases"].get(case["id"]) - expected = ( - copy.deepcopy(deviation["expected"]) - if deviation - else copy.deepcopy(case["expected"]) - ) - if case["id"] == "public-signatures": - for target in ( - "datafog:scan", - "datafog:redact", - "datafog.engine:scan", - "datafog.engine:scan_and_redact", - ): - expected["value"][target]["parameters"].append( - { - "name": "backend", - "kind": "KEYWORD_ONLY", - "required": False, - "default": None, - } - ) - if case.get("target") in {"datafog:detect", "datafog:process"}: - name = case["target"].split(":")[1] - replacement = ( - "datafog.scan()" - if name == "detect" - else "datafog.scan() or datafog.redact()" - ) - expected["warnings"] = [ - { - "type": "FutureWarning", - "message": ( - f"datafog.{name}() is deprecated and will be removed in 5.0. " - f"Use {replacement} instead. " - 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0". ' - "The earlier promise to retain this shim through 5.x has been revised. " - "See docs/retirement.rst." - ), - } - ] - return expected + """The immutable oracle plus individually explained native/wrapper fixes.""" + return expected_observation(case) + + +def test_reviewed_wrapper_deviations_are_explicit_and_in_scope(): + cases = {case["id"]: case for case in CONTRACT["cases"]} + assert set(REVIEWED_WRAPPER["cases"]) <= set(cases) + for case_id, deviation in REVIEWED_WRAPPER["cases"].items(): + assert deviation["reason"] + assert deviation["classification"] in { + "behavioral-fix", + "validation-difference", + "diagnostic-difference", + "signature-addition", + } + assert deviation["expected"] != cases[case_id]["expected"] @pytest.mark.parametrize("case", CONTRACT["cases"], ids=lambda case: case["id"]) diff --git a/tests/test_core04_integration.py b/tests/test_core04_integration.py index b0e2ae9f..9f952bac 100644 --- a/tests/test_core04_integration.py +++ b/tests/test_core04_integration.py @@ -1,4 +1,4 @@ -"""Exercise actual Core 0.4 wheels through the Python compatibility adapter. +"""Exercise reviewed native wheels through the Python compatibility adapter. Examples originate in Core's fixtures/{german,jwt,private-key,npi, us-routing-number,uuid}.jsonl. Payloads are synthetic detector fixtures, @@ -6,6 +6,7 @@ """ import importlib.metadata +import json import re import pytest @@ -13,6 +14,7 @@ import datafog from datafog import engine, v5 from datafog.compat import v4 +from tests.rust_contract import DEVIATIONS core = pytest.importorskip("datafog_core") @@ -50,7 +52,8 @@ def test_supported_wheel_version_and_capability_contract(): version = importlib.metadata.version("datafog-core") - assert version.split(".")[:1] == ["1"], version + reviewed = json.loads(DEVIATIONS.read_text())["core_version"] + assert version.split(".")[:2] == reviewed.split(".")[:2], version capabilities = core.capabilities() assert capabilities["contract_version"] == 1 supported = capabilities["supported_entities"] diff --git a/tests/test_core20_integration.py b/tests/test_core20_integration.py new file mode 100644 index 00000000..290f8e0d --- /dev/null +++ b/tests/test_core20_integration.py @@ -0,0 +1,58 @@ +"""Core 2 candidate seam regressions; exercised before the dependency floor moves.""" + +import importlib.metadata + +import pytest + +import datafog + +pytestmark = pytest.mark.skipif( + importlib.metadata.version("datafog-core").split(".")[0] != "2", + reason="qualifies the separately built Core 2 candidate before publication", +) + + +@pytest.mark.parametrize("value", ["::ffff:192.168.1.1", "64:ff9b::192.0.2.33"]) +def test_ipv6_embedded_ipv4_tail_is_fully_redacted(value): + result = datafog.redact(value) + assert result.redacted_text == "[IP_ADDRESS_1]" + assert result.mapping == {"[IP_ADDRESS_1]": value} + + +def test_contextual_email_does_not_swallow_neighboring_phone(): + text = "phone=415-555-2671&email=bob@example.com" + result = datafog.scan(text) + assert [(item.type, item.text) for item in result.entities] == [ + ("PHONE", "415-555-2671"), + ("EMAIL", "bob@example.com"), + ] + assert datafog.redact(text).redacted_text == "phone=[PHONE_1]&email=[EMAIL_1]" + + +def test_phone_followed_by_zip_does_not_become_credit_card(): + text = "555-123-4567 94105" + assert [(item.type, item.text) for item in datafog.scan(text).entities] == [ + ("PHONE", "555-123-4567"), + ("ZIP_CODE", "94105"), + ] + assert datafog.scan(text, entity_types=["PHONE"]).entities[0].text == "555-123-4567" + + +def test_adding_model_selection_does_not_drop_large_document_ssns(): + text = "ssn 123-45-6789 " * 3000 + result = datafog.scan(text, entity_types=["SSN", "PERSON"]) + assert len([item for item in result.entities if item.type == "SSN"]) == 3000 + assert all(text[item.start : item.end] == item.text for item in result.entities) + + +def test_international_email_and_person_punctuation(): + assert datafog.scan("mail jörg@müller.de").entities[0].text == "jörg@müller.de" + result = datafog.scan("Jane Doe,\n14 Penny Lane", entity_types=["PERSON"]) + assert result.entities and all( + item.text == item.text.rstrip(",;:.!?") for item in result.entities + ) + + +def test_german_context_prevents_generic_false_positive_labels(): + assert not datafog.scan("Steuer-ID 12345678901 liegt vor.").entities + assert not datafog.scan("IBAN DE44 5001 0517 5407 3249 31 ist gueltig.").entities diff --git a/tests/test_de_pii_regex.py b/tests/test_de_pii_regex.py index 920c35f3..d36cc6af 100644 --- a/tests/test_de_pii_regex.py +++ b/tests/test_de_pii_regex.py @@ -166,7 +166,10 @@ def test_german_vat_redaction_suppresses_inner_generic_ssn_match( def test_top_level_helpers_and_supported_entities_respect_locale() -> None: default_entities = get_supported_entities() - assert all(not entity.startswith("DE_") for entity in default_entities) + assert {"EMAIL", "DE_VAT_ID", "DE_IBAN", "JWT", "PERSON"} <= set(default_entities) + assert "JWT" not in get_supported_entities(backend="python") + # Supported inventory includes opt-in labels; scanning still requires locale activation. + assert "DE_TAX_ID" in default_entities german_entities = get_supported_entities(locales=["de"]) assert "DE_VAT_ID" in german_entities diff --git a/tests/test_rust_contract.py b/tests/test_rust_contract.py index c3f16bc0..73909bf4 100644 --- a/tests/test_rust_contract.py +++ b/tests/test_rust_contract.py @@ -6,7 +6,12 @@ import pytest from tests.contract_481 import FIXTURE -from tests.rust_contract import DEVIATIONS, applicable, native_observation +from tests.rust_contract import ( + DEVIATIONS, + applicable, + expected_observation, + native_observation, +) pytest.importorskip("datafog_core") CONTRACT = json.loads(FIXTURE.read_text(encoding="utf-8")) @@ -30,6 +35,4 @@ def test_native_version_and_review_inventory(): @pytest.mark.parametrize("case", CASES, ids=lambda case: case["id"]) def test_native_contract(case): - deviation = REVIEWED["cases"].get(case["id"]) - expected = deviation["expected"] if deviation else case["expected"] - assert native_observation(case) == expected, case["id"] + assert native_observation(case) == expected_observation(case), case["id"] diff --git a/tests/test_spacy_span_contract.py b/tests/test_spacy_span_contract.py new file mode 100644 index 00000000..7029b391 --- /dev/null +++ b/tests/test_spacy_span_contract.py @@ -0,0 +1,56 @@ +"""Optional real-spaCy offset integration without downloading model weights.""" + +import pytest + +from datafog import engine +from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator +from datafog.services.text_service import TextService + +spacy = pytest.importorskip("spacy") + + +@pytest.fixture +def real_annotator(monkeypatch): + nlp = spacy.blank("en") + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns( + [ + {"label": "PERSON", "pattern": "Ann"}, + {"label": "ORG", "pattern": "Acme Corp"}, + {"label": "DATE", "pattern": "yesterday"}, + ] + ) + annotator = SpacyPIIAnnotator(nlp=nlp, model_name="local-test-ruler") + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: annotator) + return annotator + + +def test_spacy_ranges_redact_actual_name_without_hitting_prefix(real_annotator): + text = "Annual report by Ann yesterday: 6011111111111117" + result = engine.scan_and_redact(text, engine="spacy") + assert ( + result.redacted_text == "Annual report by [PERSON_1] yesterday: [CREDIT_CARD_1]" + ) + assert all(text[item.start : item.end] == item.text for item in result.entities) + + +def test_spacy_service_combines_ner_and_native_detection(real_annotator): + text = "Ann at Acme Corp: 6011111111111117 ip 2001:db8::1" + service = TextService(engine="spacy") + result = service.annotate_text_sync(text) + assert result == { + "PERSON": ["Ann"], + "ORGANIZATION": ["Acme Corp"], + "CREDIT_CARD": ["6011111111111117"], + "IP_ADDRESS": ["2001:db8::1"], + } + assert all( + text[item.start : item.end] == item.text + for item in service.annotate_text_sync(text, structured=True) + ) + + +def test_spacy_repeated_names_keep_each_original_span(real_annotator): + text = "Ann wrote an Annual report for Ann." + result = engine.scan(text, engine="spacy") + assert [(item.start, item.end) for item in result.entities] == [(0, 3), (31, 34)] diff --git a/tests/test_text_service.py b/tests/test_text_service.py index 74156082..3d5973ad 100644 --- a/tests/test_text_service.py +++ b/tests/test_text_service.py @@ -1,366 +1,124 @@ -from unittest.mock import Mock, patch - -import pytest - -# Legacy coverage for the historical shadow TextService. New behavior belongs -# in datafog.services.text_service. -from datafog.services.text_service_original import TextService +"""Public TextService behavior using the shipped service and shared engine.""" +import asyncio +import threading +from types import SimpleNamespace -@pytest.fixture -def mock_annotator(): - mock = Mock() - mock.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - return mock - +import pytest -@pytest.fixture -def mock_regex_annotator(): - mock = Mock() - mock.annotate.return_value = { - "EMAIL": ["john@example.com"], - "PHONE": ["555-555-5555"], - } +from datafog import engine +from datafog.services.text_service import TextService - # Add mock for annotate_with_spans method - from datafog.processing.text_processing.regex_annotator import ( - AnnotationResult, - Span, - ) - spans = [ - Span(label="EMAIL", start=0, end=15, text="john@example.com"), - Span(label="PHONE", start=20, end=32, text="555-555-5555"), - ] - mock.annotate_with_spans.return_value = ( - {"EMAIL": ["john@example.com"], "PHONE": ["555-555-5555"]}, - AnnotationResult(text="test", spans=spans), - ) - return mock +@pytest.mark.parametrize( + "options", + [{"engine": "invalid"}, {"text_chunk_length": 0}, {"text_chunk_length": -1}], +) +def test_invalid_configuration_is_explicit(options): + with pytest.raises(ValueError): + TextService(**options) -@pytest.fixture -def text_service(mock_annotator, mock_regex_annotator): - # Configure regex annotator to return empty results so auto mode falls back to spaCy - # This ensures backward compatibility with existing tests while using 'auto' mode - mock_regex_annotator.annotate.return_value = { - key: [] - for key in ["EMAIL", "PHONE", "SSN", "CREDIT_CARD", "IP_ADDRESS", "DOB", "ZIP"] +def test_default_service_detects_native_only_entities(): + result = TextService().annotate_text_sync("card 6011111111111117 ip 2001:db8::1") + assert result == { + "CREDIT_CARD": ["6011111111111117"], + "IP_ADDRESS": ["2001:db8::1"], } - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - return_value=mock_annotator, - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - return_value=mock_regex_annotator, - ): - # Use 'auto' engine to match production default, but regex will find nothing - # so it will always fall back to spaCy, maintaining test compatibility - return TextService(text_chunk_length=10, engine="auto") - - -@pytest.fixture -def text_service_with_engine(mock_annotator, mock_regex_annotator): - def _create_service(engine="auto"): - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - return_value=mock_annotator, - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - return_value=mock_regex_annotator, - ): - return TextService(text_chunk_length=10, engine=engine) - return _create_service +def test_long_document_seams_preserve_complete_entities(): + value = "john@example.com" + text = "x" * 995 + " " + value + " " + "y" * 1010 + " ssn 123-45-6789" + result = TextService(text_chunk_length=1000).annotate_text_sync(text) + assert result == {"EMAIL": [value], "SSN": ["123-45-6789"]} -def test_init(text_service): - assert text_service.text_chunk_length == 10 - - -def test_init_with_default_engine(text_service): - assert text_service.text_chunk_length == 10 - # We're using 'auto' in our fixture to match production default - assert text_service.engine == "auto" - - -def test_init_with_custom_engine(text_service_with_engine): - service = text_service_with_engine(engine="regex") - assert service.engine == "regex" - - service = text_service_with_engine(engine="spacy") - assert service.engine == "spacy" - - service = text_service_with_engine(engine="auto") - assert service.engine == "auto" - - -def test_init_with_invalid_engine(): - with pytest.raises(AssertionError, match="Invalid engine"): - with patch( - "datafog.services.text_service_original.SpacyPIIAnnotator.create", - ): - with patch( - "datafog.services.text_service_original.RegexAnnotator", - ): - TextService(engine="invalid") - - -def test_chunk_text(text_service): - text = "This is a test sentence for chunking." - chunks = text_service._chunk_text(text) - assert len(chunks) == 4 - assert chunks == ["This is a ", "test sente", "nce for ch", "unking."] +def test_structured_output_preserves_unicode_source_ranges(): + text = "😀 café john@example.com and john@example.com" + result = TextService(text_chunk_length=10).annotate_text_sync(text, structured=True) + assert len(result) == 2 + assert all( + item.label == "EMAIL" and text[item.start : item.end] == item.text + for item in result + ) + assert result[0].start < result[1].start -def test_combine_annotations(text_service): - annotations = [ - {"PER": ["John"], "ORG": ["Acme"]}, - {"PER": ["Doe"], "LOC": ["New York"]}, +def test_batch_results_preserve_input_order_and_duplicate_documents(): + texts = ["a@example.com", "no pii", "a@example.com"] + assert TextService().batch_annotate_text_sync(texts) == [ + {"EMAIL": ["a@example.com"]}, + {}, + {"EMAIL": ["a@example.com"]}, ] - combined = text_service._combine_annotations(annotations) - assert combined == {"PER": ["John", "Doe"], "ORG": ["Acme"], "LOC": ["New York"]} - - -def test_annotate_text_sync(text_service): - result = text_service.annotate_text_sync("John Doe works at Acme Inc") - assert result == { - "PER": ["John Doe", "John Doe", "John Doe"], - "ORG": ["Acme Inc", "Acme Inc", "Acme Inc"], - } - - -def test_batch_annotate_text_sync(text_service): - texts = ["John Doe", "Acme Inc"] - result = text_service.batch_annotate_text_sync(texts) - assert result == { - "John Doe": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - "Acme Inc": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - } @pytest.mark.asyncio -async def test_annotate_text_async(text_service): - result = await text_service.annotate_text_async("John Doe works at Acme Inc") - assert result == { - "PER": ["John Doe", "John Doe", "John Doe"], - "ORG": ["Acme Inc", "Acme Inc", "Acme Inc"], - } +async def test_async_results_match_synchronous_service(): + service = TextService() + texts = ["a@example.com", "ssn 123-45-6789", "no pii"] + assert await service.batch_annotate_text_async( + texts + ) == service.batch_annotate_text_sync(texts) + assert await service.annotate_text_async( + texts[0], structured=True + ) == service.annotate_text_sync(texts[0], structured=True) @pytest.mark.asyncio -async def test_batch_annotate_text_async(text_service): - texts = ["John Doe", "Acme Inc"] - result = await text_service.batch_annotate_text_async(texts) - assert result == { - "John Doe": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - "Acme Inc": {"PER": ["John Doe"], "ORG": ["Acme Inc"]}, - } - - -def test_long_text_chunking(text_service): - long_text = "John Doe works at Acme Inc. Jane Smith is from New York City." - result = text_service.annotate_text_sync(long_text) - expected_count = len(text_service._chunk_text(long_text)) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -@pytest.mark.asyncio -async def test_long_text_chunking_async(text_service): - long_text = "John Doe works at Acme Inc. Jane Smith is from New York City." - result = await text_service.annotate_text_async(long_text) - expected_count = len(text_service._chunk_text(long_text)) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -def test_empty_string(text_service): - result = text_service.annotate_text_sync("") - assert result == {} - - -def test_short_string(text_service): - result = text_service.annotate_text_sync("Short") - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_special_characters(text_service): - result = text_service.annotate_text_sync("John@Doe.com works at Acme-Inc!!!") - expected_count = len(text_service._chunk_text("John@Doe.com works at Acme-Inc!!!")) - assert result == { - "PER": ["John Doe"] * expected_count, - "ORG": ["Acme Inc"] * expected_count, - } - - -def test_regex_engine(text_service_with_engine, mock_regex_annotator): - service = text_service_with_engine(engine="regex") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("john@example.com") - - # Should only call the regex annotator - assert mock_regex_annotator.annotate.called - assert not service.spacy_annotator.annotate.called - assert result == {"EMAIL": ["john@example.com"], "PHONE": ["555-555-5555"]} - - -def test_spacy_engine(text_service_with_engine, mock_annotator): - service = text_service_with_engine(engine="spacy") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("John Doe works at Acme Inc") - - # Should only call the spaCy annotator - assert mock_annotator.annotate.called - assert not service.regex_annotator.annotate.called - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_auto_engine_with_regex_results( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - # Configure regex annotator to return results - mock_regex_annotator.annotate.return_value = {"EMAIL": ["john@example.com"]} - - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("john@example.com") - - # Should call regex annotator but not spaCy - assert mock_regex_annotator.annotate.called - assert not mock_annotator.annotate.called - - assert result == {"EMAIL": ["john@example.com"]} - - -def test_auto_engine_with_fallback( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - # Configure regex annotator to return empty results - mock_regex_annotator.annotate.return_value = {"EMAIL": [], "PHONE": []} - - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync("John Doe works at Acme Inc") - - # Should call both annotators - assert mock_regex_annotator.annotate.called - assert mock_annotator.annotate.called - - assert result == {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - - -def test_structured_output_regex_engine(text_service_with_engine, mock_regex_annotator): - """Test structured output mode with regex engine.""" - # Set up the mock to return spans that match the input text - from datafog.processing.text_processing.regex_annotator import ( - AnnotationResult, - Span, - ) - - # Create spans that will be returned by the mock - test_text = "john@example.com" - spans = [ - # Make sure the end position matches the actual length of the text - Span(label="EMAIL", start=0, end=len(test_text), text=test_text), - ] - - # Update the mock to return spans that match the input text - mock_regex_annotator.annotate_with_spans.return_value = ( - {"EMAIL": [test_text]}, - AnnotationResult(text=test_text, spans=spans), +async def test_async_detection_runs_outside_event_loop_thread(monkeypatch): + loop_thread = threading.get_ident() + worker_threads = [] + + def detect(text, **_): + worker_threads.append(threading.get_ident()) + return SimpleNamespace(entities=[]) + + monkeypatch.setattr(engine, "scan", detect) + await asyncio.wait_for(TextService().annotate_text_async("text"), timeout=2) + assert worker_threads and all(thread != loop_thread for thread in worker_threads) + + +@pytest.mark.parametrize("mode", ["auto", "smart"]) +def test_optional_ner_adds_to_structured_findings(monkeypatch, mode): + text = "Ann owns 6011111111111117" + monkeypatch.setattr( + engine, + "_gliner_entities", + lambda _: [engine.Entity("PERSON", "Ann", 0, 3, 0.8, "gliner")], ) + result = TextService(engine=mode).annotate_text_sync(text) + assert result == {"PERSON": ["Ann"], "CREDIT_CARD": ["6011111111111117"]} - service = text_service_with_engine(engine="regex") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - result = service.annotate_text_sync(test_text, structured=True) - - # Should call regex annotator's annotate_with_spans method - assert mock_regex_annotator.annotate_with_spans.called - - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 1 # Only one span should be returned (EMAIL) - - # Verify the span has the correct properties - assert result[0].label == "EMAIL" - assert result[0].text == test_text - assert result[0].start == 0 - assert result[0].end == len(test_text) - - -def test_structured_output_spacy_engine(text_service_with_engine, mock_annotator): - """Test structured output mode with spaCy engine.""" - service = text_service_with_engine(engine="spacy") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - # Set up mock to return entities that can be found in the test text - test_text = "John Doe works at Acme Inc" - mock_annotator.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} +def test_custom_gliner_model_is_forwarded(monkeypatch): + models = [] - result = service.annotate_text_sync(test_text, structured=True) + def detect(_, model_name): + models.append(model_name) + return [] - # Should call spaCy annotator - assert mock_annotator.annotate.called - - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 2 - - # Check that spans were created correctly - per_spans = [span for span in result if span.label == "PER"] - org_spans = [span for span in result if span.label == "ORG"] - - assert len(per_spans) == 1 - assert per_spans[0].text == "John Doe" - assert per_spans[0].start == test_text.find("John Doe") - assert per_spans[0].end == test_text.find("John Doe") + len("John Doe") - - assert len(org_spans) == 1 - assert org_spans[0].text == "Acme Inc" - assert org_spans[0].start == test_text.find("Acme Inc") - assert org_spans[0].end == test_text.find("Acme Inc") + len("Acme Inc") - - -def test_structured_output_auto_engine( - text_service_with_engine, mock_regex_annotator, mock_annotator -): - """Test structured output mode with auto engine.""" - # Configure regex annotator to return empty spans - from datafog.processing.text_processing.regex_annotator import AnnotationResult - - mock_regex_annotator.annotate_with_spans.return_value = ( - {"EMAIL": [], "PHONE": []}, - AnnotationResult(text="test", spans=[]), + monkeypatch.setattr(engine, "_gliner_entities", detect) + assert ( + TextService(engine="smart", gliner_model="local/custom").annotate_text_sync( + "no pii" + ) + == {} ) + assert models == ["local/custom"] - service = text_service_with_engine(engine="auto") - # Override chunk length to avoid multiple calls - service.text_chunk_length = 1000 - - # Set up mock to return entities that can be found in the test text - test_text = "John Doe works at Acme Inc" - mock_annotator.annotate.return_value = {"PER": ["John Doe"], "ORG": ["Acme Inc"]} - result = service.annotate_text_sync(test_text, structured=True) +def test_locale_activates_additional_shared_detectors(): + text = "Aufenthaltstitel AT1234567 gueltig." + assert TextService(locales=["de"]).annotate_text_sync(text) == { + "DE_RESIDENCE_PERMIT_NUMBER": ["AT1234567"] + } - # Should call both annotators - assert mock_regex_annotator.annotate_with_spans.called - assert mock_annotator.annotate.called - # Verify the result is a list of Span objects - assert isinstance(result, list) - assert len(result) == 2 +def test_empty_and_invalid_text(): + service = TextService() + assert service.annotate_text_sync("") == {} + assert service.annotate_text_sync("", structured=True) == [] + with pytest.raises(TypeError, match="text must be a string"): + service.annotate_text_sync(None) diff --git a/tests/test_text_service_integration.py b/tests/test_text_service_integration.py index d3363f5f..538f63c8 100644 --- a/tests/test_text_service_integration.py +++ b/tests/test_text_service_integration.py @@ -5,6 +5,15 @@ from datafog.services.text_service import TextService +@pytest.fixture +def require_spacy_model(): + spacy = pytest.importorskip("spacy") + try: + spacy.load("en_core_web_lg") + except OSError: + pytest.skip("en_core_web_lg must be provisioned for trained-model integration") + + @pytest.fixture def real_text_service(): """Create a real TextService instance for integration testing.""" @@ -32,7 +41,7 @@ def test_engine_regex_detects_simple_entities(): @pytest.mark.integration -def test_engine_auto_fallbacks_to_spacy(): +def test_engine_auto_fallbacks_to_spacy(require_spacy_model): """Test that auto mode works correctly with entity detection.""" # We need to test the auto mode in a more controlled way # Create a text that contains only named entities (no emails, phones, etc.) @@ -45,7 +54,7 @@ def test_engine_auto_fallbacks_to_spacy(): # Verify spaCy finds named entities assert "PERSON" in spacy_result and spacy_result["PERSON"] - assert "ORG" in spacy_result and spacy_result["ORG"] + assert "ORGANIZATION" in spacy_result and spacy_result["ORGANIZATION"] # Now create a special text that contains both regex-detectable and spaCy-detectable entities mixed_text = "John Smith's email is john.smith@example.com" @@ -61,7 +70,7 @@ def test_engine_auto_fallbacks_to_spacy(): @pytest.mark.integration -def test_engine_spacy_only(): +def test_engine_spacy_adds_to_structured_pii(require_spacy_model): """Test that spaCy engine is always used regardless of regex potential hits.""" # Sample text with both regex-detectable and spaCy-detectable entities text = """John Smith's email is john.smith@example.com. @@ -79,11 +88,10 @@ def test_engine_spacy_only(): # Verify spaCy detected named entities assert "PERSON" in spacy_result and spacy_result["PERSON"] - assert "ORG" in spacy_result and spacy_result["ORG"] + assert "ORGANIZATION" in spacy_result and spacy_result["ORGANIZATION"] - # Verify spaCy did NOT detect the email (which confirms it's using spaCy only) - # This is because spaCy doesn't have a built-in EMAIL entity type - assert "EMAIL" not in spacy_result or not spacy_result["EMAIL"] + # NER augments the shared structured detector. + assert "john.smith@example.com" in spacy_result["EMAIL"] @pytest.mark.integration @@ -130,7 +138,7 @@ def test_structured_annotation_output(): @pytest.mark.integration -def test_debug_entity_types(): +def test_spacy_entity_labels_are_canonical(require_spacy_model): """Debug test to print the actual entity types returned by spaCy.""" # Sample text with named entities text = """John Smith works at Microsoft Corporation in Seattle. @@ -146,8 +154,8 @@ def test_debug_entity_types(): if values: # Only print non-empty lists print(f" {entity_type}: {values}") - # No assertion needed, this is just for debugging - assert True + assert "ORGANIZATION" in spacy_result + assert "ORG" not in spacy_result @pytest.mark.skip(reason="Performance benchmarking requires more setup") From c486a529202ee9c6613105fad366991e2fb618f9 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:01:06 -0800 Subject: [PATCH 05/20] fix(engine): validate entity support for each optional NER engine --- datafog/engine.py | 10 +++++++--- datafog/services/text_service.py | 13 +++++++------ tests/test_review_engine.py | 5 +++++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/datafog/engine.py b/datafog/engine.py index 8ce61a5f..884bb1de 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -85,6 +85,8 @@ _core_warned = False _cnn_cache: tuple[object, object] | None = None +SPACY_ENTITY_TYPES = {"PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"} + NER_ENTITY_TYPES = { "PERSON", "ORGANIZATION", @@ -455,7 +457,7 @@ def _spacy_entities(text: str) -> list[Entity]: return [ entity for entity in _annotator_entities(text, annotator, "spacy", 0.7) - if entity.type in NER_ENTITY_TYPES + if entity.type in SPACY_ENTITY_TYPES ] @@ -648,7 +650,9 @@ def _validate_entity_types( supported = supported | MODEL_ENTITY_TYPES else: supported = {_canonical_type(label) for label in RegexAnnotator.LABELS} - if engine != "regex": + if engine == "spacy": + supported |= SPACY_ENTITY_TYPES + elif engine in {"smart", "gliner"}: supported |= NER_ENTITY_TYPES | {"DATE"} unsupported = set(requested) - supported if unsupported: @@ -676,7 +680,7 @@ def scan( ``DATAFOG_BACKEND`` or pass ``backend="python"`` to force the fallback. The selected backend supplies structured PII for every engine mode; optional NER engines add entity spans. - Probe names and address blocks come from the embedded model on the native + Prose names and address blocks come from the embedded model on the native path, so ``PERSON`` and ``ADDRESS`` work without spaCy or a model download. Results retain legacy regex provenance and confidence; these are diff --git a/datafog/services/text_service.py b/datafog/services/text_service.py index eee7ca90..a7826a7e 100644 --- a/datafog/services/text_service.py +++ b/datafog/services/text_service.py @@ -1,8 +1,8 @@ """Lean text processing service for PII annotation. -Provides synchronous and asynchronous methods for annotating text with personally -identifiable information (PII) using regex patterns. Supports chunking long texts -and batch processing. SpaCy integration available as optional extra. +Provides synchronous, asynchronous and batch access to the shared PII engine. +Structured PII spans are scanned across complete documents; optional NLP engines +add spans with their original character coordinates. """ import asyncio @@ -31,8 +31,8 @@ class TextService: Lightweight service for annotating text with PII entities using regex patterns. This service provides methods to detect and annotate personally identifiable information (PII) - in text using fast regex patterns. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. + in text through the shared engine. Complete-document structured scanning preserves + entities across boundaries, and optional NLP engines add canonical entity spans. For advanced NLP-based detection using spaCy, install the 'nlp' extra: pip install datafog[nlp] @@ -49,7 +49,8 @@ def __init__( Initialize the TextService with specified chunk length and annotation engine. Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. + text_chunk_length: Positive compatibility setting. Structured detection scans the + complete document; the shared engine bounds native model chunks separately. engine: The annotation engine to use. Options are: - "regex": (Default) Use RegexAnnotator for fast pattern-based entity detection - "spacy": Use SpacyPIIAnnotator for NLP-based entity detection (requires nlp extra) diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py index 020f0935..9f47318a 100644 --- a/tests/test_review_engine.py +++ b/tests/test_review_engine.py @@ -324,3 +324,8 @@ def test_fallback_email_context_is_trimmed_without_losing_mailboxes(text): "dev@example.com", "bob@example.com", } + + +def test_spacy_selection_rejects_gliner_only_labels(): + with pytest.raises(ValueError, match="PASSPORT_NUMBER"): + engine.scan("passport SECRET", engine="spacy", entity_types=["PASSPORT_NUMBER"]) From 0102e136b34dbb13ee12463b74485f1e74b889f1 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:02:26 -0800 Subject: [PATCH 06/20] fix: complete review entrypoint, packaging and release safeguards --- .bumpversion.cfg | 2 +- .github/workflows/release.yml | 133 ++++---- .gitignore | 1 + .pre-commit-config.yaml | 4 +- CHANGELOG.MD | 33 ++ MANIFEST.in | 5 + README.md | 46 +-- datafog/__about__.py | 2 +- datafog/__init__.py | 72 +++-- datafog/__init___lean.py | 196 ------------ datafog/__init___original.py | 63 ---- datafog/_legacy_retirement.py | 4 +- datafog/cli.py | 17 + datafog/client.py | 24 +- datafog/compat/v4.py | 12 +- datafog/core.py | 26 +- datafog/main.py | 38 ++- datafog/main_lean.py | 196 ------------ datafog/main_original.py | 263 --------------- datafog/models/annotator.py | 37 ++- datafog/models/anonymizer.py | 70 ++-- datafog/models/common.py | 4 +- datafog/native.py | 10 +- .../spark_processing/pyspark_udfs.py | 16 +- datafog/services/text_service_lean.py | 197 ------------ datafog/services/text_service_original.py | 299 ------------------ docs/migration-4.10.md | 4 +- docs/retirement.rst | 18 +- docs/review-remediation.md | 73 +++++ pyproject.toml | 3 + scripts/check_rust_install.py | 17 +- scripts/release_version.py | 68 ++++ setup.py | 20 +- setup_lean.py | 109 ------- setup_original.py | 92 ------ tests/contracts/wrapper-4.12.0.json | 38 +-- tests/test_client.py | 2 +- tests/test_donut_lazy_import.py | 12 +- tests/test_image_service.py | 5 +- tests/test_legacy_retirement.py | 8 +- tests/test_litellm_guardrail.py | 5 +- tests/test_no_network_core.py | 18 +- tests/test_review_entrypoints.py | 229 ++++++++++++++ tests/test_runtime_dependency_safety.py | 6 +- tox.ini | 2 +- 45 files changed, 803 insertions(+), 1696 deletions(-) create mode 100644 MANIFEST.in delete mode 100644 datafog/__init___lean.py delete mode 100644 datafog/__init___original.py create mode 100644 datafog/cli.py delete mode 100644 datafog/main_lean.py delete mode 100644 datafog/main_original.py delete mode 100644 datafog/services/text_service_lean.py delete mode 100644 datafog/services/text_service_original.py create mode 100644 docs/review-remediation.md create mode 100644 pyproject.toml create mode 100644 scripts/release_version.py delete mode 100644 setup_lean.py delete mode 100644 setup_original.py create mode 100644 tests/test_review_entrypoints.py diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e72c9e2a..46abe29a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.9.0 +current_version = 4.12.0 commit = True tag = True tag_name = v{new_version} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d93c6bd8..32952a23 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,9 +2,9 @@ name: Release on: schedule: - # Monday-Wednesday at 2 AM UTC: alpha builds from dev + # Scheduled runs validate candidates without publishing. - cron: "0 2 * * 1-3" - # Thursday at 2 AM UTC: beta builds from dev + # Thursday beta qualification also stops before publication. - cron: "0 2 * * 4" workflow_dispatch: inputs: @@ -19,7 +19,7 @@ on: dry_run: description: "Dry run (skip PyPI publish)" required: false - default: false + default: true type: boolean dry_run_ref: description: "Candidate ref to validate (allowed only with dry_run)" @@ -86,6 +86,15 @@ jobs: fetch-depth: 0 ref: ${{ steps.resolve.outputs.checkout_ref }} + - name: Reject a development branch missing stable changes + if: steps.resolve.outputs.release_type != 'stable' + run: | + git fetch origin main + git merge-base --is-ancestor origin/main HEAD || { + echo "Development source is missing stable changes; merge main into dev first" + exit 1 + } + - name: Freeze source commit id: source run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" @@ -125,7 +134,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v6 with: @@ -272,7 +281,7 @@ jobs: pip install build twine bump2version - name: Configure git - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true run: | git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]" @@ -282,57 +291,20 @@ jobs: env: VERSION_OVERRIDE: ${{ inputs.version_override }} run: | - set -e + set -euo pipefail git fetch --tags - TYPE="${{ needs.determine-release.outputs.release_type }}" - CURRENT=$(sed -n 's/^__version__ = "\([^"]*\)"/\1/p' datafog/__about__.py) - if [ -z "$CURRENT" ]; then - echo "Failed to parse current version from datafog/__about__.py" - exit 1 - fi - echo "Current version: $CURRENT" - - # Strip any pre-release suffix to get base version - BASE=$(echo "$CURRENT" | sed -E 's/(a|b)[0-9]+([.][0-9A-Za-z]+)?$//') - if [ -n "$VERSION_OVERRIDE" ]; then - BASE="$VERSION_OVERRIDE" - if echo "$BASE" | grep -Eq '(a|b)[0-9]+([.][0-9A-Za-z]+)?$'; then - echo "version_override must be a stable base version like 4.4.0, not a prerelease" - exit 1 - fi - fi - if ! echo "$BASE" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then - echo "Release base must have numeric major.minor.patch format" - exit 1 - fi - echo "Base version: $BASE" - - if [ "$TYPE" = "alpha" ]; then - ALPHA_NUM=1 - while git tag -l "v${BASE}a${ALPHA_NUM}" | grep -q .; do - ALPHA_NUM=$((ALPHA_NUM + 1)) - done - VERSION="${BASE}a${ALPHA_NUM}" - - elif [ "$TYPE" = "beta" ]; then - BETA_NUM=1 - while git tag -l "v${BASE}b${BETA_NUM}" | grep -q .; do - BETA_NUM=$((BETA_NUM + 1)) - done - VERSION="${BASE}b${BETA_NUM}" - - else - VERSION="$BASE" - fi - + VERSION=$(python scripts/release_version.py --type "$TYPE" --override "$VERSION_OVERRIDE") echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "Publishing version: $VERSION" - - sed -i "s/__version__ = \".*\"/__version__ = \"$VERSION\"/" datafog/__about__.py - if grep -q 'version="' setup.py 2>/dev/null; then - sed -i "s/version=\".*\"/version=\"$VERSION\"/" setup.py - fi + RELEASE_VERSION="$VERSION" python - <<'PYCODE' + import os, re + from pathlib import Path + version = os.environ["RELEASE_VERSION"] + about = Path("datafog/__about__.py") + about.write_text(re.sub(r'^__version__ = ".*"', f'__version__ = "{version}"', about.read_text(), flags=re.M)) + config = Path(".bumpversion.cfg") + config.write_text(re.sub(r'^current_version = .*', f'current_version = {version}', config.read_text(), flags=re.M)) + PYCODE - name: Generate changelog env: @@ -371,7 +343,7 @@ jobs: if-no-files-found: error - name: Verify release branch still matches tested source - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: RELEASE_BRANCH: ${{ needs.determine-release.outputs.target_branch }} TESTED_SHA: ${{ needs.determine-release.outputs.source_sha }} @@ -382,30 +354,35 @@ jobs: exit 1 fi + - name: Commit and tag the validated release before publication + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true + env: + VERSION: ${{ steps.version.outputs.version }} + BRANCH: ${{ needs.determine-release.outputs.target_branch }} + run: | + set -euo pipefail + git add datafog/__about__.py .bumpversion.cfg + if ! git diff --cached --quiet; then + git commit -m "chore: bump version to $VERSION [skip ci]" + fi + git tag -a "v$VERSION" -m "Release $VERSION" + git push --atomic origin "HEAD:$BRANCH" "v$VERSION" + - name: Publish to PyPI - if: inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | python -m twine upload dist/* --verbose - - name: Commit version bump & create release - if: inputs.dry_run != true + - name: Create GitHub release + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true env: GITHUB_TOKEN: ${{ secrets.GH_PAT }} + VERSION: ${{ steps.version.outputs.version }} + TYPE: ${{ needs.determine-release.outputs.release_type }} run: | - VERSION="${{ steps.version.outputs.version }}" - TYPE="${{ needs.determine-release.outputs.release_type }}" - BRANCH="${{ needs.determine-release.outputs.target_branch }}" - - git add datafog/__about__.py setup.py - git commit -m "chore: bump version to $VERSION [skip ci]" || echo "No version changes to commit" - git push origin "HEAD:$BRANCH" - - git tag -a "v$VERSION" -m "Release $VERSION" - git push origin "v$VERSION" - PRERELEASE_FLAG="" TITLE="" if [ "$TYPE" = "alpha" ]; then @@ -426,7 +403,7 @@ jobs: dist/* - name: Dry run summary - if: inputs.dry_run == true + if: github.event_name != 'workflow_dispatch' || inputs.dry_run == true run: | echo "DRY RUN COMPLETE" echo "Would have published: ${{ steps.version.outputs.version }}" @@ -434,9 +411,25 @@ jobs: ls -la dist/ # ── 4. Cleanup old pre-releases ─────────────────────────────────────── + sync-development: + needs: [determine-release, publish] + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.determine-release.outputs.release_type == 'stable' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + ref: dev + fetch-depth: 0 + token: ${{ secrets.GH_PAT }} + - name: Bring stable release metadata back to development + run: | + git fetch origin main + git merge --ff-only origin/main + git push origin HEAD:dev + cleanup: needs: [determine-release, publish] - if: needs.determine-release.outputs.release_type != 'stable' && inputs.dry_run != true + if: github.event_name == 'workflow_dispatch' && inputs.dry_run != true && needs.determine-release.outputs.release_type != 'stable' runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/.gitignore b/.gitignore index 46214511..a594afa1 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ docs/* !docs/migration-4.8.1-contract.md !docs/migration-4.9.md !docs/core-0.4-integration.md +!docs/review-remediation.md !docs/agents/ !docs/agents/** !docs/audit/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4063257e..f6b76791 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort args: [--profile=black] exclude: .venv - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 26.3.1 hooks: - id: black language_version: python3 diff --git a/CHANGELOG.MD b/CHANGELOG.MD index ddc47dd9..6bf46b28 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,5 +1,38 @@ # ChangeLog +## [Unreleased] + +#### Security + +- Preserve native structured PII detection in smart, spaCy, GLiNER, legacy and + CLI paths. Normalize lone surrogates without moving source spans; retain + structured findings on bounded model inputs. Require review when hook payloads + exceed scan budgets, rather than silently allowing unscanned tails. +- Bound user allowlist matching with timeouts and remove quadratic overlap and + Python fallback regex behavior. Redact exact NER offsets and whole TextService + inputs so repeated names and chunk boundaries cannot expose PII. +- Cover LiteLLM tool arguments, prompt/input requests and complete streamed + responses, with request-scoped tokens. Hide configured API keys in CLI output. + +#### Changed + +- Hash APIs honor the selected algorithm and emit full digests. Optional + `hash_key` enables HMAC with a key of at least 32 bytes. Plain hashes provide + deterministic pseudonyms and do not resist guessing of low-entropy PII. +- Reuse tokens for repeated values, avoid existing token collisions, reject + invalid explicit ranges and unsupported entity selections, and report backend + entity capabilities. Async decorators await results before filtering. +- Keep Core within the reviewed minor series, raise the Pydantic floor to 2.1, + include spaCy in `nlp-advanced`, and allow Torch 2.x versions compatible with + `datafog-core[gliner2]`. Add an isolated build declaration and runnable sdist + tests; remove unused duplicate implementation modules. +- Repair Pydantic optional defaults, legacy anonymizer enum filters and overlaps, + Spark label aliases, implicit pipeline scanning and invalid CLI operations. +- Scheduled release jobs qualify candidates without publication. Manual runs + default to dry runs, reject stale version bases and development history, gate + Python 3.13, record release refs before upload, and synchronize stable metadata + back to development after publication. + ## [4.11.0] #### Changed diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..75a7cea8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include LICENSE README.md pyproject.toml tox.ini requirements-test.txt +recursive-include tests *.py *.json *.txt +recursive-include scripts *.py +prune tests/.datafog_env +global-exclude __pycache__ *.py[cod] diff --git a/README.md b/README.md index 172c0936..375cbc53 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DataFog Python +# DataFog Python DataFog is a Python library for detecting and redacting personally identifiable information (PII). @@ -85,8 +85,9 @@ multilingual tokenizer; installing the OCR extra is not required for GLiNER. `backend` defaults to `auto`: the native engine when `datafog-core` is importable, the pure-Python regex engine otherwise. `DATAFOG_BACKEND` sets the default and `backend="python"` / `backend="rust"` override it per call. -`auto` never selects the native engine for `engine="spacy"`, `"gliner"` or -`"smart"`, which stay on the Python ML path. +Every engine uses this backend for structured PII. `engine="spacy"`, +`"gliner"` and `"smart"` add model spans to the same structured findings; +selecting an ML engine does not disable native card or credential detection. The native engine adds detectors the Python engine never had — `API_KEY`, `BEARER_TOKEN`, `JWT`, `PRIVATE_KEY`, `CREDENTIAL_URI`, `ACCOUNT_NUMBER`, @@ -95,7 +96,7 @@ and `ADDRESS` from an embedded model, so name and address detection needs no spaCy and no model download. Findings keep `engine="regex"` provenance and `confidence=1.0` as compatibility values, not native confidence estimates. Reviewed differences from the Python engine are listed in -[tests/contracts/rust-1.3.0.json](tests/contracts/rust-1.3.0.json). +[tests/contracts/rust-1.4.0.json](tests/contracts/rust-1.4.0.json). `datafog.native` exposes the native schema directly. Python 3.13 support is certified for the core SDK, CLI, `nlp`, @@ -151,10 +152,8 @@ import datafog text = "Steuer-ID 12345678901 liegt vor." -print(datafog.scan(text, engine="regex").entities) -# [] - -print(datafog.scan(text, engine="regex", locales=["de"]).entities) +print(datafog.scan(text, engine="regex", locales=["de"], + entity_types=["DE_TAX_ID"]).entities) # [Entity(type='DE_TAX_ID', text='12345678901', ...)] ``` @@ -215,7 +214,8 @@ scan/redact helpers, or guardrail helpers. DataFog deprecates OCR and Spark with visible use-time warnings; their APIs and extras will be removed in 5.0. The last supported -version is 4.10.0 — pin it with `pip install "datafog==4.10.0"` — and every +version in this release train is 4.12.0 — after publication, pin it with +`pip install "datafog==4.12.0"` — and every retired surface is tracked in the [retirement schedule](docs/retirement.rst). See the [4.9 migration guide](docs/migration-4.9.md) for the transition plan, and [VERSIONING.md](VERSIONING.md) for what a version number promises. @@ -245,6 +245,8 @@ print(result["PHONE"]) ## CLI +Install the command dependencies with `pip install "datafog[cli]"`. + ```bash # Scan text datafog scan-text "john@example.com" @@ -284,20 +286,18 @@ Telemetry does not include input text or detected PII values. ## Development -The [4.9 migration guide](docs/migration-4.9.md) explains opt-in Rust detection, -the native `datafog.v5` preview, and the revised 5.0 retirement schedule for -`detect`/`process`, OCR, and Spark. The Python detector remains the default. - -4.10.0 requires Core `>=1.3.0,<2` and capability contract 1, and installs it by -default. See the [4.10.0 release notes](RELEASE_NOTES_4.10.0.md) and the -[4.10 migration guide](docs/migration-4.10.md). -Entity labels, locales, and activation settings come from the installed -Core, allowing compatible releases to add detectors without a Python update. -German detection is opt-in through locale or entity selection; UUID is opt-in -through `entity_types=["UUID"]`. Core's structured-only `PERSON` is unavailable -for explicit Rust text selection. The legacy overlap policy can suppress NPI in -favor of PHONE; the migration guide explains native alternatives. Lock the Core -version if detection output must remain reproducible. +The native engine is the default, with a warned Python fallback. The 4.12 +review fixes preserve structured detection across every engine and legacy API. +The dependency stays within the reviewed Core 1.4 minor series (`~=1.4.0`); +Core 2.0 is a separate release candidate that must be published and qualified +before this SDK raises its floor. See [release preparation](docs/review-remediation.md). + +Entity labels, locales and activation settings come from the installed Core. +`get_supported_entities()` reports that backend's supported inventory, including +opt-in labels; it does not mean every label is enabled by default. German +detection is opt-in through locale or entity selection. `UUID`, `PERSON` and +`ADDRESS` can be selected explicitly. Unsupported labels and invalid arguments +raise errors instead of silently disabling detection. The [4.8.1 compatibility contract](docs/migration-4.8.1-contract.md) records published Python behavior for the Rust migration, with frozen fixtures and diff --git a/datafog/__about__.py b/datafog/__about__.py index bdb8bda6..ade1e2b4 100644 --- a/datafog/__about__.py +++ b/datafog/__about__.py @@ -1 +1 @@ -__version__ = "4.11.0" +__version__ = "4.12.0" diff --git a/datafog/__init__.py b/datafog/__init__.py index b0c7f2a8..f9a1695e 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -157,7 +157,7 @@ def scan( *, backend: str | None = None, ) -> ScanResult: - """Scan using the 4.x compatibility API; Rust detection is opt-in.""" + """Scan using the 4.x compatibility API; native detection is the default.""" return _compat_v4.scan( text, engine, @@ -181,6 +181,8 @@ def redact( allowlist_patterns: list[str] | None = None, *, backend: str | None = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """Redact using the 4.x compatibility API and legacy strategies.""" return _compat_v4.redact( @@ -194,6 +196,8 @@ def redact( allowlist, allowlist_patterns, backend=backend, + hash_type=hash_type, + hash_key=hash_key, ) @@ -242,18 +246,15 @@ def _detect_impl(text: str) -> list: _start = _time.monotonic() - _lazy_import_regex_annotator() - annotator = RegexAnnotator() - # Use the structured output to get proper positions - _, result = annotator.annotate_with_spans(text) + result = scan(text) # Convert to simple format, filtering out empty matches entities = [] - for span in result.spans: + for span in result.entities: if span.text.strip(): # Only include non-empty matches entities.append( { - "type": span.label, + "type": span.type, "value": span.text, "start": span.start, "end": span.end, @@ -284,7 +285,14 @@ def _detect_impl(text: str) -> list: return entities -def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: +def process( + text: str, + anonymize: bool = False, + method: str = "redact", + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, +) -> dict: """ Process text to detect and optionally anonymize PII. @@ -311,31 +319,37 @@ def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: _start = _time.monotonic() + if method not in {"redact", "replace", "hash"}: + raise ValueError("method must be 'redact', 'replace', or 'hash'") findings = _detect_impl(text) result = {"original": text, "findings": findings} - if anonymize: - anonymized = text - # Simple anonymization - replace from end to start to preserve positions - for finding in sorted(findings, key=lambda x: x["start"], reverse=True): - start, end = finding["start"], finding["end"] - entity_type = finding["type"] - - if method == "redact": - replacement = f"[{entity_type}_REDACTED]" - elif method == "replace": - replacement = f"[{entity_type}_XXXXX]" - elif method == "hash": - import hashlib - - replacement = f"[{entity_type}_{hashlib.md5(finding['value'].encode()).hexdigest()[:8]}]" - else: - replacement = f"[{entity_type}]" - - anonymized = anonymized[:start] + replacement + anonymized[end:] - - result["anonymized"] = anonymized + if anonymize and method == "hash": + entities = [ + Entity( + item["type"], item["value"], item["start"], item["end"], 1.0, "regex" + ) + for item in findings + ] + result["anonymized"] = redact( + text, + entities=entities, + strategy="hash", + hash_type=hash_type, + hash_key=hash_key, + ).redacted_text + elif anonymize: + parts = [] + cursor = 0 + suffix = "REDACTED" if method == "redact" else "XXXXX" + for finding in sorted(findings, key=lambda item: item["start"]): + parts.extend( + (text[cursor : finding["start"]], f"[{finding['type']}_{suffix}]") + ) + cursor = finding["end"] + parts.append(text[cursor:]) + result["anonymized"] = "".join(parts) try: from .telemetry import _get_duration_bucket, track_function_call diff --git a/datafog/__init___lean.py b/datafog/__init___lean.py deleted file mode 100644 index 3f1165b8..00000000 --- a/datafog/__init___lean.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Historical shadow package export module. - -This file is not the live DataFog 4.5 package export surface. Use -``datafog/__init__.py`` for current package exports, lazy optional imports, -top-level helpers, and compatibility shims. This snapshot remains importable -only as historical reference until legacy cleanup can remove it safely. - -DataFog: Lightning-fast PII detection and anonymization library. - -Core package provides regex-based PII detection, 100x+ faster than NER-based -detection on identical payloads (reproduce: python benchmarks/run.py). -Optional extras available for advanced features: -- pip install datafog[nlp] - for spaCy integration -- pip install datafog[ocr] - for image/OCR processing -- pip install datafog[all] - for all features -""" - -from .__about__ import __version__ - -# Core imports - always available -from .models.annotator import AnnotationResult, AnnotatorRequest -from .models.anonymizer import ( - AnonymizationResult, - Anonymizer, - AnonymizerRequest, - AnonymizerType, -) -from .models.common import EntityTypes -from .processing.text_processing.regex_annotator import RegexAnnotator - -# Optional imports with graceful fallback -try: - from .client import app -except ImportError: - app = None - -try: - from .main import DataFog, TextPIIAnnotator -except ImportError: - DataFog = None - TextPIIAnnotator = None - -try: - from .services.text_service import TextService -except ImportError: - TextService = None - - -# Optional heavy features - only import if dependencies available -def _optional_import(name, module_path, extra_name): - """Helper to import optional modules with helpful error messages.""" - try: - module = __import__(module_path, fromlist=[name]) - return getattr(module, name) - except ImportError: - - def _missing_dependency(*args, **kwargs): - raise ImportError( - f"{name} requires additional dependencies. " - f"Install with: pip install datafog[{extra_name}]" - ) - - return _missing_dependency - - -# OCR/Image processing - requires 'ocr' extra -DonutProcessor = _optional_import( - "DonutProcessor", "datafog.processing.image_processing.donut_processor", "ocr" -) -PytesseractProcessor = _optional_import( - "PytesseractProcessor", - "datafog.processing.image_processing.pytesseract_processor", - "ocr", -) -ImageService = _optional_import("ImageService", "datafog.services.image_service", "ocr") - -# NLP processing - requires 'nlp' extra -SpacyPIIAnnotator = _optional_import( - "SpacyPIIAnnotator", "datafog.processing.text_processing.spacy_pii_annotator", "nlp" -) - -# Distributed processing - requires 'distributed' extra -SparkService = _optional_import( - "SparkService", "datafog.services.spark_service", "distributed" -) - - -# Simple API for core functionality -def detect(text: str) -> list: - """ - Detect PII in text using regex patterns. - - Args: - text: Input text to scan for PII - - Returns: - List of detected PII entities - - Example: - >>> from datafog import detect - >>> detect("Contact john@example.com") - [{'type': 'EMAIL', 'value': 'john@example.com', 'start': 8, 'end': 24}] - """ - annotator = RegexAnnotator() - result = annotator.annotate(text) - - # Convert to simple format - entities = [] - for entity_type, matches in result.items(): - for match in matches: - entities.append( - { - "type": entity_type, - "value": match, - "start": text.find(match), - "end": text.find(match) + len(match), - } - ) - - return entities - - -def process(text: str, anonymize: bool = False, method: str = "redact") -> dict: - """ - Process text to detect and optionally anonymize PII. - - Args: - text: Input text to process - anonymize: Whether to anonymize detected PII - method: Anonymization method ('redact', 'replace', 'hash') - - Returns: - Dictionary with original text, anonymized text (if requested), and findings - - Example: - >>> from datafog import process - >>> process("Contact john@example.com", anonymize=True) - { - 'original': 'Contact john@example.com', - 'anonymized': 'Contact [EMAIL_REDACTED]', - 'findings': [{'type': 'EMAIL', 'value': 'john@example.com', ...}] - } - """ - findings = detect(text) - - result = {"original": text, "findings": findings} - - if anonymize: - anonymized = text - # Simple anonymization - replace from end to start to preserve positions - for finding in sorted(findings, key=lambda x: x["start"], reverse=True): - start, end = finding["start"], finding["end"] - entity_type = finding["type"] - - if method == "redact": - replacement = f"[{entity_type}_REDACTED]" - elif method == "replace": - replacement = f"[{entity_type}_XXXXX]" - elif method == "hash": - import hashlib - - replacement = f"[{entity_type}_{hashlib.md5(finding['value'].encode()).hexdigest()[:8]}]" - else: - replacement = f"[{entity_type}]" - - anonymized = anonymized[:start] + replacement + anonymized[end:] - - result["anonymized"] = anonymized - - return result - - -# Core exports -__all__ = [ - "__version__", - "detect", - "process", - "AnnotationResult", - "AnnotatorRequest", - "AnonymizationResult", - "Anonymizer", - "AnonymizerRequest", - "AnonymizerType", - "EntityTypes", - "RegexAnnotator", - # Optional exports (may be None if dependencies missing) - "DataFog", - "TextPIIAnnotator", - "TextService", - "app", - "DonutProcessor", - "PytesseractProcessor", - "ImageService", - "SpacyPIIAnnotator", - "SparkService", -] diff --git a/datafog/__init___original.py b/datafog/__init___original.py deleted file mode 100644 index 380511b1..00000000 --- a/datafog/__init___original.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Historical shadow package export module. - -This file is not the live DataFog 4.5 package export surface. Use -``datafog/__init__.py`` for current package exports, lazy optional imports, -top-level helpers, and compatibility shims. This eager-export snapshot remains -importable only as historical reference until legacy cleanup can remove it -safely. -""" - -from .__about__ import __version__ -from .client import app -from .config import OperationType, get_config -from .main import DataFog, TextPIIAnnotator -from .models.annotator import ( - AnalysisExplanation, - AnnotationResult, - AnnotationResultWithAnaysisExplanation, - AnnotatorRequest, -) -from .models.anonymizer import ( - AnonymizationResult, - Anonymizer, - AnonymizerRequest, - AnonymizerType, -) -from .models.common import AnnotatorMetadata, EntityTypes, Pattern, PatternRecognizer -from .models.spacy_nlp import SpacyAnnotator -from .processing.image_processing.donut_processor import DonutProcessor -from .processing.image_processing.image_downloader import ImageDownloader -from .processing.image_processing.pytesseract_processor import PytesseractProcessor -from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator -from .services.image_service import ImageService -from .services.spark_service import SparkService -from .services.text_service import TextService - -__all__ = [ - "DonutProcessor", - "DataFog", - "ImageService", - "OperationType", - "SparkService", - "TextPIIAnnotator", - "TextService", - "SpacyPIIAnnotator", - "ImageDownloader", - "PytesseractProcessor", - "__version__", - "app", - "AnalysisExplanation", - "AnnotationResult", - "AnnotationResultWithAnaysisExplanation", - "AnnotatorRequest", - "AnnotatorMetadata", - "EntityTypes", - "Pattern", - "PatternRecognizer", - "get_config", - "SpacyAnnotator", - "AnonymizerType", - "AnonymizerRequest", - "AnonymizationResult", - "Anonymizer", -] diff --git a/datafog/_legacy_retirement.py b/datafog/_legacy_retirement.py index e86506db..41753955 100644 --- a/datafog/_legacy_retirement.py +++ b/datafog/_legacy_retirement.py @@ -2,8 +2,10 @@ import warnings +from .__about__ import __version__ + #: Final release on which retired optional surfaces still work. -LAST_SUPPORTED = "4.10.0" +LAST_SUPPORTED = __version__ class LegacySurfaceWarning(FutureWarning): diff --git a/datafog/cli.py b/datafog/cli.py new file mode 100644 index 00000000..d5a86302 --- /dev/null +++ b/datafog/cli.py @@ -0,0 +1,17 @@ +"""Console bootstrap with an actionable optional dependency error.""" + + +def main(): + try: + import typer # noqa: F401 + except ImportError: + import sys + + print( + 'The datafog command requires the CLI extra: pip install "datafog[cli]"', + file=sys.stderr, + ) + raise SystemExit(1) from None + from .client import app + + app() diff --git a/datafog/client.py b/datafog/client.py index fd9a303d..cde33431 100644 --- a/datafog/client.py +++ b/datafog/client.py @@ -46,6 +46,15 @@ def list_entities(): app = typer.Typer() +def _operations(value: str) -> list[OperationType]: + try: + return [OperationType(op.strip()) for op in value.split(",")] + except ValueError: + raise typer.BadParameter( + "operations must use scan, extract, redact, replace, or hash" + ) from None + + @app.command() def scan_image( image_urls: List[str] = typer.Argument( @@ -70,9 +79,8 @@ def scan_image( typer.echo("No image URLs or file paths provided. Please provide at least one.") raise typer.Exit(code=1) - logging.basicConfig(level=logging.INFO) # Convert comma-separated string operations to a list of OperationType objects - operation_list = [OperationType(op.strip()) for op in operations.split(",")] + operation_list = _operations(operations) ocr_client = DataFog(operations=operation_list) try: results = asyncio.run(ocr_client.run_ocr_pipeline(image_urls=image_urls)) @@ -130,9 +138,8 @@ def scan_text( typer.echo("No texts provided.") raise typer.Exit(code=1) - logging.basicConfig(level=logging.INFO) # Convert comma-separated string operations to a list of OperationType objects - operation_list = [OperationType(op.strip()) for op in operations.split(",")] + operation_list = _operations(operations) text_client = DataFog(operations=operation_list, locales=locale) try: results = text_client.run_text_pipeline_sync(str_list=str_list) @@ -179,7 +186,11 @@ def show_config(): Prints the current DataFog configuration. """ - typer.echo(get_config()) + config = get_config() + values = dict(config) if isinstance(config, dict) else config.model_dump() + if values.get("api_key"): + values["api_key"] = "[REDACTED]" + typer.echo(values) @app.command() @@ -432,12 +443,11 @@ def hash_text( typer.echo("No text provided to hash.") raise typer.Exit(code=1) - # HashType is retained for backward-compatible CLI signature. - _ = hash_type result = scan_and_redact( text=text, engine="smart", strategy="hash", + hash_type=hash_type.value, locales=locale, ) typer.echo(result.redacted_text) diff --git a/datafog/compat/v4.py b/datafog/compat/v4.py index 96d619aa..ce6f00f4 100644 --- a/datafog/compat/v4.py +++ b/datafog/compat/v4.py @@ -66,6 +66,8 @@ def redact( allowlist_patterns: list[str] | None = None, *, backend: str | None = None, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> RedactResult: """ Redact with legacy strategies and result schema. @@ -92,7 +94,13 @@ def redact( "allowlist/allowlist_patterns cannot be combined with explicit " "entities; filter the entities before calling redact" ) - return _redact_entities(text=text, entities=entities, strategy=strategy) + return _redact_entities( + text=text, + entities=entities, + strategy=strategy, + hash_type=hash_type, + hash_key=hash_key, + ) return _scan_and_redact( text=text, @@ -103,4 +111,6 @@ def redact( allowlist=allowlist, allowlist_patterns=allowlist_patterns, backend=backend, + hash_type=hash_type, + hash_key=hash_key, ) diff --git a/datafog/core.py b/datafog/core.py index e56c2aa5..2153ee94 100644 --- a/datafog/core.py +++ b/datafog/core.py @@ -89,6 +89,9 @@ def anonymize_text( text: str, method: Union[str, AnonymizerType] = "redact", locales: List[str] | None = None, + *, + hash_type: str = "sha256", + hash_key: bytes | None = None, ) -> str: """ Simple text anonymization using lightweight regex engine. @@ -103,7 +106,7 @@ def anonymize_text( Example: >>> result = anonymize_text("Contact john@example.com", method="redact") >>> print(result) - "Contact [EMAIL_REDACTED]" + "Contact [EMAIL_1]" """ import time as _time @@ -129,6 +132,8 @@ def anonymize_text( engine=REGEX_ENGINE, strategy=strategy_map[method], locales=locales, + hash_type=hash_type, + hash_key=hash_key, ) try: @@ -211,18 +216,29 @@ def scan_text( return result -def get_supported_entities(locales: List[str] | None = None) -> List[str]: +def get_supported_entities( + locales: List[str] | None = None, *, backend: str | None = None +) -> List[str]: """ - Get list of PII entity types supported by the regex engine. + Get the selected backend's supported entity types, including opt-in labels. Returns: List of supported entity type names Example: >>> entities = get_supported_entities() - >>> print(entities) - ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DATE', 'ZIP_CODE'] + >>> "EMAIL" in entities + True """ + from datafog.engine import _require_core, resolve_backend + + if resolve_backend(backend, "regex") == "rust": + from datafog._core_capabilities import _read_capabilities, scan_configs + + core = _require_core() + supported, _, _ = _read_capabilities(core.capabilities) + scan_configs(core, set(), locales) + return sorted(supported) annotator = RegexAnnotator(locales=locales) legacy_map = {"DOB": "DATE", "ZIP": "ZIP_CODE"} result = [] diff --git a/datafog/main.py b/datafog/main.py index 0dc6f5d5..30fd1eed 100644 --- a/datafog/main.py +++ b/datafog/main.py @@ -15,12 +15,12 @@ from datafog._legacy_retirement import warn_legacy_surface from .config import OperationType +from .core import get_supported_entities from .engine import scan, scan_and_redact from .models.anonymizer import Anonymizer, AnonymizerType, HashType from .processing.text_processing.regex_annotator import RegexAnnotator logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) class DataFog: @@ -42,6 +42,7 @@ def __init__( hash_type: HashType = HashType.SHA256, anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, locales: List[str] | None = None, + hash_key: bytes | None = None, ): self.locales = locales self.regex_annotator = RegexAnnotator(locales=locales) @@ -59,6 +60,7 @@ def __init__( hash_type=hash_type, anonymizer_type=anonymizer_type ) self.hash_type = hash_type + self.hash_key = hash_key self.anonymizer_type = anonymizer_type self.logger = logging.getLogger(__name__) self.logger.info("Initializing lightweight DataFog class with regex engine") @@ -106,7 +108,15 @@ def run_text_pipeline_sync(self, str_list: List[str]) -> List: _start = _time.monotonic() try: self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: + if any( + op in self.operations + for op in ( + OperationType.SCAN, + OperationType.REDACT, + OperationType.REPLACE, + OperationType.HASH, + ) + ): annotated_text = [self.detect(text) for text in str_list] self.logger.info( @@ -190,8 +200,11 @@ def detect(self, text: str) -> dict: # Only pre-populate keys for labels active under the configured # locales so the default output shape matches v4.4.0 (no DE_* keys # unless German locale support is enabled). - result = {label: [] for label in RegexAnnotator.active_labels_for(self.locales)} legacy_map = {"DATE": "DOB", "ZIP_CODE": "ZIP"} + result = { + legacy_map.get(label, label): [] + for label in get_supported_entities(self.locales) + } for entity in scan_result.entities: label = legacy_map.get(entity.type, entity.type) result.setdefault(label, []).append(entity.text) @@ -239,6 +252,8 @@ def process( _start = _time.monotonic() + if method not in {"redact", "replace", "hash"}: + raise ValueError("method must be 'redact', 'replace', or 'hash'") annotations_dict = self.detect(text) result = {"original": text, "findings": annotations_dict} @@ -249,12 +264,18 @@ def process( "replace": "pseudonymize", "hash": "hash", } - strategy = strategy_map.get(method, "token") + strategy = strategy_map[method] redact_result = scan_and_redact( text=text, engine="regex", strategy=strategy, locales=self.locales, + hash_type=( + self.hash_type.value + if isinstance(self.hash_type, HashType) + else self.hash_type + ), + hash_key=self.hash_key, ) result["anonymized"] = redact_result.redacted_text @@ -298,6 +319,7 @@ class TextPIIAnnotator: """ def __init__(self, locales: List[str] | None = None): + self.locales = locales self.regex_annotator = RegexAnnotator(locales=locales) def run(self, text, output_path=None): @@ -312,7 +334,13 @@ def run(self, text, output_path=None): Dictionary mapping entity types to lists of found entities """ try: - annotated_text = self.regex_annotator.annotate(text) + result = scan(text=text, engine="regex", locales=self.locales) + legacy = {"DATE": "DOB", "ZIP_CODE": "ZIP"} + annotated_text = {} + for entity in result.entities: + annotated_text.setdefault( + legacy.get(entity.type, entity.type), [] + ).append(entity.text) # Optionally, output the results to a JSON file if output_path: diff --git a/datafog/main_lean.py b/datafog/main_lean.py deleted file mode 100644 index f67501e6..00000000 --- a/datafog/main_lean.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Historical shadow main module. - -This file is not the live DataFog 4.5 ``DataFog`` implementation. Use -``datafog/main.py`` for the current backward-compatible class and text/OCR -compatibility methods. This lean snapshot remains importable only as historical -reference until legacy cleanup can remove it safely. - -Lean main module for DataFog core functionality. - -This module contains the lightweight core classes for DataFog: -- DataFog: Main class for regex-based PII detection -- TextPIIAnnotator: Class for annotating PII in text using regex patterns - -These classes provide the core PII detection functionality without heavy dependencies. -""" - -import json -import logging -from typing import List - -from .config import OperationType -from .models.anonymizer import Anonymizer, AnonymizerType, HashType -from .processing.text_processing.regex_annotator import RegexAnnotator - -logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) - - -class DataFog: - """ - Lightweight main class for regex-based PII detection and anonymization. - - Handles text processing operations using fast regex patterns for PII detection. - For advanced features like OCR, spaCy, or Spark, install additional extras. - - Attributes: - regex_annotator: Core regex-based PII annotator. - operations: List of operations to perform. - anonymizer: Anonymizer for PII redaction, replacement, or hashing. - """ - - def __init__( - self, - operations: List[OperationType] = [OperationType.SCAN], - hash_type: HashType = HashType.SHA256, - anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, - ): - self.regex_annotator = RegexAnnotator() - self.operations: List[OperationType] = operations - self.anonymizer = Anonymizer( - hash_type=hash_type, anonymizer_type=anonymizer_type - ) - self.logger = logging.getLogger(__name__) - self.logger.info("Initializing lightweight DataFog class with regex engine") - self.logger.info(f"Operations: {operations}") - self.logger.info(f"Hash Type: {hash_type}") - self.logger.info(f"Anonymizer Type: {anonymizer_type}") - - def run_text_pipeline_sync(self, str_list: List[str]) -> List[str]: - """ - Run the text pipeline synchronously on a list of input text. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List[str]: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: - annotated_text = [] - - for text in str_list: - # Use regex annotator for core PII detection - annotations = self.regex_annotator.annotate(text) - annotated_text.append(annotations) - - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if any( - op in self.operations - for op in [ - OperationType.REDACT, - OperationType.REPLACE, - OperationType.HASH, - ] - ): - return [ - self.anonymizer.anonymize(text, annotations).anonymized_text - for text, annotations in zip( - str_list, annotated_text, strict=True - ) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return str_list - except Exception as e: - self.logger.error(f"Error in run_text_pipeline_sync: {str(e)}") - raise - - def detect(self, text: str) -> dict: - """ - Simple PII detection using regex patterns. - - Args: - text: Input text to scan for PII - - Returns: - Dictionary mapping entity types to lists of found entities - """ - return self.regex_annotator.annotate(text) - - def process( - self, text: str, anonymize: bool = False, method: str = "redact" - ) -> dict: - """ - Process text to detect and optionally anonymize PII. - - Args: - text: Input text to process - anonymize: Whether to anonymize detected PII - method: Anonymization method ('redact', 'replace', 'hash') - - Returns: - Dictionary with original text, anonymized text (if requested), and findings - """ - annotations = self.detect(text) - - result = {"original": text, "findings": annotations} - - if anonymize: - if method == "redact": - anonymizer_type = AnonymizerType.REDACT - elif method == "replace": - anonymizer_type = AnonymizerType.REPLACE - elif method == "hash": - anonymizer_type = AnonymizerType.HASH - else: - anonymizer_type = AnonymizerType.REDACT - - anonymized_result = self.anonymizer.anonymize( - text, annotations, anonymizer_type - ) - result["anonymized"] = anonymized_result.anonymized_text - - return result - - -class TextPIIAnnotator: - """ - Lightweight class for annotating PII in text using regex patterns. - - Provides functionality to detect and annotate Personally Identifiable Information (PII) - in text using fast regex patterns instead of heavy NLP models. - - Attributes: - regex_annotator: RegexAnnotator instance for text annotation. - """ - - def __init__(self): - self.regex_annotator = RegexAnnotator() - - def run(self, text, output_path=None): - """ - Run PII annotation on text using regex patterns. - - Args: - text: Input text to annotate - output_path: Optional path to save results as JSON - - Returns: - Dictionary mapping entity types to lists of found entities - """ - try: - annotated_text = self.regex_annotator.annotate(text) - - # Optionally, output the results to a JSON file - if output_path: - with open(output_path, "w") as f: - json.dump(annotated_text, f) - - return annotated_text - - except Exception as e: - logging.error(f"Error in TextPIIAnnotator.run: {str(e)}") - raise diff --git a/datafog/main_original.py b/datafog/main_original.py deleted file mode 100644 index 9594c824..00000000 --- a/datafog/main_original.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Historical shadow main module. - -This file is not the live DataFog 4.5 ``DataFog`` implementation. Use -``datafog/main.py`` for the current backward-compatible class and use the -optional OCR/Spark service modules directly when those surfaces are needed. -This original full-featured snapshot remains importable for legacy tests until -cleanup can remove it safely. - -Main module for DataFog. - -This module contains the core classes for DataFog: -- DataFog: Main class for running OCR and text processing pipelines. -- TextPIIAnnotator: Class for annotating PII in text. - -These classes provide high-level interfaces for image and text processing, -including OCR, PII detection, annotation, and anonymization. -""" - -import json -import logging -from typing import List - -from .config import OperationType -from .models.anonymizer import Anonymizer, AnonymizerType, HashType -from .processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator -from .services.image_service import ImageService -from .services.spark_service import SparkService -from .services.text_service import TextService - -logger = logging.getLogger("datafog_logger") -logger.setLevel(logging.INFO) - - -class DataFog: - """ - Main class for running OCR and text processing pipelines. - - Handles image and text processing operations, including OCR, PII detection, and anonymization. - - Attributes: - image_service: Service for image processing and OCR. - text_service: Service for text processing and annotation. - spark_service: Optional Spark service for distributed processing. - operations: List of operations to perform. - anonymizer: Anonymizer for PII redaction, replacement, or hashing. - """ - - def __init__( - self, - image_service=None, - text_service=None, - spark_service=None, - operations: List[OperationType] = [OperationType.SCAN], - hash_type: HashType = HashType.SHA256, - anonymizer_type: AnonymizerType = AnonymizerType.REPLACE, - ): - self.image_service = image_service or ImageService() - self.text_service = text_service or TextService() - self.spark_service: SparkService = spark_service - self.operations: List[OperationType] = operations - self.anonymizer = Anonymizer( - hash_type=hash_type, anonymizer_type=anonymizer_type - ) - self.logger = logging.getLogger(__name__) - self.logger.info( - "Initializing DataFog class with the following services and operations:" - ) - self.logger.info(f"Image Service: {type(self.image_service)}") - self.logger.info(f"Text Service: {type(self.text_service)}") - self.logger.info( - f"Spark Service: {type(self.spark_service) if self.spark_service else 'None'}" - ) - self.logger.info(f"Operations: {operations}") - self.logger.info(f"Hash Type: {hash_type}") - self.logger.info(f"Anonymizer Type: {anonymizer_type}") - - async def run_ocr_pipeline(self, image_urls: List[str]): - """ - Run the OCR pipeline asynchronously on a list of images provided via URL. - - This method performs optical character recognition (OCR) on the images specified by the URLs. - If PII annotation is enabled, it also annotates the extracted text for personally identifiable information. - If redaction, replacement, or hashing is enabled, it applies the corresponding anonymization. - - Args: - image_urls (List[str]): A list of URLs pointing to the images to be processed. - - Returns: - List: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the OCR or text processing. - """ - try: - extracted_text = await self.image_service.ocr_extract(image_urls) - self.logger.info(f"OCR extraction completed for {len(image_urls)} images.") - - return await self._process_text(extracted_text) - except Exception as e: - logging.error(f"Error in run_ocr_pipeline: {str(e)}") - return [f"Error: {str(e)}"] - - async def run_text_pipeline(self, str_list: List[str]): - """ - Run the text pipeline asynchronously on a list of input text. - - This method processes a list of text strings, potentially annotating them for personally - identifiable information (PII) and applying anonymization if enabled. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - return await self._process_text(str_list) - except Exception as e: - self.logger.error(f"Error in run_text_pipeline: {str(e)}") - raise - - async def _process_text(self, text_list: List[str]): - """ - Internal method to process text based on enabled operations. - """ - if OperationType.SCAN in self.operations: - annotated_text = await self.text_service.batch_annotate_text_async( - text_list - ) - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if OperationType.REDACT in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.REDACT - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - elif OperationType.REPLACE in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.REPLACE - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - elif OperationType.HASH in self.operations: - return [ - self.anonymizer.anonymize( - text, annotations, AnonymizerType.HASH - ).anonymized_text - for text, annotations in zip(text_list, annotated_text, strict=True) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return text_list - - def run_text_pipeline_sync(self, str_list: List[str]) -> List[str]: - """ - Run the text pipeline synchronously on a list of input text. - - Args: - str_list (List[str]): A list of text strings to be processed. - - Returns: - List[str]: Processed text results based on the enabled operations. - - Raises: - Exception: Any error encountered during the text processing. - """ - try: - self.logger.info(f"Starting text pipeline with {len(str_list)} texts.") - if OperationType.SCAN in self.operations: - annotated_text = self.text_service.batch_annotate_text_sync(str_list) - self.logger.info( - f"Text annotation completed with {len(annotated_text)} annotations." - ) - - if any( - op in self.operations - for op in [ - OperationType.REDACT, - OperationType.REPLACE, - OperationType.HASH, - ] - ): - return [ - self.anonymizer.anonymize(text, annotations).anonymized_text - for text, annotations in zip( - str_list, annotated_text, strict=True - ) - ] - else: - return annotated_text - - self.logger.info( - "No annotation or anonymization operation found; returning original texts." - ) - return str_list - except Exception as e: - self.logger.error(f"Error in run_text_pipeline_sync: {str(e)}") - raise - - def _add_attributes(self, attributes: dict): - """ - Add multiple attributes to the DataFog instance. - - This private method allows for the dynamic addition of multiple attributes to the - DataFog instance. It iterates through the provided dictionary of attributes and - adds each key-value pair as an attribute. - - Args: - attributes (dict): A dictionary where keys are attribute names and values are - the corresponding attribute values to be added. - - Note: - This method is intended for internal use and may be used for extending the - functionality of the DataFog class dynamically. Care should be taken when - using this method to avoid overwriting existing attributes. - """ - for key, value in attributes.items(): - setattr(self, key, value) - - -class TextPIIAnnotator: - """ - Class for annotating PII in text. - - Provides functionality to detect and annotate Personally Identifiable Information (PII) in text. - - Attributes: - text_annotator: SpacyPIIAnnotator instance for text annotation. - spark_processor: Optional SparkService for distributed processing. - """ - - def __init__(self): - self.text_annotator = SpacyPIIAnnotator.create() - self.spark_processor: SparkService = None - - def run(self, text, output_path=None): - try: - annotated_text = self.text_annotator.annotate(text) - - # Optionally, output the results to a JSON file - if output_path: - with open(output_path, "w") as f: - json.dump(annotated_text, f) - - return annotated_text - - finally: - # Ensure Spark resources are released - if self.spark_processor: - self.spark_processor.stop() diff --git a/datafog/models/annotator.py b/datafog/models/annotator.py index 2f915ea4..489689b5 100644 --- a/datafog/models/annotator.py +++ b/datafog/models/annotator.py @@ -22,12 +22,12 @@ class AnnotatorRequest(BaseModel): text: str language: str - correlation_id: Optional[str] - score_threshold: Optional[float] - entities: Optional[List[EntityTypes]] - return_decision_process: Optional[bool] - ad_hoc_recognizers: Optional[List[PatternRecognizer]] - context: Optional[List[str]] + correlation_id: Optional[str] = None + score_threshold: Optional[float] = None + entities: Optional[List[EntityTypes]] = None + return_decision_process: Optional[bool] = None + ad_hoc_recognizers: Optional[List[PatternRecognizer]] = None + context: Optional[List[str]] = None class AnnotationResult(BaseModel): @@ -39,16 +39,19 @@ class AnnotationResult(BaseModel): start: int end: int - score: Optional[float] + score: Optional[float] = None entity_type: str - recognition_metadata: Optional[AnnotatorMetadata] + recognition_metadata: Optional[AnnotatorMetadata] = None - @field_validator("entity_type") + @field_validator("entity_type", mode="before") @classmethod def validate_entity_type(cls, v): - if v not in EntityTypes.__members__: - return "UNKNOWN" - return v + from datafog.engine import ALL_ENTITY_TYPES, CANONICAL_TYPE_MAP + + if isinstance(v, EntityTypes): + v = v.name + v = CANONICAL_TYPE_MAP.get(v, v) + return v if v in ALL_ENTITY_TYPES else "UNKNOWN" class AnalysisExplanation(BaseModel): @@ -60,14 +63,14 @@ class AnalysisExplanation(BaseModel): """ recognizer: str - pattern_name: Optional[str] - pattern: Optional[str] + pattern_name: Optional[str] = None + pattern: Optional[str] = None original_score: float score: float - textual_explanation: Optional[str] + textual_explanation: Optional[str] = None score_context_improvement: float supportive_context_word: str - validation_result: Optional[float] + validation_result: Optional[float] = None class AnnotationResultWithAnaysisExplanation(AnnotationResult): @@ -75,4 +78,4 @@ class AnnotationResultWithAnaysisExplanation(AnnotationResult): Extends AnnotationResult with detailed analysis explanation. """ - analysis_explanation: Optional[AnalysisExplanation] + analysis_explanation: Optional[AnalysisExplanation] = None diff --git a/datafog/models/anonymizer.py b/datafog/models/anonymizer.py index 6b81f5be..30d079bb 100644 --- a/datafog/models/anonymizer.py +++ b/datafog/models/anonymizer.py @@ -50,6 +50,33 @@ class Anonymizer(BaseModel): def __init__(self, **data): super().__init__(**data) + def _selected_annotations(self, text, annotations): + from datafog.engine import CANONICAL_TYPE_MAP + + selected = ( + None + if self.entities is None + else { + CANONICAL_TYPE_MAP.get(entity.name, entity.name) + for entity in self.entities + } + ) + merged = [] + for annotation in sorted(annotations, key=lambda a: (a.start, -a.end)): + if not 0 <= annotation.start < annotation.end <= len(text): + raise ValueError( + "Annotation range must be nonempty and inside the text" + ) + if selected is not None and annotation.entity_type not in selected: + continue + if merged and annotation.start < merged[-1].end: + merged[-1] = merged[-1].model_copy( + update={"end": max(merged[-1].end, annotation.end)} + ) + else: + merged.append(annotation) + return reversed(merged) + def anonymize( self, text: str, annotations: List[AnnotationResult] ) -> AnonymizationResult: @@ -68,19 +95,18 @@ def replace_pii( ) -> AnonymizationResult: """Replace PII in text with anonymized values.""" replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if not self.entities or annotation.entity_type in self.entities: - replacement = self._generate_replacement( - text[annotation.start : annotation.end], annotation.entity_type - ) - replacements.append( - { - "original": text[annotation.start : annotation.end], - "replacement": replacement, - "entity_type": annotation.entity_type, - } - ) - text = text[: annotation.start] + replacement + text[annotation.end :] + for annotation in self._selected_annotations(text, annotations): + replacement = self._generate_replacement( + text[annotation.start : annotation.end], annotation.entity_type + ) + replacements.append( + { + "original": text[annotation.start : annotation.end], + "replacement": replacement, + "entity_type": annotation.entity_type, + } + ) + text = text[: annotation.start] + replacement + text[annotation.end :] return AnonymizationResult( anonymized_text=text, anonymized_entities=replacements @@ -88,13 +114,13 @@ def replace_pii( def _generate_replacement(self, original: str, entity_type: EntityTypes) -> str: """Generate a replacement for the given entity.""" - if entity_type == EntityTypes.PERSON: + if entity_type == "PERSON": return f"[PERSON_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.ORGANIZATION: + elif entity_type == "ORGANIZATION": return f"[ORGANIZATION_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.LOCATION: + elif entity_type == "LOCATION": return f"[LOCATION_{secrets.token_hex(4).upper()}]" - elif entity_type == EntityTypes.DATE: + elif entity_type == "DATE": return "[REDACTED_DATE]" else: return f"[{entity_type}_{secrets.token_hex(4).upper()}]" @@ -104,10 +130,7 @@ def hash_pii( ) -> AnonymizationResult: """Hash PII in text.""" replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if self.entities and annotation.entity_type not in self.entities: - continue - + for annotation in self._selected_annotations(text, annotations): start, end = annotation.start, annotation.end original = text[start:end] replacement = self._hash_text(original) @@ -139,10 +162,7 @@ def redact_pii( self, text: str, annotations: List[AnnotationResult] ) -> AnonymizationResult: replacements = [] - for annotation in sorted(annotations, key=lambda x: x.start, reverse=True): - if self.entities and annotation.entity_type not in self.entities: - continue - + for annotation in self._selected_annotations(text, annotations): start, end = annotation.start, annotation.end original = text[start:end] replacement = "[REDACTED]" diff --git a/datafog/models/common.py b/datafog/models/common.py index f01d6651..1f72d4aa 100644 --- a/datafog/models/common.py +++ b/datafog/models/common.py @@ -40,8 +40,8 @@ class PatternRecognizer(BaseModel): name: str supported_language: str patterns: List[Pattern] - deny_list: Optional[List[str]] - context: Optional[List[str]] + deny_list: Optional[List[str]] = None + context: Optional[List[str]] = None supported_entity: str diff --git a/datafog/native.py b/datafog/native.py index cde64934..44b6fb29 100644 --- a/datafog/native.py +++ b/datafog/native.py @@ -40,7 +40,7 @@ def _core() -> Any: except ImportError as exc: # pragma: no cover - base install always has it raise ImportError( "datafog.native requires datafog-core. Install with: " - 'pip install "datafog==4.10.0"' + 'pip install "datafog[rust]"' ) from exc @@ -52,7 +52,13 @@ def __getattr__(name: str) -> Any: def version() -> str: """Installed Core version, for bug reports and reproducibility locks.""" - return importlib.metadata.version("datafog-core") + _core() + try: + return importlib.metadata.version("datafog-core") + except importlib.metadata.PackageNotFoundError as exc: + raise ImportError( + "datafog-core distribution metadata is missing; reinstall datafog[rust]" + ) from exc def contract_version() -> int: diff --git a/datafog/processing/spark_processing/pyspark_udfs.py b/datafog/processing/spark_processing/pyspark_udfs.py index aaa53ece..bb9b8c18 100644 --- a/datafog/processing/spark_processing/pyspark_udfs.py +++ b/datafog/processing/spark_processing/pyspark_udfs.py @@ -28,8 +28,9 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]: if text: if len(text) > MAXIMAL_STRING_SIZE: - # Cut the strings for required sizes - text = text[:MAXIMAL_STRING_SIZE] + raise ValueError( + "Spark PII input exceeds the maximum supported string length" + ) nlp = broadcasted_nlp.value doc = nlp(text) @@ -38,8 +39,15 @@ def pii_annotator(text: str, broadcasted_nlp) -> list[list[str]]: _label: [] for _label in PII_ANNOTATION_LABELS } for ent in doc.ents: - # Add entities from extracted values - classified_entities[ent.label_].append(ent.text) + label = { + "PERSON": "PER", + "GPE": "LOC", + "NORP": "NRP", + "DATE": "DATE_TIME", + "TIME": "DATE_TIME", + }.get(ent.label_, ent.label_) + if label in classified_entities: + classified_entities[label].append(ent.text) return [_ent for _ent in classified_entities.values()] else: diff --git a/datafog/services/text_service_lean.py b/datafog/services/text_service_lean.py deleted file mode 100644 index 1262db3f..00000000 --- a/datafog/services/text_service_lean.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Historical shadow text service module. - -This file is not the live DataFog 4.5 text service. Use -``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER, -auto, and smart engine boundary. This lean snapshot remains importable only as -historical reference until legacy cleanup can remove it safely. - -Lean text processing service for PII annotation. - -Provides synchronous and asynchronous methods for annotating text with personally -identifiable information (PII) using regex patterns. Supports chunking long texts -and batch processing. SpaCy integration available as optional extra. -""" - -import asyncio -from typing import Dict, List, Union - -from datafog.processing.text_processing.regex_annotator.regex_annotator import ( - RegexAnnotator, - Span, -) - - -class TextService: - """ - Lightweight service for annotating text with PII entities using regex patterns. - - This service provides methods to detect and annotate personally identifiable information (PII) - in text using fast regex patterns. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. - - For advanced NLP-based detection using spaCy, install the 'nlp' extra: - pip install datafog[nlp] - """ - - def __init__(self, text_chunk_length: int = 1000, engine: str = "regex"): - """ - Initialize the TextService with specified chunk length and annotation engine. - - Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. - engine: The annotation engine to use. Options are: - - "regex": (Default) Use RegexAnnotator for fast pattern-based entity detection - - "spacy": Use SpacyPIIAnnotator for NLP-based entity detection (requires nlp extra) - - "auto": Try RegexAnnotator first and fall back to SpacyPIIAnnotator if no entities found - - Raises: - AssertionError: If an invalid engine type is provided - ImportError: If spacy engine is requested but nlp extra is not installed - """ - assert engine in {"regex", "spacy", "auto"}, "Invalid engine" - self.engine = engine - self.regex_annotator = RegexAnnotator() - self.text_chunk_length = text_chunk_length - - # Only initialize spacy if needed and available - self.spacy_annotator = None - if engine in {"spacy", "auto"}: - try: - from datafog.processing.text_processing.spacy_pii_annotator import ( - SpacyPIIAnnotator, - ) - - self.spacy_annotator = SpacyPIIAnnotator.create() - except ImportError: - if engine == "spacy": - raise ImportError( - "SpaCy engine requires additional dependencies. " - "Install with: pip install datafog[nlp]" - ) - # For auto mode, just continue with regex only - self.spacy_annotator = None - - def _chunk_text(self, text: str) -> List[str]: - """Split the text into chunks of specified length.""" - return [ - text[i : i + self.text_chunk_length] - for i in range(0, len(text), self.text_chunk_length) - ] - - def _combine_annotations( - self, chunk_annotations: List[Dict[str, List[str]]] - ) -> Dict[str, List[str]]: - """Combine annotations from multiple chunks.""" - combined = {} - for annotations in chunk_annotations: - for entity_type, entities in annotations.items(): - if entity_type not in combined: - combined[entity_type] = [] - combined[entity_type].extend(entities) - return combined - - def annotate_text_sync( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text synchronously for PII entities. - - Args: - text: The text to annotate - structured: If True, return structured Span objects. If False, return dict format. - - Returns: - Dictionary mapping entity types to lists of entities, or list of Span objects - """ - if len(text) <= self.text_chunk_length: - # Single chunk processing - if self.engine == "regex": - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return self.regex_annotator.annotate(text) - elif self.engine == "spacy": - if self.spacy_annotator is None: - raise ImportError( - "SpaCy engine not available. Install with: pip install datafog[nlp]" - ) - return self.spacy_annotator.annotate(text) - elif self.engine == "auto": - # Try regex first - regex_result = self.regex_annotator.annotate(text) - - # Check if regex found any entities - if any(entities for entities in regex_result.values()): - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return regex_result - - # Fall back to spacy if available - if self.spacy_annotator is not None: - return self.spacy_annotator.annotate(text) - - # Return regex result even if empty - if structured: - _, result = self.regex_annotator.annotate_with_spans(text) - return result.spans - return regex_result - else: - # Multi-chunk processing - chunks = self._chunk_text(text) - chunk_annotations = [] - - for chunk in chunks: - chunk_result = self.annotate_text_sync(chunk, structured=False) - chunk_annotations.append(chunk_result) - - if structured: - # For structured output with chunking, we need to recalculate positions - # This is more complex, so for now return dict format - return self._combine_annotations(chunk_annotations) - - return self._combine_annotations(chunk_annotations) - - async def annotate_text_async( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text asynchronously for PII entities. - - Args: - text: The text to annotate - structured: If True, return structured Span objects. If False, return dict format. - - Returns: - Dictionary mapping entity types to lists of entities, or list of Span objects - """ - # For regex processing, we can just run synchronously since it's fast - return self.annotate_text_sync(text, structured) - - def batch_annotate_text_sync(self, texts: List[str]) -> List[Dict[str, List[str]]]: - """ - Annotate multiple texts synchronously. - - Args: - texts: List of texts to annotate - - Returns: - List of annotation dictionaries, one per input text - """ - return [self.annotate_text_sync(text) for text in texts] - - async def batch_annotate_text_async( - self, texts: List[str] - ) -> List[Dict[str, List[str]]]: - """ - Annotate multiple texts asynchronously. - - Args: - texts: List of texts to annotate - - Returns: - List of annotation dictionaries, one per input text - """ - # For better performance with many texts, we can process them concurrently - tasks = [self.annotate_text_async(text) for text in texts] - return await asyncio.gather(*tasks) diff --git a/datafog/services/text_service_original.py b/datafog/services/text_service_original.py deleted file mode 100644 index 8ad576a9..00000000 --- a/datafog/services/text_service_original.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Historical shadow text service module. - -This file is not the live DataFog 4.5 text service. Use -``datafog/services/text_service.py`` for the current regex, spaCy, GLiNER, -auto, and smart engine boundary. This original snapshot remains importable for -legacy tests until cleanup can remove it safely. - -Text processing service for PII annotation. - -Provides synchronous and asynchronous methods for annotating text with personally identifiable information (PII) using SpaCy or regex patterns. Supports chunking long texts and batch processing. -""" - -import asyncio -from typing import Dict, List, Union - -from datafog.processing.text_processing.regex_annotator.regex_annotator import ( - RegexAnnotator, - Span, -) -from datafog.processing.text_processing.spacy_pii_annotator import SpacyPIIAnnotator - - -class TextService: - """ - Service for annotating text with PII entities. - - This service provides methods to detect and annotate personally identifiable information (PII) - in text using different annotation engines. It supports chunking long texts for efficient processing - and combining annotations from multiple chunks. - """ - - def __init__(self, text_chunk_length: int = 1000, engine: str = "auto"): - """ - Initialize the TextService with specified chunk length and annotation engine. - - Args: - text_chunk_length: Maximum length of text chunks for processing. Default is 1000 characters. - engine: The annotation engine to use. Options are: - - "regex": Use only the RegexAnnotator for pattern-based entity detection - - "spacy": Use only the SpacyPIIAnnotator for NLP-based entity detection - - "auto": (Default) Try RegexAnnotator first and fall back to SpacyPIIAnnotator if no entities are found - - Raises: - AssertionError: If an invalid engine type is provided - """ - assert engine in {"regex", "spacy", "auto"}, "Invalid engine" - self.engine = engine - self.spacy_annotator = SpacyPIIAnnotator.create() - self.regex_annotator = RegexAnnotator() - self.text_chunk_length = text_chunk_length - - def _chunk_text(self, text: str) -> List[str]: - """Split the text into chunks of specified length.""" - return [ - text[i : i + self.text_chunk_length] - for i in range(0, len(text), self.text_chunk_length) - ] - - def _combine_annotations( - self, annotations: List[Dict[str, List[str]]] - ) -> Dict[str, List[str]]: - """Combine annotations from multiple chunks.""" - combined: Dict[str, List[str]] = {} - for annotation in annotations: - for key, value in annotation.items(): - if key not in combined: - combined[key] = [] - combined[key].extend(value) - return combined - - def _annotate_with_engine( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Annotate text using the selected engine based on the engine parameter. - - This method implements the engine selection logic: - - For "regex" mode: Uses only the RegexAnnotator - - For "spacy" mode: Uses only the SpacyPIIAnnotator - - For "auto" mode: Tries RegexAnnotator first and falls back to SpacyPIIAnnotator if no entities are found - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary of annotations by entity type where keys are entity types (e.g., "EMAIL", "PERSON", "ORG") - and values are lists of detected entities of that type - If structured=True: List of Span objects with entity information - """ - if structured: - # Handle structured output mode - if self.engine == "regex": - _, annotation_result = self.regex_annotator.annotate_with_spans(text) - return annotation_result.spans - elif self.engine == "spacy": - # For spaCy, we need to convert the dictionary format to spans - spacy_dict = self.spacy_annotator.annotate(text) - spacy_spans: List[Span] = [] - for label, entities in spacy_dict.items(): - for entity in entities: - # Find the start and end positions of the entity in the text - start = text.find(entity) - if start >= 0: - end = start + len(entity) - span = Span(start=start, end=end, label=label, text=entity) - spacy_spans.append(span) - return spacy_spans - else: # "auto" mode - # Try regex first - _, annotation_result = self.regex_annotator.annotate_with_spans(text) - if annotation_result.spans: - return annotation_result.spans - - # If regex found nothing, fall back to spaCy - spacy_dict = self.spacy_annotator.annotate(text) - auto_spans: List[Span] = [] - for label, entities in spacy_dict.items(): - for entity in entities: - # Find the start and end positions of the entity in the text - start = text.find(entity) - if start >= 0: - end = start + len(entity) - span = Span(start=start, end=end, label=label, text=entity) - auto_spans.append(span) - return auto_spans - else: - # Handle legacy dictionary output mode - if self.engine == "regex": - return self.regex_annotator.annotate(text) - elif self.engine == "spacy": - return self.spacy_annotator.annotate(text) - else: # auto mode - # Try regex first - regex_dict = self.regex_annotator.annotate(text) - - # Check if any VALID entities were found (ignore empty strings) - has_entities = any( - any(entity.strip() for entity in entities) - for entities in regex_dict.values() - ) - - # If regex found entities, return those results - if has_entities: - return regex_dict - - # Otherwise, fall back to spaCy - return self.spacy_annotator.annotate(text) - - def annotate_text_sync( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Synchronously annotate a text string. - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary mapping entity types to lists of strings - If structured=True: List of Span objects with entity information - """ - if not text: - return [] if structured else {} - - chunks = self._chunk_text(text) - - if structured: - # Handle structured output mode - all_spans: List[Span] = [] - chunk_offset = 0 # Track the offset for each chunk in the original text - - for chunk in chunks: - # Process each chunk and get spans - chunk_spans = self._annotate_with_engine(chunk, structured=True) - if not isinstance(chunk_spans, list): - continue # Skip if not a list of spans - - # Adjust span positions based on chunk offset in the original text - for span in chunk_spans: - if not isinstance(span, Span): - continue # Skip if not a Span object - span.start += chunk_offset - span.end += chunk_offset - # Verify the span text matches the text at the adjusted position - if span.start < len(text) and span.end <= len(text): - span.text = text[span.start : span.end] - all_spans.append(span) - - # Update offset for the next chunk - chunk_offset += len(chunk) - - print(f"Done processing {text.split()[0]}") - return all_spans - else: - # Handle legacy dictionary output mode - annotations: List[Dict[str, List[str]]] = [] - for chunk in chunks: - res = self._annotate_with_engine(chunk) - if isinstance(res, dict): - annotations.append(res) - combined = self._combine_annotations(annotations) - print(f"Done processing {text.split()[0]}") - return combined - - def batch_annotate_text_sync( - self, texts: List[str], structured: bool = False - ) -> Dict[str, Union[Dict[str, List[str]], List[Span]]]: - """ - Synchronously annotate a list of text input. - - Args: - texts: List of text strings to annotate - structured: If True, return structured output (list of Span objects) for each text - - Returns: - Dictionary mapping each input text to its annotation result - """ - results = [ - self.annotate_text_sync(text, structured=structured) for text in texts - ] - return dict(zip(texts, results, strict=True)) - - async def annotate_text_async( - self, text: str, structured: bool = False - ) -> Union[Dict[str, List[str]], List[Span]]: - """ - Asynchronously annotate a text string. - - Args: - text: The text to annotate - structured: If True, return structured output (list of Span objects) - - Returns: - If structured=False: Dictionary mapping entity types to lists of strings - If structured=True: List of Span objects with entity information - """ - if not text: - return [] if structured else {} - - chunks = self._chunk_text(text) - - if structured: - # Handle structured output mode asynchronously - all_spans: List[Span] = [] - chunk_offset = 0 # Track the offset for each chunk in the original text - - for chunk in chunks: - # We can't easily parallelize this due to the need to track offsets sequentially - # In a production environment, you might want a more sophisticated approach - chunk_spans = self._annotate_with_engine(chunk, structured=True) - if not isinstance(chunk_spans, list): - continue # Skip if not a list of spans - - # Adjust span positions based on chunk offset in the original text - for span in chunk_spans: - if not isinstance(span, Span): - continue # Skip if not a Span object - span.start += chunk_offset - span.end += chunk_offset - # Verify the span text matches the text at the adjusted position - if span.start < len(text) and span.end <= len(text): - span.text = text[span.start : span.end] - all_spans.append(span) - - # Update offset for the next chunk - chunk_offset += len(chunk) - - return all_spans - else: - # Handle legacy dictionary output mode asynchronously - tasks = [ - asyncio.to_thread(self._annotate_with_engine, chunk) for chunk in chunks - ] - results = await asyncio.gather(*tasks) - annotations: List[Dict[str, List[str]]] = [ - r for r in results if isinstance(r, dict) - ] - return self._combine_annotations(annotations) - - async def batch_annotate_text_async( - self, texts: List[str], structured: bool = False - ) -> Dict[str, Union[Dict[str, List[str]], List[Span]]]: - """ - Asynchronously annotate a list of text input. - - Args: - texts: List of text strings to annotate - structured: If True, return structured output (list of Span objects) for each text - - Returns: - Dictionary mapping each input text to its annotation result - """ - tasks = [ - self.annotate_text_async(text, structured=structured) for text in texts - ] - results = await asyncio.gather(*tasks) - return dict(zip(texts, results, strict=True)) diff --git a/docs/migration-4.10.md b/docs/migration-4.10.md index 9a730c3b..6ea5db00 100644 --- a/docs/migration-4.10.md +++ b/docs/migration-4.10.md @@ -40,7 +40,7 @@ The native engine finds things the Python regex engine never did: `API_KEY`, stay opt-in. These differences go the other way and are recorded, not hidden, in -[tests/contracts/rust-1.3.0.json](../tests/contracts/rust-1.3.0.json): +[tests/contracts/rust-1.4.0.json](../tests/contracts/rust-1.4.0.json): | Input | Python | Native | Why | | --- | --- | --- | --- | @@ -105,5 +105,5 @@ a contract bump is a Core MAJOR and raises the floor here. ## What is not changing yet `detect()`, `process()`, OCR and Spark all still work and still warn. They are -removed in 5.0, not here. The last supported version is 4.10.0 — see the +removed in 5.0, not here. The last supported version in the current release train is 4.12.0 — see the [retirement schedule](retirement.rst). diff --git a/docs/retirement.rst b/docs/retirement.rst index 17c5c0af..01b2fc53 100644 --- a/docs/retirement.rst +++ b/docs/retirement.rst @@ -21,37 +21,37 @@ Removed or scheduled - Removed in - Deprecated in * - ``OCRMService`` - - ``4.10.0`` + - ``4.12.0`` - none - ``5.0.0`` - ``4.9.0`` * - ``DonutProcessor`` - - ``4.10.0`` + - ``4.12.0`` - none - ``5.0.0`` - ``4.9.0`` * - ``[ocr]`` extra (Donut, Tesseract, Pillow) - - ``4.10.0`` + - ``4.12.0`` - none - ``5.0.0`` - ``4.9.0`` * - ``[distributed]`` extra (PySpark, pandas) - - ``4.10.0`` + - ``4.12.0`` - :ref:`spark-recipe` - ``5.0.0`` - ``4.9.0`` * - ``detect()`` - - ``4.10.0`` + - ``4.12.0`` - ``scan()`` - ``5.0.0`` - ``4.9.0`` * - ``process()`` - - ``4.10.0`` + - ``4.12.0`` - ``redact()`` - ``5.0.0`` - ``4.9.0`` -Installing ``pip install "datafog==4.10.0"`` keeps every row above working. +After the 4.12.0 release is published, installing ``pip install "datafog==4.12.0"`` keeps every row above working. That version stays installable on PyPI permanently. OCR @@ -63,9 +63,9 @@ pin. .. code-block:: bash - pip install "datafog==4.10.0[ocr]" + pip install "datafog[ocr]==4.12.0" -If you need continued OCR support beyond that, stay on ``4.10.0``. Track the +If you need continued OCR support beyond that, stay on ``4.12.0``. Track the surfaces you depend on by name, not by staying on a major version range, so a lock file review shows the pin. diff --git a/docs/review-remediation.md b/docs/review-remediation.md new file mode 100644 index 00000000..2bcb8ddd --- /dev/null +++ b/docs/review-remediation.md @@ -0,0 +1,73 @@ +# November 2026 review remediation + +This candidate addresses the review of DataFog 4.11.0 and Core 1.4.0. The +review's suggested fixes were assessed against the local implementations; +regression tests cover the implemented behavior. + +## Observable behavior + +All engine modes and legacy scanning APIs share the structured scan boundary. +Optional NER adds spans with its own character offsets. TextService scans the +whole document; `text_chunk_length` remains validated for compatibility. +Unsupported labels, invalid ranges and invalid locales fail explicitly. Source +code-point offsets remain stable when lone surrogate characters are replaced +for native transport. `get_supported_entities()` reports supported labels, +including opt-in ones, rather than only currently active default labels. + +The `create_guardrail` decorator supports sync and async functions returning +strings. Non-string results raise TypeError; they cannot silently bypass the +filter. Apply a guardrail directly to the text fields of structured results. + +LiteLLM scans messages, nested tool/function arguments, prompt and input fields. +Image and binary parts are retained without OCR. Tokens are assigned per request +or response and repeated values reuse their token. Streamed response text and +tool arguments are buffered until completion, then scanned before any chunk is +emitted. This adds completion latency. Exceeding eight million text characters +or 100,000 chunks rejects the stream. Engine failures follow the explicit +`fail_policy`; deployments requiring protected egress should select `closed`. +The Claude Code hook requests review or denies oversized PreToolUse inputs; +nonblocking events receive an explicit incomplete-scan notice. Unexpected hook +failures retain the documented fail-open contract and log exception types only. + +`hash_type` selects md5, sha256 or sha3_256 on the CLI and legacy model APIs. +The engine also accepts algorithms provided by hashlib. Digests use full hex +lengths. Plain deterministic hashes are pseudonymization: SSNs, phones and +other guessable inputs can be enumerated. Use `hash_key` (at least 32 random +bytes) for keyed HMAC and keep the key outside logs and source control: + +```python +import secrets +import datafog +key = secrets.token_bytes(32) # store securely to reproduce pseudonyms +result = datafog.redact("mail jane@example.com", strategy="hash", hash_key=key) +``` + +## Release sequence and qualification + +The Python candidate is 4.12.0. Its standalone CI uses the published Core 1.4 +minor series, pinned with `~=1.4.0`. Core 2.0.0 is prepared separately because +safe mask-reveal and automatic-overlap transformation semantics are incompatible +changes under Core's version policy. Both use capability contract 1. Before +shipping the combined fixes, publish the qualified Core candidate, then raise +both Python dependency declarations to `~=2.0.0`, update the reviewed native +contract/corpus, and run Python's installed-wheel and release qualification +against the published artifacts. Source-candidate integration is additional +evidence; it does not substitute for that post-publication gate. + +Scheduled Python jobs only qualify candidates. Manual `release.yml` runs default +to `dry_run: true`; explicitly selecting false is the publication boundary. +Candidate bases must be newer than every stable Git tag and the latest stable +PyPI release. Development qualification refuses a ref missing `main` history. +The workflow atomically pushes the tested version commit and immutable tag +before uploading to PyPI, then fast-forwards `dev` after a stable publication. +The user must authorize deployment before any publishing workflow is dispatched. + +If registry upload fails after the atomic ref push, keep the tag. Inspect PyPI +for partial artifacts, compare hashes to the retained workflow artifact and tag, +and resume only the missing upload from those exact validated artifacts. Do not +move the tag, reuse the version for different bytes, or rerun version generation +with an existing release tag. A changed branch must be qualified again. + +Core's individual registries have separate availability; never infer that a +Python version is present on npm or crates.io. Native wheel coverage and source +build requirements are documented in Core's platform policy. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..cb600497 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=77", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/scripts/check_rust_install.py b/scripts/check_rust_install.py index bf84c45d..57604d3f 100644 --- a/scripts/check_rust_install.py +++ b/scripts/check_rust_install.py @@ -3,6 +3,8 @@ import importlib.metadata import sys +from packaging.requirements import Requirement + def main(): assert sys.flags.isolated, "Run with python -I" @@ -14,7 +16,20 @@ def main(): from datafog.compat import v4 installed = importlib.metadata.version("datafog-core") - assert installed.split(".")[0] == "1", installed + requirements = [ + Requirement(value) for value in importlib.metadata.requires("datafog") or [] + ] + core_requirements = [ + requirement + for requirement in requirements + if requirement.name == "datafog-core" and requirement.marker is None + ] + assert ( + core_requirements + ), "Installed DataFog must declare its Core compatibility range" + assert all( + installed in requirement.specifier for requirement in core_requirements + ), installed assert datafog_core.capabilities()["contract_version"] == 1 assert v4.Entity is datafog.Entity assert v5.Finding is datafog_core.Finding diff --git a/scripts/release_version.py b/scripts/release_version.py new file mode 100644 index 00000000..0ce2c585 --- /dev/null +++ b/scripts/release_version.py @@ -0,0 +1,68 @@ +"""Select a release version while refusing stale or already published bases.""" + +import argparse +import json +import re +import subprocess +import urllib.request +from pathlib import Path + +from packaging.version import Version + + +def select_version(current, release_type, tags, latest_stable, override=None): + if release_type not in {"alpha", "beta", "stable"}: + raise ValueError("Unknown release type") + source = Version(current) + base = override or source.base_version + if not re.fullmatch(r"\d+\.\d+\.\d+", base): + raise ValueError("Release base must be numeric major.minor.patch") + if Version(base) <= Version(latest_stable): + raise ValueError("Release base must be newer than the latest stable release") + if release_type == "stable": + candidate = base + else: + suffix = "a" if release_type == "alpha" else "b" + number = 1 + while f"v{base}{suffix}{number}" in tags: + number += 1 + candidate = f"{base}{suffix}{number}" + if f"v{candidate}" in tags: + raise ValueError( + "Release tag already exists; inspect publication state before retrying" + ) + return candidate + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--type", required=True, choices=["alpha", "beta", "stable"]) + parser.add_argument("--override", default="") + parser.add_argument( + "--latest-stable", help="Offline preflight only; workflow reads PyPI" + ) + args = parser.parse_args() + tags = subprocess.check_output(["git", "tag", "-l"], text=True).splitlines() + stable_tags = [tag[1:] for tag in tags if re.fullmatch(r"v\d+\.\d+\.\d+", tag)] + if args.latest_stable: + published = args.latest_stable + else: + with urllib.request.urlopen( + "https://pypi.org/pypi/datafog/json", timeout=30 + ) as response: + payload = json.load(response) + releases = [ + Version(version) + for version in payload["releases"] + if not Version(version).is_prerelease + ] + published = str(max(releases)) + latest = str(max(Version(version) for version in [published, *stable_tags])) + about = Path("datafog/__about__.py") + current = re.search(r'^__version__ = "([^"]+)"', about.read_text(), re.M).group(1) + candidate = select_version(current, args.type, tags, latest, args.override or None) + print(candidate) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 7ce0366d..bffb7ff5 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,13 @@ # Core lightweight dependencies only core_deps = [ - # Native engine, the default detection path. Exact upper bound: detection - # output is contract-versioned and must stay reproducible. - "datafog-core>=1.4.0,<2", - "pydantic>=2.0,<3.0", + # Core minor releases may change findings; stay on this reviewed minor. + "datafog-core~=1.4.0", + "pydantic>=2.1,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", + # Match timeouts bound untrusted allowlist expressions. + "regex>=2024.11.6", ] # Optional heavy dependencies @@ -34,7 +35,7 @@ nlp_advanced_deps = [ "gliner>=0.2.5", - "torch>=2.1.0,<2.7", + "torch>=2.1.0,<3", "transformers>=4.20.0", "huggingface-hub>=0.16.0", "sentencepiece>=0.2.0", @@ -88,9 +89,9 @@ extras_require = { # Retained for explicitness; the base install already pulls Core. - "rust": ["datafog-core>=1.4.0,<2"], + "rust": ["datafog-core~=1.4.0"], "nlp": nlp_deps, - "nlp-advanced": nlp_advanced_deps, + "nlp-advanced": nlp_deps + nlp_advanced_deps, "ocr": ocr_deps, "distributed": distributed_deps, "web": web_deps, @@ -117,6 +118,8 @@ version=version, author="Sid Mohan", author_email="sid@datafog.ai", + license="MIT", + license_files=["LICENSE"], description="Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)", long_description=long_description, long_description_content_type="text/markdown", @@ -127,14 +130,13 @@ python_requires=">=3.10", entry_points={ "console_scripts": [ - "datafog=datafog.client:app [cli]", # Requires cli extra + "datafog=datafog.cli:main", "datafog-hook=datafog.integrations.claude_code:main", # Core only ], }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/setup_lean.py b/setup_lean.py deleted file mode 100644 index 57315788..00000000 --- a/setup_lean.py +++ /dev/null @@ -1,109 +0,0 @@ -from setuptools import find_packages, setup - -# Historical shadow packaging snapshot. The live DataFog packaging input is -# setup.py. Do not use this file for builds, releases, or dependency changes. - -# Read README for the long description -with open("README.md", "r") as f: - long_description = f.read() - -# Use a single source of truth for the version -__version__ = "4.1.1" - -project_urls = { - "Homepage": "https://datafog.ai", - "Documentation": "https://docs.datafog.ai", - "Discord": "https://discord.gg/bzDth394R4", - "Twitter": "https://twitter.com/datafoginc", - "GitHub": "https://github.com/datafog/datafog-python", -} - -# Core lightweight dependencies only -core_deps = [ - "pydantic>=2.0,<3.0", - "typing-extensions>=4.0", -] - -# Optional heavy dependencies -extras_require = { - "nlp": [ - "spacy>=3.7.0,<4.0", - ], - "ocr": [ - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - ], - "distributed": [ - "pandas>=2.0.0", - "numpy>=1.24.0", - ], - "web": [ - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - ], - "cli": [ - "typer>=0.12.0", - ], - "crypto": [ - "cryptography>=40.0.0", - ], - "dev": [ - "pytest>=7.0.0", - "pytest-asyncio>=0.21.0", - "pytest-cov>=4.0.0", - "sphinx>=7.0.0", - ], - # Convenience bundles - "all": [ - "spacy>=3.7.0,<4.0", - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - "pandas>=2.0.0", - "numpy>=1.24.0", - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - "typer>=0.12.0", - "cryptography>=40.0.0", - ], -} - -setup( - name="datafog", - version=__version__, - author="Sid Mohan", - author_email="sid@datafog.ai", - description="Lightning-fast PII detection and anonymization library, 100x+ faster than NER-based detection (see benchmarks/)", - long_description=long_description, - long_description_content_type="text/markdown", - packages=find_packages(), - install_requires=core_deps, - extras_require=extras_require, - python_requires=">=3.10", - entry_points={ - "console_scripts": [ - "datafog=datafog.client:app [cli]", # Requires cli extra - ], - }, - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Text Processing", - "Topic :: Security", - ], - project_urls=project_urls, - keywords="pii detection anonymization privacy regex performance", -) diff --git a/setup_original.py b/setup_original.py deleted file mode 100644 index 006a3608..00000000 --- a/setup_original.py +++ /dev/null @@ -1,92 +0,0 @@ -from setuptools import find_packages, setup - -# Historical shadow packaging snapshot. The live DataFog packaging input is -# setup.py. Do not use this file for builds, releases, or dependency changes. - -# Read README for the long description -with open("README.md", "r") as f: - long_description = f.read() - -# Use a single source of truth for the version -__version__ = "4.1.1" - -project_urls = { - "Homepage": "https://datafog.ai", - "Documentation": "https://docs.datafog.ai", - "Discord": "https://discord.gg/bzDth394R4", - "Twitter": "https://twitter.com/datafoginc", - "GitHub": "https://github.com/datafog/datafog-python", -} - -setup( - name="datafog", - version=__version__, - author="Sid Mohan", - author_email="sid@datafog.ai", - description="Scan, redact, and manage PII in your documents before they get uploaded to a Retrieval Augmented Generation (RAG) system.", - long_description=long_description, - long_description_content_type="text/markdown", - packages=find_packages(), - install_requires=[ - "pandas", - "requests==2.32.3", - "spacy==3.7.5", - "pydantic", - "Pillow", - "sentencepiece", - "protobuf", - "pytesseract", - "aiohttp", - "pytest-asyncio", - "numpy", - "fastapi", - "asyncio", - "setuptools", - "pydantic-settings==2.3.4", - "typer==0.12.3", - "sphinx", - "cryptography", - ], - python_requires=">=3.10", - entry_points={ - "console_scripts": [ - "datafog=datafog.client:app", - ], - }, - classifiers=[ - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Framework :: tox", - "Framework :: Pytest", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "Intended Audience :: Information Technology", - "Intended Audience :: System Administrators", - ], - keywords="pii, redaction, nlp, rag, retrieval augmented generation", - maintainer="DataFog", - maintainer_email="hi@datafog.ai", - url="https://datafog.ai", - project_urls=project_urls, - license="MIT", - extras_require={ - "dev": [ - "just", - "isort", - "black", - "blacken-docs", - "flake8", - "tox", - "pytest", - "pytest-codeblocks", - "pytest-cov", - "build", - "twine", - ], - }, -) diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json index e4c0835b..b3ddf293 100644 --- a/tests/contracts/wrapper-4.12.0.json +++ b/tests/contracts/wrapper-4.12.0.json @@ -1,9 +1,6 @@ { "wrapper_version": "4.12.0", - "core_versions": [ - "1.4.0", - "2.0.0" - ], + "core_versions": ["1.4.0", "2.0.0"], "source_oracle": "4.8.1.json (unchanged)", "review": "2026-11-15 PDF remediation: explicit reviewed wrapper changes, not a regenerated historical oracle", "cases": { @@ -542,9 +539,7 @@ "reason": "PY-H3/PY-L6 route TextService through shared native detection and return populated canonical labels.", "expected": { "value": { - "EMAIL": [ - "alice@example.com" - ] + "EMAIL": ["alice@example.com"] }, "warnings": [] } @@ -554,9 +549,7 @@ "reason": "PY-H3/PY-L6 route TextService through shared native detection with overlap resolution and populated canonical labels.", "expected": { "value": { - "DE_TAX_ID": [ - "12345678901" - ] + "DE_TAX_ID": ["12345678901"] }, "warnings": [] } @@ -567,9 +560,7 @@ "expected": { "value": [ { - "EMAIL": [ - "alice@example.com" - ] + "EMAIL": ["alice@example.com"] }, {} ], @@ -920,14 +911,7 @@ "default": null } ], - "fields": [ - "type", - "text", - "start", - "end", - "confidence", - "engine" - ] + "fields": ["type", "text", "start", "end", "confidence", "engine"] }, "datafog.engine:ScanResult": { "parameters": [ @@ -950,11 +934,7 @@ "default": null } ], - "fields": [ - "entities", - "text", - "engine_used" - ] + "fields": ["entities", "text", "engine_used"] }, "datafog.engine:RedactResult": { "parameters": [ @@ -977,11 +957,7 @@ "default": null } ], - "fields": [ - "redacted_text", - "mapping", - "entities" - ] + "fields": ["redacted_text", "mapping", "entities"] }, "datafog.engine:scan": { "parameters": [ diff --git a/tests/test_client.py b/tests/test_client.py index d5b042d1..24734be1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -165,7 +165,7 @@ def test_anonymizer_outputs(): ), AnnotationResult( start=33, - end=41, + end=len(text), score=1.0, entity_type=EntityTypes.LOCATION, recognition_metadata=AnnotatorMetadata(), diff --git a/tests/test_donut_lazy_import.py b/tests/test_donut_lazy_import.py index 9b2a28f1..aab7bcfc 100644 --- a/tests/test_donut_lazy_import.py +++ b/tests/test_donut_lazy_import.py @@ -20,8 +20,7 @@ def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]: def test_no_torch_import_when_donut_disabled(): """Test that torch is not imported when use_donut is False""" - _run_isolated_python( - """ + _run_isolated_python(""" import sys from datafog.services.image_service import ImageService @@ -29,8 +28,7 @@ def test_no_torch_import_when_donut_disabled(): assert "torch" not in sys.modules assert "transformers" not in sys.modules -""" - ) +""") def test_lazy_import_mechanism(): @@ -39,8 +37,7 @@ def test_lazy_import_mechanism(): # to use lazy imports. We don't need to actually test the imports themselves, # just that the structure is correct. - _run_isolated_python( - """ + _run_isolated_python(""" import sys from datafog.processing.image_processing.donut_processor import DonutProcessor @@ -50,5 +47,4 @@ def test_lazy_import_mechanism(): assert "transformers" not in sys.modules assert hasattr(processor, "extract_text_from_image") assert not hasattr(processor, "ensure_installed") -""" - ) +""") diff --git a/tests/test_image_service.py b/tests/test_image_service.py index da8e6deb..ada09646 100644 --- a/tests/test_image_service.py +++ b/tests/test_image_service.py @@ -3,9 +3,10 @@ import asyncio import pytest -from PIL import Image -from datafog.services.image_service import ImageService +Image = pytest.importorskip("PIL.Image", reason="Image tests require datafog[ocr]") + +from datafog.services.image_service import ImageService # noqa: E402 urls = [ "https://www.pdffiller.com/preview/101/35/101035394.png", diff --git a/tests/test_legacy_retirement.py b/tests/test_legacy_retirement.py index f8be98f0..ed5910a0 100644 --- a/tests/test_legacy_retirement.py +++ b/tests/test_legacy_retirement.py @@ -31,7 +31,7 @@ def unexpected_import(*args): LegacySurfaceWarning, match=( f"{surface} support is deprecated in 4.9 and will be removed in 5.0. " - 'Last supported version is 4.10.0. Pin it with: pip install "datafog==4.10.0".' + 'Last supported version is 4.12.0. Pin it with: pip install "datafog==4.12.0".' ), ): (ImageService if surface == "OCR" else SparkService)() @@ -43,7 +43,7 @@ def unavailable(name): monkeypatch.setattr(importlib, "import_module", unavailable) with ( - pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"), + pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.12.0"), pytest.raises(ImportError, match=r"datafog\[distributed\]"), ): SparkService() @@ -85,7 +85,7 @@ def test_donut_mock_output_is_preserved(monkeypatch): monkeypatch.setattr(donut_processor, "DONUT_TESTING_ENABLED", False) with pytest.warns(LegacySurfaceWarning): processor = DonutProcessor() - with pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.10.0"): + with pytest.warns(LegacySurfaceWarning, match="Last supported version is 4.12.0"): assert asyncio.run(processor.extract_text_from_image(object())) == ( '{"text": "Mock OCR text for testing"}' ) @@ -245,7 +245,7 @@ def test_image_cli_notice_visible_under_default_python_filters(image_argument): args.append("image.png") result = CliRunner().invoke(client.app, args) assert "deprecated in 4.9 and will be removed in 5.0" in result.stderr, result.output -assert 'Last supported version is 4.10.0' in result.stderr, result.output +assert 'Last supported version is 4.12.0' in result.stderr, result.output if sys.argv[1] == "True": assert result.exit_code == 0, result.output assert "OCR Pipeline Results: ['OCR text']" in result.stdout diff --git a/tests/test_litellm_guardrail.py b/tests/test_litellm_guardrail.py index 1d7656a4..c4c1d697 100644 --- a/tests/test_litellm_guardrail.py +++ b/tests/test_litellm_guardrail.py @@ -152,13 +152,14 @@ async def test_post_call_fail_open_returns_unredacted_response(self, monkeypatch @pytest.mark.asyncio class TestEdgeShapes: - async def test_data_without_messages_passes_through(self): + async def test_embedding_input_is_redacted(self): guardrail = DataFogGuardrail(guardrail_name="datafog-pii") data = {"input": f"embed {EMAIL}"} result = await guardrail.async_pre_call_hook( user_api_key_dict=None, cache=None, data=data, call_type="aembedding" ) - assert result == data + assert EMAIL not in result["input"] + assert result["input"].startswith("embed [EMAIL_") async def test_message_without_content_key_passes_through(self): guardrail = DataFogGuardrail(guardrail_name="datafog-pii") diff --git a/tests/test_no_network_core.py b/tests/test_no_network_core.py index f06e8360..cbdf8c08 100644 --- a/tests/test_no_network_core.py +++ b/tests/test_no_network_core.py @@ -20,8 +20,7 @@ def _run_isolated_python(script: str) -> subprocess.CompletedProcess[str]: def test_import_scan_and_redact_do_not_open_network() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import socket import urllib.request @@ -39,8 +38,7 @@ def blocked(*_args, **_kwargs): redact_result = datafog.redact("Email jane@example.com or call 415-555-1212") assert "jane@example.com" not in redact_result.redacted_text assert "415-555-1212" not in redact_result.redacted_text -""" - ) +""") def test_core_defaults_do_not_initialize_optional_engines(monkeypatch) -> None: @@ -78,8 +76,7 @@ def fail_optional_engine_probe(): def test_import_probes_do_not_load_optional_models() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import sys import types @@ -105,13 +102,11 @@ def from_pretrained(*_args, **_kwargs): import datafog assert datafog.scan("Email jane@example.com").entities -""" - ) +""") def test_core_path_does_not_import_optional_dependency_modules() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import importlib.abc import sys @@ -144,5 +139,4 @@ def find_spec(self, fullname, path=None, target=None): assert datafog.scan_prompt("Email jane@example.com").entities assert datafog.filter_output("Email jane@example.com").redacted_text == "Email [EMAIL_1]" assert datafog.create_guardrail().filter("Email jane@example.com").redacted_text == "Email [EMAIL_1]" -""" - ) +""") diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py new file mode 100644 index 00000000..7833772a --- /dev/null +++ b/tests/test_review_entrypoints.py @@ -0,0 +1,229 @@ +"""Regression contracts for the November 2026 entrypoint review.""" + +import asyncio +import hashlib +import importlib.util +import logging +from pathlib import Path +from types import SimpleNamespace + +import pytest +from typer.testing import CliRunner + +import datafog +from datafog.agent import create_guardrail +from datafog.client import app +from datafog.config import OperationType, get_config +from datafog.main import DataFog, TextPIIAnnotator +from datafog.models.annotator import AnnotationResult, AnnotatorRequest +from datafog.models.anonymizer import Anonymizer, AnonymizerType, HashType +from datafog.models.common import EntityTypes + +EMAIL = "jane@" "example.com" + + +@pytest.mark.asyncio +async def test_decorator_awaits_and_filters(): + @create_guardrail() + async def answer(): + await asyncio.sleep(0) + return EMAIL + + assert EMAIL not in await answer() + + +@pytest.mark.parametrize("output", [[EMAIL], {"text": EMAIL}, None, 1]) +def test_guarded_non_strings_rejected(output): + @create_guardrail() + def answer(): + return output + + with pytest.raises(TypeError, match="must return a string"): + answer() + + +@pytest.mark.parametrize( + "operation", [OperationType.REDACT, OperationType.REPLACE, OperationType.HASH] +) +def test_anonymizing_operations_imply_scan(operation): + service = DataFog(operations=[operation]) + assert EMAIL not in service.run_text_pipeline_sync([EMAIL])[0] + assert EMAIL not in service.process_text(EMAIL) + + +@pytest.mark.parametrize("algorithm", list(HashType)) +def test_hash_algorithm_honored_across_api_and_cli(algorithm): + digest = hashlib.new(algorithm.value, EMAIL.encode()).hexdigest() + assert ( + digest + in DataFog(hash_type=algorithm).process(EMAIL, anonymize=True, method="hash")[ + "anonymized" + ] + ) + assert ( + digest + in datafog.process( + EMAIL, anonymize=True, method="hash", hash_type=algorithm.value + )["anonymized"] + ) + result = CliRunner().invoke( + app, ["hash-text", EMAIL, "--hash-type", algorithm.value] + ) + assert result.exit_code == 0, result.output + assert digest in result.stdout + + +def test_keyed_hash_public_facades(): + import hmac + + key = b"k" * 32 + digest = hmac.new(key, EMAIL.encode(), "sha256").hexdigest() + assert digest in datafog.redact(EMAIL, strategy="hash", hash_key=key).redacted_text + assert digest in datafog.anonymize_text(EMAIL, method="hash", hash_key=key) + assert ( + digest + in datafog.process(EMAIL, anonymize=True, method="hash", hash_key=key)[ + "anonymized" + ] + ) + + +@pytest.mark.parametrize( + "factory", [lambda: datafog.process, lambda: DataFog().process] +) +def test_bad_method_rejected(factory): + with pytest.raises(ValueError, match="method"): + factory()(EMAIL, anonymize=True, method="bogus") + + +def test_legacy_detection_uses_core(): + text = "card 6011111111111117 ip 2001:db8::1" + assert {item["type"] for item in datafog.detect(text)} >= { + "CREDIT_CARD", + "IP_ADDRESS", + } + assert "6011111111111117" not in datafog.process(text, anonymize=True)["anonymized"] + assert "IP_ADDRESS" in TextPIIAnnotator().run(text) + assert "JWT" in datafog.get_supported_entities() + + +def test_annotation_defaults_and_labels(): + assert AnnotatorRequest(text="x", language="en").score_threshold is None + for label in ["SSN", "ORG", "GPE", "IP_ADDRESS"]: + assert ( + AnnotationResult(start=0, end=1, entity_type=label).entity_type != "UNKNOWN" + ) + + +def test_enum_filter_and_date_replacement(): + annotation = AnnotationResult( + start=0, end=len(EMAIL), entity_type=EntityTypes.EMAIL + ) + assert ( + Anonymizer(anonymizer_type="redact", entities=[EntityTypes.EMAIL]) + .anonymize(EMAIL, [annotation]) + .anonymized_text + == "[REDACTED]" + ) + date = AnnotationResult(start=0, end=10, entity_type="DATE") + assert ( + Anonymizer().anonymize("2024-01-02", [date]).anonymized_text + == "[REDACTED_DATE]" + ) + + +@pytest.mark.parametrize("strategy", list(AnonymizerType)) +def test_legacy_overlaps_never_leave_partial_value(strategy): + text = "abcdefghij" + spans = [ + AnnotationResult(start=0, end=6, entity_type="EMAIL"), + AnnotationResult(start=4, end=10, entity_type="SSN"), + ] + result = Anonymizer(anonymizer_type=strategy).anonymize(text, spans) + assert "ghij" not in result.anonymized_text + assert len(result.anonymized_entities) == 1 + assert result.anonymized_entities[0]["original"] == text + + +def test_cli_hides_key_and_handles_bad_operation(): + config = get_config() + old_key = config.api_key + try: + config.api_key = "private-test-key" + result = CliRunner().invoke(app, ["show-config"]) + assert result.exit_code == 0 + assert "private-test-key" not in result.stdout + assert "REDACTED" in result.stdout + finally: + config.api_key = old_key + result = CliRunner().invoke(app, ["scan-text", EMAIL, "--operations", "bogus"]) + assert result.exit_code == 2 + assert "operations must use" in result.output + assert "Traceback" not in result.output + + +def test_import_and_cli_preserve_logger_level(): + logger = logging.getLogger("datafog_logger") + old = logger.level + try: + logger.setLevel(logging.ERROR) + import importlib + + import datafog.main + + importlib.reload(datafog.main) + assert logger.level == logging.ERROR + finally: + logger.setLevel(old) + + +def test_spark_real_spacy_labels_and_oversize(monkeypatch): + from datafog.processing.spark_processing import pyspark_udfs as spark + + monkeypatch.setattr(spark, "ensure_installed", lambda _: None) + + def nlp(text): + return SimpleNamespace( + ents=[ + SimpleNamespace(label_="PERSON", text="Ann"), + SimpleNamespace(label_="ORG", text="Acme"), + ] + ) + + with pytest.warns(FutureWarning): + result = spark.pii_annotator("Ann at Acme", SimpleNamespace(value=nlp)) + assert result[spark.PII_ANNOTATION_LABELS.index("PER")] == ["Ann"] + assert result[spark.PII_ANNOTATION_LABELS.index("ORG")] == ["Acme"] + with pytest.warns(FutureWarning), pytest.raises(ValueError, match="maximum"): + spark.pii_annotator( + "x" * (spark.MAXIMAL_STRING_SIZE + 1), SimpleNamespace(value=nlp) + ) + + +def _version_selector(): + path = Path(__file__).parents[1] / "scripts/release_version.py" + spec = importlib.util.spec_from_file_location("release_version", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.select_version + + +@pytest.mark.parametrize( + "current, kind, override", + [ + ("4.9.0a2", "alpha", None), + ("4.11.0", "beta", None), + ("4.12.0", "stable", "4.10.0"), + ], +) +def test_release_rejects_stale_bases(current, kind, override): + with pytest.raises(ValueError, match="newer than"): + _version_selector()(current, kind, [], "4.11.0", override) + + +def test_release_version_positive_and_duplicate_paths(): + select = _version_selector() + assert select("4.12.0", "alpha", ["v4.12.0a1"], "4.11.0") == "4.12.0a2" + assert select("4.12.0", "stable", [], "4.11.0") == "4.12.0" + with pytest.raises(ValueError, match="already exists"): + select("4.12.0", "stable", ["v4.12.0"], "4.11.0") diff --git a/tests/test_runtime_dependency_safety.py b/tests/test_runtime_dependency_safety.py index d34ceb8a..b311fff4 100644 --- a/tests/test_runtime_dependency_safety.py +++ b/tests/test_runtime_dependency_safety.py @@ -42,8 +42,7 @@ def test_runtime_code_does_not_install_packages() -> None: def test_ocr_and_spark_public_services_do_not_require_optional_imports() -> None: - _run_isolated_python( - """ + _run_isolated_python(""" import importlib.abc import sys @@ -74,8 +73,7 @@ def find_spec(self, fullname, path=None, target=None): assert TextService is not None assert datafog.ImageService is ImageService assert datafog.SparkService is SparkService -""" - ) +""") def test_spacy_pii_missing_model_requires_explicit_download( diff --git a/tox.ini b/tox.ini index 83d3f793..a7a1565c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py310,py311,py312 +envlist = py310,py311,py312,py313,py314 isolated_build = True [testenv:integration] From dd01fa9ee9068ce44ad0ba80997466f16aad08b5 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:03:11 -0800 Subject: [PATCH 07/20] docs: align contributor release guidance with safe qualification --- AGENTS.md | 14 +++++++------- VERSIONING.md | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 13439692..871e8012 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,9 +13,9 @@ ## Current Project Status -**Stable version: 4.9.0** +**Stable version: 4.11.0** -**Development version: 4.9.0** +**Development candidate: 4.12.0** **Next major target: 5.0.0** @@ -242,14 +242,14 @@ except ImportError: | Workflow | Purpose | Trigger | | --------------- | ----------------------------------- | -------------------------- | | `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | -| `release.yml` | Alpha/Beta/Stable publishing | Schedule + manual dispatch | +| `release.yml` | Candidate qualification and publishing | Schedule validates; manual dispatch publishes | | `benchmark.yml` | Performance benchmarks | Push/PR/weekly | ### Release Cadence -- **Alpha** (Mon-Wed 2AM UTC): Automatic from `dev`, date+commit versioning -- **Beta** (Thursday 2AM UTC): Automatic from `dev`, incremental beta numbers -- **Stable** (manual dispatch): From `main`, base version or override +- **Alpha** (Mon-Wed 2AM UTC): Candidate qualification from `dev`, without publication +- **Beta** (Thursday 2AM UTC): Candidate qualification from `dev`, without publication +- **Publication** (manual dispatch): Dry run by default; explicit non-dry runs publish alpha/beta from `dev` or stable from `main` ### Release Pipeline @@ -320,7 +320,7 @@ Before beginning any task please checkout a branch from `dev` and create a pull ### Release Preparation -- Alpha/beta releases are automated via `release.yml` schedule +- Alpha/beta candidate qualification is automated via `release.yml`; publication requires an explicit manual dispatch - Stable releases: merge `dev` → `main`, then trigger `release.yml` with `stable` type - Use `dry_run: true` to validate before actual publish - Performance validation on realistic data sets diff --git a/VERSIONING.md b/VERSIONING.md index 9f511c41..802af079 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -96,9 +96,9 @@ for `datafog-core`, and requires a matching floor in `datafog`. Both are exposed separately so a bug report is attributable: ```python -import datafog, datafog.core +import datafog, datafog.native datafog.__version__ # application version -datafog.core.__version__ # engine version +datafog.native.version() # engine version ``` ## Yanking From bd42280c063c197d12d00af755d0cd7a8bcbd27a Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:04:39 -0800 Subject: [PATCH 08/20] fix(guardrails): enforce matching stream group counts --- datafog/integrations/litellm_guardrail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafog/integrations/litellm_guardrail.py b/datafog/integrations/litellm_guardrail.py index 5611bec9..74a56169 100644 --- a/datafog/integrations/litellm_guardrail.py +++ b/datafog/integrations/litellm_guardrail.py @@ -404,7 +404,7 @@ async def async_post_call_streaming_iterator_hook( ] session = _RedactionSession(texts) changes = [] - for group, text in zip(groups.values(), texts): + for group, text in zip(groups.values(), texts, strict=True): redacted, _ = self._process_content(text, session) changes.extend( (target, field, redacted if i == 0 else "") From 2cd5127c8ec96c2bdac6d35d80187dce727545b0 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:05:09 -0800 Subject: [PATCH 09/20] fix(release): account for untagged publications and preserve release target --- .github/workflows/release.yml | 1 + scripts/release_version.py | 15 +++++++++++---- tests/test_review_entrypoints.py | 5 +++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32952a23..0e2b8c63 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -382,6 +382,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_PAT }} VERSION: ${{ steps.version.outputs.version }} TYPE: ${{ needs.determine-release.outputs.release_type }} + BRANCH: ${{ needs.determine-release.outputs.target_branch }} run: | PRERELEASE_FLAG="" TITLE="" diff --git a/scripts/release_version.py b/scripts/release_version.py index 0ce2c585..f78afd28 100644 --- a/scripts/release_version.py +++ b/scripts/release_version.py @@ -10,10 +10,13 @@ from packaging.version import Version -def select_version(current, release_type, tags, latest_stable, override=None): +def select_version( + current, release_type, tags, latest_stable, override=None, published=() +): if release_type not in {"alpha", "beta", "stable"}: raise ValueError("Unknown release type") source = Version(current) + occupied = set(tags) | {f"v{Version(version)}" for version in published} base = override or source.base_version if not re.fullmatch(r"\d+\.\d+\.\d+", base): raise ValueError("Release base must be numeric major.minor.patch") @@ -24,10 +27,10 @@ def select_version(current, release_type, tags, latest_stable, override=None): else: suffix = "a" if release_type == "alpha" else "b" number = 1 - while f"v{base}{suffix}{number}" in tags: + while f"v{base}{suffix}{number}" in occupied: number += 1 candidate = f"{base}{suffix}{number}" - if f"v{candidate}" in tags: + if f"v{candidate}" in occupied: raise ValueError( "Release tag already exists; inspect publication state before retrying" ) @@ -46,6 +49,7 @@ def main(): stable_tags = [tag[1:] for tag in tags if re.fullmatch(r"v\d+\.\d+\.\d+", tag)] if args.latest_stable: published = args.latest_stable + published_versions = [published] else: with urllib.request.urlopen( "https://pypi.org/pypi/datafog/json", timeout=30 @@ -57,10 +61,13 @@ def main(): if not Version(version).is_prerelease ] published = str(max(releases)) + published_versions = list(payload["releases"]) latest = str(max(Version(version) for version in [published, *stable_tags])) about = Path("datafog/__about__.py") current = re.search(r'^__version__ = "([^"]+)"', about.read_text(), re.M).group(1) - candidate = select_version(current, args.type, tags, latest, args.override or None) + candidate = select_version( + current, args.type, tags, latest, args.override or None, published_versions + ) print(candidate) diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py index 7833772a..d05c40ab 100644 --- a/tests/test_review_entrypoints.py +++ b/tests/test_review_entrypoints.py @@ -227,3 +227,8 @@ def test_release_version_positive_and_duplicate_paths(): assert select("4.12.0", "stable", [], "4.11.0") == "4.12.0" with pytest.raises(ValueError, match="already exists"): select("4.12.0", "stable", ["v4.12.0"], "4.11.0") + + +def test_release_skips_published_prereleases_without_tags(): + select = _version_selector() + assert select("4.12.0", "alpha", [], "4.11.0", published=["4.12.0a1"]) == "4.12.0a2" From c6824095ddd91820cb64709d6ed047abded662d0 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:05:34 -0800 Subject: [PATCH 10/20] style: format updated release guidance --- AGENTS.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 871e8012..921d2d55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -239,11 +239,11 @@ except ImportError: ### Workflow Architecture (3 workflows) -| Workflow | Purpose | Trigger | -| --------------- | ----------------------------------- | -------------------------- | -| `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | +| Workflow | Purpose | Trigger | +| --------------- | -------------------------------------- | --------------------------------------------- | +| `ci.yml` | Lint + Test + Coverage + Wheel size | Push/PR to main/dev | | `release.yml` | Candidate qualification and publishing | Schedule validates; manual dispatch publishes | -| `benchmark.yml` | Performance benchmarks | Push/PR/weekly | +| `benchmark.yml` | Performance benchmarks | Push/PR/weekly | ### Release Cadence From 0eaa6e7d3e9f682d0263c6f6bae69171ec7b9f55 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:09:42 -0800 Subject: [PATCH 11/20] fix(ci): provision benchmark models and validate current annotation contracts --- .github/workflows/benchmark.yml | 9 +-- tests/benchmark_text_service.py | 4 +- tests/simple_performance_test.py | 2 +- tests/test_gliner_annotator.py | 114 +++++++++++++++---------------- tests/test_image_service.py | 4 +- tests/test_ocr_integration.py | 3 +- 6 files changed, 64 insertions(+), 72 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index bcc51e1b..944cd756 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,6 +27,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[nlp]" + python -m spacy download en_core_web_lg pip install -r requirements-benchmark.txt # Verify pytest-benchmark is installed and working python -c "import pytest_benchmark; print('pytest-benchmark version:', pytest_benchmark.__version__)" @@ -54,13 +55,7 @@ jobs: # Run benchmarks with optimal performance settings (no memory debugging) echo "Running benchmarks with performance-optimized settings..." - # Try pytest-benchmark first - if python -m pytest tests/benchmark_text_service.py -v --benchmark-autosave --benchmark-json=benchmark-results.json --tb=short -p no:cacheprovider; then - echo "✅ pytest-benchmark tests completed successfully" - else - echo "⚠️ pytest-benchmark failed, running simple performance test as fallback" - python tests/simple_performance_test.py - fi + python -m pytest tests/benchmark_text_service.py -v --benchmark-autosave --benchmark-json=benchmark-results.json --tb=short -p no:cacheprovider - name: Check for performance regression run: | diff --git a/tests/benchmark_text_service.py b/tests/benchmark_text_service.py index 3178c61e..e802c40e 100644 --- a/tests/benchmark_text_service.py +++ b/tests/benchmark_text_service.py @@ -107,7 +107,7 @@ def test_spacy_performance(benchmark, sample_text_10kb, spacy_service): # Verify spaCy found expected entities assert "PERSON" in result or "PER" in result - assert "ORG" in result + assert "ORGANIZATION" in result # Print some stats about the results entity_counts = {key: len(values) for key, values in result.items() if values} @@ -178,7 +178,7 @@ def test_auto_engine_fallback_performance(benchmark, spacy_only_text, auto_servi ) # Check if we have spaCy entities (depends on spaCy availability) - spacy_entities = ["PERSON", "ORG", "GPE", "CARDINAL", "DATE", "TIME"] + spacy_entities = ["PERSON", "ORGANIZATION", "LOCATION", "ADDRESS"] has_spacy_entities = any(entity in result for entity in spacy_entities) # If no spaCy entities, check if spaCy is available diff --git a/tests/simple_performance_test.py b/tests/simple_performance_test.py index 6da82d88..589ef59b 100644 --- a/tests/simple_performance_test.py +++ b/tests/simple_performance_test.py @@ -90,7 +90,7 @@ def test_simple_spacy_performance(): # Verify functionality assert "PERSON" in stats["result"] or "PER" in stats["result"] - assert "ORG" in stats["result"] + assert "ORGANIZATION" in stats["result"] return stats diff --git a/tests/test_gliner_annotator.py b/tests/test_gliner_annotator.py index fbf0bbdf..b9a82ad5 100644 --- a/tests/test_gliner_annotator.py +++ b/tests/test_gliner_annotator.py @@ -11,8 +11,8 @@ import pytest -@pytest.fixture(scope="session", autouse=True) -def mock_gliner_module(): +@pytest.fixture(autouse=True) +def mock_gliner_module(monkeypatch): """Mock the gliner module for all tests.""" # Create a mock gliner module mock_gliner = MagicMock() @@ -20,22 +20,42 @@ def mock_gliner_module(): # Configure the mock model behavior mock_model = MagicMock() - mock_model.predict_entities.return_value = [ - {"label": "person", "text": "John Doe", "start": 0, "end": 8}, - {"label": "email", "text": "john@example.com", "start": 20, "end": 36}, - {"label": "phone number", "text": "555-123-4567", "start": 50, "end": 62}, - ] + + def predict(text, labels): + findings = [] + for label, value in [ + ("person", "John Doe"), + ("email", "john@example.com"), + ("phone number", "555-123-4567"), + ]: + start = text.find(value) + if start >= 0 and label in labels: + findings.append( + { + "label": label, + "text": value, + "start": start, + "end": start + len(value), + } + ) + return findings + + mock_model.predict_entities.side_effect = predict mock_gliner_class.from_pretrained.return_value = mock_model mock_gliner.GLiNER = mock_gliner_class - # Add to sys.modules to make import work - sys.modules["gliner"] = mock_gliner + from datafog import engine + gliner_factory, spacy_factory = ( + engine._get_gliner_annotator, + engine._get_spacy_annotator, + ) + gliner_factory.cache_clear() + spacy_factory.cache_clear() + monkeypatch.setitem(sys.modules, "gliner", mock_gliner) yield mock_gliner_class, mock_model - - # Cleanup - if "gliner" in sys.modules: - del sys.modules["gliner"] + gliner_factory.cache_clear() + spacy_factory.cache_clear() class TestGLiNERAnnotatorWithDependencies: @@ -119,15 +139,9 @@ def test_gliner_annotate_long_text(self, mock_gliner_module): # Create text longer than MAXIMAL_STRING_SIZE long_text = "A" * 1000001 # Exceeds 1M character limit - with patch( - "datafog.processing.text_processing.gliner_annotator.logging" - ) as mock_logging: - result = annotator.annotate(long_text) - - # Should log a warning about truncation - mock_logging.warning.assert_called_once() - # Should still return a valid result - assert isinstance(result, dict) + with pytest.raises(ValueError, match="exceeds 1000000"): + annotator.annotate(long_text) + mock_model.predict_entities.assert_not_called() def test_gliner_download_model(self, mock_gliner_module): """Test GLiNER model download functionality.""" @@ -326,21 +340,19 @@ def test_text_service_gliner_engine_without_dependencies(self): ): TextService(engine="gliner") - def test_text_service_smart_engine_without_dependencies(self): - """Test smart engine degrades gracefully when GLiNER dependencies are missing.""" + def test_text_service_smart_engine_without_dependencies(self, monkeypatch): + """Missing optional engines preserve the structured native layer.""" + from datafog import engine from datafog.services.text_service import TextService - with patch.object(TextService, "_create_gliner_annotator", return_value=None): - with patch.object( - TextService, "_create_spacy_annotator", return_value=None - ): - service = TextService(engine="smart") - with pytest.warns(UserWarning, match="GLiNER not available"): - result = service.annotate_text_sync( - "John Doe from Acme Corporation needs follow up." - ) - assert "EMAIL" in result - assert result["EMAIL"] == [] + unavailable = engine._UnavailableAnnotator("optional model unavailable") + monkeypatch.setattr(engine, "_get_gliner_annotator", lambda: unavailable) + monkeypatch.setattr(engine, "_get_spacy_annotator", lambda: unavailable) + with pytest.warns(UserWarning, match="GLiNER unavailable"): + result = TextService(engine="smart").annotate_text_sync( + "mail john@example.com" + ) + assert result == {"EMAIL": ["john@example.com"]} def test_text_service_valid_engines(self): """Test that all valid engines are accepted.""" @@ -382,10 +394,10 @@ def test_text_service_valid_engines(self): assert service.engine == engine def test_text_service_invalid_engine(self): - """Test that invalid engines raise AssertionError.""" + """Test that invalid engines raise a configuration error.""" from datafog.services.text_service import TextService - with pytest.raises(AssertionError, match="Invalid engine"): + with pytest.raises(ValueError, match="engine must be one of"): TextService(engine="invalid_engine") @pytest.mark.parametrize( @@ -414,30 +426,14 @@ def test_cascade_should_stop_logic(self, engine, expected_count): result_above_threshold = {"TYPE1": ["entity1"] * (expected_count + 1)} assert service._cascade_should_stop(engine, result_above_threshold) - def test_smart_cascade_flow(self, mock_gliner_annotator): - """Test the smart cascading flow.""" + def test_smart_engine_combines_gliner_with_structured_detection(self): + """GLiNER adds exact NER spans without removing structured findings.""" from datafog.services.text_service import TextService - # Inject annotators directly to keep this cascade test deterministic - # across Python versions and import ordering. - mock_regex = Mock() - mock_regex.annotate.return_value = {"EMAIL": []} - - mock_spacy = Mock() - mock_spacy.annotate.return_value = {"PERSON": ["John Doe"]} - - service = TextService(engine="smart") - service._regex_annotator = mock_regex - service._gliner_annotator = mock_gliner_annotator - service._gliner_import_attempted = True - service._spacy_annotator = mock_spacy - service._spacy_import_attempted = True - - service.annotate_text_sync("John Doe works at john@example.com") - - # Should have tried regex first, then GLiNER. - mock_regex.annotate.assert_called_once() - mock_gliner_annotator.annotate.assert_called_once() + text = "John Doe owns 6011111111111117 and john@example.com" + result = TextService(engine="smart").annotate_text_sync(text, structured=True) + assert {item.label for item in result} == {"PERSON", "CREDIT_CARD", "EMAIL"} + assert all(text[item.start : item.end] == item.text for item in result) # Test CLI updates as well diff --git a/tests/test_image_service.py b/tests/test_image_service.py index ada09646..03d472b1 100644 --- a/tests/test_image_service.py +++ b/tests/test_image_service.py @@ -4,9 +4,9 @@ import pytest -Image = pytest.importorskip("PIL.Image", reason="Image tests require datafog[ocr]") +from datafog.services.image_service import ImageService -from datafog.services.image_service import ImageService # noqa: E402 +Image = pytest.importorskip("PIL.Image", reason="Image tests require datafog[ocr]") urls = [ "https://www.pdffiller.com/preview/101/35/101035394.png", diff --git a/tests/test_ocr_integration.py b/tests/test_ocr_integration.py index 9ceef4a2..c7333dbd 100644 --- a/tests/test_ocr_integration.py +++ b/tests/test_ocr_integration.py @@ -9,11 +9,12 @@ from unittest.mock import patch import pytest -from PIL import Image from datafog.processing.image_processing.donut_processor import DonutProcessor from datafog.services.image_service import ImageService +Image = pytest.importorskip("PIL.Image", reason="OCR tests require datafog[ocr]") + # Mark all tests in this file as integration tests pytestmark = pytest.mark.integration From ad36d6a9aa7857bd282b1c4fade793d1b1e8db98 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:11:14 -0800 Subject: [PATCH 12/20] docs: record every review finding and its regression proof --- docs/review-remediation.md | 44 ++++++++++++++++++++++++++++++++ tests/test_review_entrypoints.py | 15 +++++++++++ 2 files changed, 59 insertions(+) diff --git a/docs/review-remediation.md b/docs/review-remediation.md index 2bcb8ddd..86e37cc4 100644 --- a/docs/review-remediation.md +++ b/docs/review-remediation.md @@ -71,3 +71,47 @@ with an existing release tag. A changed branch must be qualified again. Core's individual registries have separate availability; never infer that a Python version is present on npm or crates.io. Native wheel coverage and source build requirements are documented in Core's platform policy. + +## Python finding disposition + +All 35 Python findings have a concrete resolution below. Core-owned fixes are +separately tracked in Core's `docs/reviews/2026-11-core-findings.md`; their combined +release still requires the Core publication and Python dependency update above. + +| Finding | Resolution and proof | +| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| PY-H1 | Replace lone surrogates only for native transport, preserving source coordinates. Engine and hook/LiteLLM block regressions cover mixed payloads. | +| PY-H2 | Interval queries replace pairwise overlap checks; redaction and legacy assembly are linear after sorting. Dense scan/redaction benchmarks and overlap tests cover the change. | +| PY-H3 | TextService scans complete documents through the shared engine. Boundary-crossing email/SSN tests pass. | +| PY-H4 | spaCy/GLiNER return actual character spans. Stub and real-spaCy tests distinguish Ann from Annual and repeated names. | +| PY-H5 | Every engine and legacy/CLI path uses the same structured layer before optional NER. Discover/IPv6/credential tests cover the paths. | +| PY-M1 | Coroutine decorators await results; non-string results raise a documented TypeError rather than silently escaping inspection. | +| PY-M2 | Redact, replace and hash pipeline operations imply scanning. All three are tested without an explicit SCAN operation. | +| PY-M3 | Fallback email consumes each lexical run once; German separator expressions avoid ambiguous repetition. Adversarial 80K-character measurements and positive matches pass. | +| PY-M4 | Native backend selection also supplies the structured stage of smart/spaCy/GLiNER. CLI commands work with DATAFOG_BACKEND=rust. | +| PY-M5 | Request-scoped tokens, nested tool/function arguments, prompt/input fields and buffered streaming protect all tested request/response shapes. Live proxy deployment is not claimed. | +| PY-M6 | Bare strings, invalid labels and labels unavailable to the selected backend/engine raise explicit errors. Supported aliases remain accepted. | +| PY-M7 | APIs and CLI honor hash_type, use full digests and support keyed HMAC. Plain-hash guessing limitations are documented; algorithm/key tests pass. | +| PY-M8 | Hook per-string/total budget exhaustion emits review/deny or an incomplete-scan notice. Tests put PII past the cap. | +| PY-M9 | Allowlist expressions run with match timeouts; timed-out expressions retain findings. Adversarial patterns and safe quantified groups are tested. | +| PY-M10 | Dependency resolution stays within the explicitly reviewed Core minor, replacing the broad less-than-2 range. Exact output locks should still pin patch versions. | +| PY-M11 | Reconcile stable history, reject stale/PyPI-occupied versions, gate 3.13, sync version metadata and push immutable refs before upload. Scheduled runs are nonpublishing; selector tests and actionlint pass. | +| PY-M12 | Spark maps ordinary PERSON/ORG/GPE/NORP/DATE labels into its fixed output schema. Stub tests cover real spaCy label spellings and oversized input errors. | +| PY-M13 | Optional Pydantic fields have defaults, labels use canonical names, enum filters work and overlaps merge before replacement. Model/anonymizer regressions pass. | +| PY-M14 | Preserve supported GLiNER labels, use exact offsets, exclude relative spaCy DATE mentions, propagate model failures and reject oversized annotator input explicitly. | +| PY-M15 | Embedded model calls use bounded overlapping chunks. Large PERSON/ADDRESS plus structured-PII tests succeed with published and candidate Core. Native model limits remain enforced. | +| PY-M16 | Core 2 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | +| PY-L1 | Console bootstrap prints the CLI-extra install hint on a base install. A clean installed-wheel check confirms no traceback. | +| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 2 suppresses the labelled-tax-ID PHONE false positive. | +| PY-L3 | Retirement notices derive the current supported release version, and the native install hint no longer names an older release. | +| PY-L4 | Supported-entity reporting follows the selected native/Python capability inventory, including opt-in labels. Tests separate supported from default-active labels. | +| PY-L5 | Document actual token/pseudonym outputs and reject invalid methods in both process APIs. | +| PY-L6 | Legacy APIs and TextPIIAnnotator route through the shared engine; service/engine integration covers native-only entities and locale behavior. Explicit Python fallback remains a narrower documented detector set. | +| PY-L7 | Tokens reuse values and avoid existing markers; invalid explicit ranges/text raise errors. Collision, repeated-value and validation tests pass. | +| PY-L8 | Remove unused implementation copies, declare an isolated build, include test packages/corpora/scripts in sdists, align formatter pins and extend tox through 3.14. Optional OCR imports skip cleanly when absent. | +| PY-L9 | CLI configuration hides API keys; import and commands preserve host logging levels. CLI/logger regressions pass. | +| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 2 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | +| PY-L11 | TextService validates positive chunk lengths and engine names with explicit exceptions, including optimized Python execution. | +| PY-L12 | Remove incompatible Torch upper bound and include spaCy in nlp-advanced. Resolving it with Core GLiNER2 succeeds with Torch 2.12.0. | +| PY-L13 | Declare Pydantic 2.1 minimum. A clean Python 3.10 base install with Pydantic 2.1 imports and scans successfully. | +| PY-L14 | Correct native-default documentation, normalize missing native metadata to ImportError, clarify missing-model hints, validate locales consistently and surface invalid CLI operations as usage errors. | diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py index d05c40ab..fb7cc555 100644 --- a/tests/test_review_entrypoints.py +++ b/tests/test_review_entrypoints.py @@ -232,3 +232,18 @@ def test_release_version_positive_and_duplicate_paths(): def test_release_skips_published_prereleases_without_tags(): select = _version_selector() assert select("4.12.0", "alpha", [], "4.11.0", published=["4.12.0a1"]) == "4.12.0a2" + + +def test_native_version_uses_import_error_when_metadata_is_missing(monkeypatch): + import importlib.metadata + + import datafog.native + + assert datafog.native.version() + + def missing(_): + raise importlib.metadata.PackageNotFoundError("datafog-core") + + monkeypatch.setattr(importlib.metadata, "version", missing) + with pytest.raises(ImportError, match="metadata is missing"): + datafog.native.version() From fb262c8dcc1888ab0141bbea91a446c7508dafe3 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:19:36 -0800 Subject: [PATCH 13/20] fix(hash): preserve existing pseudonyms while honoring algorithms and HMAC --- CHANGELOG.MD | 7 +- datafog/__init__.py | 41 +++---- datafog/engine.py | 31 +++++- docs/review-remediation.md | 12 +- tests/contracts/wrapper-4.12.0.json | 163 +--------------------------- tests/test_review_engine.py | 17 ++- tests/test_review_entrypoints.py | 21 +++- 7 files changed, 96 insertions(+), 196 deletions(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 6bf46b28..ab2e41b6 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -16,9 +16,10 @@ #### Changed -- Hash APIs honor the selected algorithm and emit full digests. Optional - `hash_key` enables HMAC with a key of at least 32 bytes. Plain hashes provide - deterministic pseudonyms and do not resist guessing of low-entropy PII. +- Hash APIs honor the selected algorithm while preserving existing plain-hash + marker widths and defaults. Optional `hash_key` enables full-length HMAC with + a key of at least 32 bytes. Plain hashes provide deterministic pseudonyms and + do not resist guessing of low-entropy PII. - Reuse tokens for repeated values, avoid existing token collisions, reject invalid explicit ranges and unsupported entity selections, and report backend entity capabilities. Async decorators await results before filtering. diff --git a/datafog/__init__.py b/datafog/__init__.py index f9a1695e..467f5a70 100644 --- a/datafog/__init__.py +++ b/datafog/__init__.py @@ -290,7 +290,7 @@ def process( anonymize: bool = False, method: str = "redact", *, - hash_type: str = "sha256", + hash_type: str | None = None, hash_key: bytes | None = None, ) -> dict: """ @@ -300,6 +300,9 @@ def process( text: Input text to process anonymize: Whether to anonymize detected PII method: Anonymization method ('redact', 'replace', 'hash') + hash_type: Optional algorithm. The legacy plain-hash default is MD5; + keyed hashes default to SHA256. + hash_key: At least 32 bytes for a full keyed HMAC digest. Returns: Dictionary with original text, anonymized text (if requested), and findings @@ -321,32 +324,32 @@ def process( if method not in {"redact", "replace", "hash"}: raise ValueError("method must be 'redact', 'replace', or 'hash'") + if anonymize and method == "hash": + from .engine import _hash_value, _validate_hash_parameters + + algorithm = ( + hash_type + if hash_type is not None + else ("sha256" if hash_key is not None else "md5") + ) + _validate_hash_parameters("hash", algorithm, hash_key) findings = _detect_impl(text) result = {"original": text, "findings": findings} - if anonymize and method == "hash": - entities = [ - Entity( - item["type"], item["value"], item["start"], item["end"], 1.0, "regex" - ) - for item in findings - ] - result["anonymized"] = redact( - text, - entities=entities, - strategy="hash", - hash_type=hash_type, - hash_key=hash_key, - ).redacted_text - elif anonymize: + if anonymize: parts = [] cursor = 0 suffix = "REDACTED" if method == "redact" else "XXXXX" for finding in sorted(findings, key=lambda item: item["start"]): - parts.extend( - (text[cursor : finding["start"]], f"[{finding['type']}_{suffix}]") - ) + if method == "hash": + digest = _hash_value( + finding["value"], algorithm, hash_key, plain_length=8 + ) + replacement = f"[{finding['type']}_{digest}]" + else: + replacement = f"[{finding['type']}_{suffix}]" + parts.extend((text[cursor : finding["start"]], replacement)) cursor = finding["end"] parts.append(text[cursor:]) result["anonymized"] = "".join(parts) diff --git a/datafog/engine.py b/datafog/engine.py index 884bb1de..0d4a787e 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -809,6 +809,26 @@ def _validate_hash_parameters( raise ValueError(f"Unsupported hash_type: {hash_type!r}") from None +def _hash_value( + text: str, + hash_type: str, + hash_key: bytes | None, + *, + plain_length: int = 12, +) -> str: + """Retain legacy plain-hash widths; keyed HMAC always uses its full digest. + + Unkeyed hashes are deterministic pseudonyms and remain vulnerable to + dictionary attacks regardless of digest width. Supply a private key to + prevent direct enumeration of low-entropy identifiers. + """ + _validate_hash_parameters("hash", hash_type, hash_key) + encoded = text.encode("utf-8", "surrogatepass") + if hash_key is not None: + return hmac.new(hash_key, encoded, hash_type).hexdigest() + return hashlib.new(hash_type, encoded).hexdigest()[:plain_length] + + def _validated_redaction_entities(text: str, entities: list[Entity]) -> list[Entity]: valid_entities = [] for entity in entities: @@ -842,6 +862,10 @@ def redact( type priority and then confidence. Suppressed spans are omitted from ``RedactResult.entities`` and ``mapping``. + Plain hashes retain the historical 12-character digest prefix and provide + deterministic pseudonymization. With ``hash_key`` (at least 32 bytes), HMAC + uses the complete digest. Unkeyed hashes do not resist dictionary attacks. + Invalid ranges, mismatched source text, and malformed labels raise ValueError. ``mapping`` for the ``mask`` strategy is keyed by per-type indexed keys @@ -881,12 +905,7 @@ def redact( break occupied_tokens.add(mapping_key) elif strategy == "hash": - encoded = original.encode("utf-8", "surrogatepass") - digest = ( - hmac.new(hash_key, encoded, hash_type).hexdigest() - if hash_key is not None - else hashlib.new(hash_type, encoded).hexdigest() - ) + digest = _hash_value(original, hash_type, hash_key) key = (entity.type, original) if key not in pseudonym_by_value: candidate = f"[{entity.type}_{digest}]" diff --git a/docs/review-remediation.md b/docs/review-remediation.md index 86e37cc4..04e010ea 100644 --- a/docs/review-remediation.md +++ b/docs/review-remediation.md @@ -30,8 +30,14 @@ nonblocking events receive an explicit incomplete-scan notice. Unexpected hook failures retain the documented fail-open contract and log exception types only. `hash_type` selects md5, sha256 or sha3_256 on the CLI and legacy model APIs. -The engine also accepts algorithms provided by hashlib. Digests use full hex -lengths. Plain deterministic hashes are pseudonymization: SSNs, phones and +The engine also accepts fixed-length algorithms provided by hashlib. Existing +plain-hash marker formats remain compatible: engine, class facade and CLI +markers use 12 hex characters with SHA256 as their default; deprecated top-level +`process` uses eight hex characters and retains its MD5 default. Its explicit +`hash_type` option uses that same legacy width. The legacy Anonymizer model keeps +its existing full plain digests. Explicit keyed hashes use the full HMAC digest; +an omitted algorithm selects SHA256 for this new keyed mode. +Plain deterministic hashes are pseudonymization: SSNs, phones and other guessable inputs can be enumerated. Use `hash_key` (at least 32 random bytes) for keyed HMAC and keep the key outside logs and source control: @@ -91,7 +97,7 @@ release still requires the Core publication and Python dependency update above. | PY-M4 | Native backend selection also supplies the structured stage of smart/spaCy/GLiNER. CLI commands work with DATAFOG_BACKEND=rust. | | PY-M5 | Request-scoped tokens, nested tool/function arguments, prompt/input fields and buffered streaming protect all tested request/response shapes. Live proxy deployment is not claimed. | | PY-M6 | Bare strings, invalid labels and labels unavailable to the selected backend/engine raise explicit errors. Supported aliases remain accepted. | -| PY-M7 | APIs and CLI honor hash_type, use full digests and support keyed HMAC. Plain-hash guessing limitations are documented; algorithm/key tests pass. | +| PY-M7 | APIs and CLI honor hash_type, preserve existing plain-hash formats and support full-length keyed HMAC. Plain-hash guessing limitations are documented; algorithm/key and compatibility tests pass. | | PY-M8 | Hook per-string/total budget exhaustion emits review/deny or an incomplete-scan notice. Tests put PII past the cap. | | PY-M9 | Allowlist expressions run with match timeouts; timed-out expressions retain findings. Adversarial patterns and safe quantified groups are tested. | | PY-M10 | Dependency resolution stays within the explicitly reviewed Core minor, replacing the broad less-than-2 range. Exact output locks should still pin patch versions. | diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json index b3ddf293..3f47b3d6 100644 --- a/tests/contracts/wrapper-4.12.0.json +++ b/tests/contracts/wrapper-4.12.0.json @@ -131,151 +131,6 @@ "warnings": [] } }, - "redact-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", - "expected": { - "value": { - "result_type": "RedactResult", - "fields": { - "redacted_text": "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c] [EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [SSN_01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0]", - "mapping": { - "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]": "a@example.com", - "[EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c]": "b@example.com", - "[SSN_01a54629efb952287e554eb23ef69c52097a75aecc0e3a93ca0855ab6d7a31a0]": "123-45-6789" - }, - "entities": [ - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "a@example.com", - "start": 0, - "end": 13, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "b@example.com", - "start": 14, - "end": 27, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "a@example.com", - "start": 28, - "end": 41, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "SSN", - "text": "123-45-6789", - "start": 42, - "end": 53, - "confidence": 1.0, - "engine": "regex" - } - } - ] - } - }, - "warnings": [] - } - }, - "engine-redact-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", - "expected": { - "value": { - "result_type": "RedactResult", - "fields": { - "redacted_text": "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a] [EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c] [EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]", - "mapping": { - "[EMAIL_08168cd80dfd534ab0f10af10f1303fe00af2d43ab5c1432360d137f8197e17a]": "a@example.com", - "[EMAIL_e8f39b3e1382367d6d41ab34dc270d4e7533f978c9e9a775dfe2185b2f96b96c]": "b@example.com" - }, - "entities": [ - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "a@example.com", - "start": 0, - "end": 13, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "b@example.com", - "start": 14, - "end": 27, - "confidence": 1.0, - "engine": "regex" - } - }, - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "a@example.com", - "start": 28, - "end": 41, - "confidence": 1.0, - "engine": "regex" - } - } - ] - } - }, - "warnings": [] - } - }, - "preset-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 emits the complete chosen digest and supports keyed HMAC.", - "expected": { - "value": { - "result_type": "RedactResult", - "fields": { - "redacted_text": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]", - "mapping": { - "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]": "alice@example.com" - }, - "entities": [ - { - "result_type": "Entity", - "fields": { - "type": "EMAIL", - "text": "alice@example.com", - "start": 0, - "end": 17, - "confidence": 1.0, - "engine": "regex" - } - } - ] - } - }, - "warnings": [] - } - }, "explicit-invalid-bounds": { "classification": "validation-difference", "reason": "PY-L7 rejects malformed explicit spans rather than silently skipping them.", @@ -495,8 +350,8 @@ } }, "process-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 replaces legacy truncated MD5 with the complete selected digest; PY-L3 updates the retirement notice.", + "classification": "diagnostic-difference", + "reason": "PY-L3 retirement notices identify the current release. Legacy default MD5 and its eight-character prefix remain compatible; explicit algorithms and keyed HMAC are additional options.", "expected": { "value": { "original": "alice@example.com", @@ -508,7 +363,7 @@ "end": 17 } ], - "anonymized": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]" + "anonymized": "[EMAIL_c160f8cc]" }, "warnings": [ { @@ -569,7 +424,7 @@ }, "public-signatures": { "classification": "signature-addition", - "reason": "PY-M7 adds keyword-only hash_type/hash_key; TextService custom models use the engine gliner_model keyword; backend remains keyword-only.", + "reason": "PY-M7 adds keyword-only hash_type/hash_key while preserving existing unkeyed output widths and omitted legacy hash defaults; TextService custom models use the engine gliner_model keyword; backend remains keyword-only.", "expected": { "value": { "datafog:scan": { @@ -762,7 +617,7 @@ "name": "hash_type", "kind": "KEYWORD_ONLY", "required": false, - "default": "sha256" + "default": null }, { "name": "hash_key", @@ -1112,14 +967,6 @@ }, "warnings": [] } - }, - "anonymize-text-hash": { - "classification": "behavioral-fix", - "reason": "PY-M7 emits the complete chosen digest rather than a truncated unsalted token.", - "expected": { - "value": "[EMAIL_ff8d9819fc0e12bf0d24892e45987e249a28dce836a85cad60e28eaaa8c6d976]", - "warnings": [] - } } } } diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py index 9f47318a..b65342a7 100644 --- a/tests/test_review_engine.py +++ b/tests/test_review_engine.py @@ -236,7 +236,7 @@ def test_hash_algorithm_and_hmac_are_honored(algorithm): text = "a@b.com" entities = engine.scan(text, engine="regex").entities plain = engine.redact(text, entities, strategy="hash", hash_type=algorithm) - assert hashlib.new(algorithm, text.encode()).hexdigest() in plain.redacted_text + assert hashlib.new(algorithm, text.encode()).hexdigest()[:12] in plain.redacted_text key = bytes(range(32)) keyed = engine.redact( text, entities, strategy="hash", hash_type=algorithm, hash_key=key @@ -298,7 +298,7 @@ def test_fallback_adversarial_patterns_complete(): def test_hash_marker_cannot_collide_with_literal_input(): value = "a@b.com" - digest = hashlib.sha256(value.encode()).hexdigest() + digest = hashlib.sha256(value.encode()).hexdigest()[:12] marker = f"[EMAIL_{digest}]" text = f"{marker} {value} {value}" result = engine.scan_and_redact(text, engine="regex", strategy="hash") @@ -329,3 +329,16 @@ def test_fallback_email_context_is_trimmed_without_losing_mailboxes(text): def test_spacy_selection_rejects_gliner_only_labels(): with pytest.raises(ValueError, match="PASSPORT_NUMBER"): engine.scan("passport SECRET", engine="spacy", entity_types=["PASSPORT_NUMBER"]) + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256", "sha3_256"]) +def test_shared_hash_helper_retains_legacy_widths_and_full_hmac(algorithm): + value = "a@b.com" + digest = hashlib.new(algorithm, value.encode()).hexdigest() + assert engine._hash_value(value, algorithm, None) == digest[:12] + assert engine._hash_value(value, algorithm, None, plain_length=8) == digest[:8] + key = bytes(range(32)) + assert ( + engine._hash_value(value, algorithm, key, plain_length=8) + == hmac.new(key, value.encode(), algorithm).hexdigest() + ) diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py index fb7cc555..c471bf06 100644 --- a/tests/test_review_entrypoints.py +++ b/tests/test_review_entrypoints.py @@ -55,14 +55,14 @@ def test_anonymizing_operations_imply_scan(operation): def test_hash_algorithm_honored_across_api_and_cli(algorithm): digest = hashlib.new(algorithm.value, EMAIL.encode()).hexdigest() assert ( - digest - in DataFog(hash_type=algorithm).process(EMAIL, anonymize=True, method="hash")[ + f"[EMAIL_{digest[:12]}]" + == DataFog(hash_type=algorithm).process(EMAIL, anonymize=True, method="hash")[ "anonymized" ] ) assert ( - digest - in datafog.process( + f"[EMAIL_{digest[:8]}]" + == datafog.process( EMAIL, anonymize=True, method="hash", hash_type=algorithm.value )["anonymized"] ) @@ -70,7 +70,18 @@ def test_hash_algorithm_honored_across_api_and_cli(algorithm): app, ["hash-text", EMAIL, "--hash-type", algorithm.value] ) assert result.exit_code == 0, result.output - assert digest in result.stdout + assert result.stdout.strip() == f"[EMAIL_{digest[:12]}]" + + +def test_existing_plain_hash_defaults_remain_compatible(): + assert ( + datafog.redact(EMAIL, strategy="hash").redacted_text + == f"[EMAIL_{hashlib.sha256(EMAIL.encode()).hexdigest()[:12]}]" + ) + assert ( + datafog.process(EMAIL, anonymize=True, method="hash")["anonymized"] + == f"[EMAIL_{hashlib.md5(EMAIL.encode()).hexdigest()[:8]}]" + ) def test_keyed_hash_public_facades(): From 0d7519dbba5b38019998fe2fbf0803bb7f7e0780 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:23:44 -0800 Subject: [PATCH 14/20] chore(release): prepare DataFog 4.12 with Core 2 --- CHANGELOG.MD | 7 ++++- README.md | 11 +++++--- RELEASE_NOTES_4.12.0.md | 57 ++++++++++++++++++++++++++++++++++++++ docs/review-remediation.md | 23 +++++++++------ setup.py | 4 +-- 5 files changed, 86 insertions(+), 16 deletions(-) create mode 100644 RELEASE_NOTES_4.12.0.md diff --git a/CHANGELOG.MD b/CHANGELOG.MD index ab2e41b6..62acaddb 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -23,7 +23,12 @@ - Reuse tokens for repeated values, avoid existing token collisions, reject invalid explicit ranges and unsupported entity selections, and report backend entity capabilities. Async decorators await results before filtering. -- Keep Core within the reviewed minor series, raise the Pydantic floor to 2.1, +- Require the reviewed Core 2.0 minor series (`datafog-core~=2.0.0`) in the base + package and `rust` extra; review native differences in `rust-2.0.0.json`. + Core's complete mapped-IPv6 spans, email boundaries, Unicode mailboxes and + phone-plus-ZIP classification are available through the wrapper. Published + Core artifacts must pass dependent qualification before this candidate ships. +- Raise the Pydantic floor to 2.1, include spaCy in `nlp-advanced`, and allow Torch 2.x versions compatible with `datafog-core[gliner2]`. Add an isolated build declaration and runnable sdist tests; remove unused duplicate implementation modules. diff --git a/README.md b/README.md index 375cbc53..7ee0f892 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ and `ADDRESS` from an embedded model, so name and address detection needs no spaCy and no model download. Findings keep `engine="regex"` provenance and `confidence=1.0` as compatibility values, not native confidence estimates. Reviewed differences from the Python engine are listed in -[tests/contracts/rust-1.4.0.json](tests/contracts/rust-1.4.0.json). +[tests/contracts/rust-2.0.0.json](tests/contracts/rust-2.0.0.json). `datafog.native` exposes the native schema directly. Python 3.13 support is certified for the core SDK, CLI, `nlp`, @@ -288,9 +288,12 @@ Telemetry does not include input text or detected PII values. The native engine is the default, with a warned Python fallback. The 4.12 review fixes preserve structured detection across every engine and legacy API. -The dependency stays within the reviewed Core 1.4 minor series (`~=1.4.0`); -Core 2.0 is a separate release candidate that must be published and qualified -before this SDK raises its floor. See [release preparation](docs/review-remediation.md). +This release candidate declares the reviewed Core 2.0 minor series +(`datafog-core~=2.0.0`) in the base package and the `rust` extra. Core 2.0 is a +separate release candidate; this branch does not publish it. Merge and release +require Core 2.0 publication followed by qualification against its actual +published artifacts. Local source-wheel results are additional evidence. +See [release preparation](docs/review-remediation.md). Entity labels, locales and activation settings come from the installed Core. `get_supported_entities()` reports that backend's supported inventory, including diff --git a/RELEASE_NOTES_4.12.0.md b/RELEASE_NOTES_4.12.0.md new file mode 100644 index 00000000..e392b15c --- /dev/null +++ b/RELEASE_NOTES_4.12.0.md @@ -0,0 +1,57 @@ +# DataFog 4.12.0 + +DataFog 4.12.0 carries structured PII detection through every engine, legacy +API and CLI anonymization path. Smart, spaCy and GLiNER modes retain the same +native card, IP and credential findings as the regex mode, then add NER spans. +TextService scans complete documents, and spaCy/GLiNER character offsets prevent +an earlier substring or repeated name from receiving the wrong redaction. + +This candidate requires `datafog-core~=2.0.0` in the base package and the `rust` +extra. Core 2.0 supplies complete IPv4-mapped IPv6 redaction, corrected email +field boundaries and Unicode mailboxes, and phone-plus-ZIP classification that +does not suppress the phone behind a false card finding. Native differences +are recorded in `tests/contracts/rust-2.0.0.json`. Capability contract 1 remains +unchanged; Python and Core have independent version numbers. + +Lone surrogate characters no longer disable native scans. Dense overlap +suppression, Python fallback patterns and allowlist matching have bounded work; +typoed entity labels, invalid ranges and invalid locales raise explicit errors. +Anonymizing pipeline operations imply scanning. Async guardrail decorators +await their result, and non-string results raise TypeError. + +LiteLLM scans nested tool/function arguments and prompt/input requests with a +shared token assignment for the request. Repeated values reuse tokens, and +existing markers do not collide. Streaming buffers the complete upstream +response before emitting its first redacted chunk, including text and tool +arguments split across deltas. This adds the full upstream completion latency +to first output. Streams exceeding eight million text characters or 100,000 +chunks are rejected. Engine errors retain the configured `open` or `closed` +failure policy. Image and binary parts remain opaque and receive no OCR. + +The Claude Code hook asks for review, or denies according to configuration, +when input exceeds its scan budget; it no longer silently permits an unscanned +tail. Unexpected hook failures retain the documented fail-open policy. + +Hash APIs honor the selected algorithm while preserving existing plain-hash +formats. Engine APIs, the DataFog class and CLI retain SHA-256 with 12 hex +characters by default. Deprecated top-level `datafog.process(method="hash")` +retains MD5 with eight hex characters. The legacy Anonymizer model retains its +existing full plain digest. Explicit `hash_key` adds full-length HMAC with at +least 32 key bytes; omitting the algorithm in keyed mode selects SHA-256. +Unkeyed hashes remain guessable deterministic pseudonyms, not protection +against enumerating SSNs, phone numbers or other low-entropy values. + +The minimum Pydantic version is 2.1. The `nlp-advanced` extra includes spaCy and +permits Torch 2.x versions compatible with Core's GLiNER2 extra. Base installs +show a CLI-extra installation hint instead of a missing-Typer traceback. +Unused duplicate implementations are removed, source distributions include +runnable tests and corpora, and CLI configuration output hides API keys. + +Scheduled release jobs qualify candidates without publishing. Manual jobs +default to dry runs, reject stale or occupied versions, and record immutable +release refs before uploading artifacts. + +This is a release candidate prepared for review. Core 2.0 publication and +qualification of this Python candidate against the actual published Core +artifacts remain mandatory before merge and release. Local source-wheel +qualification does not claim either package has been published. diff --git a/docs/review-remediation.md b/docs/review-remediation.md index 04e010ea..adfd9933 100644 --- a/docs/review-remediation.md +++ b/docs/review-remediation.md @@ -50,14 +50,18 @@ result = datafog.redact("mail jane@example.com", strategy="hash", hash_key=key) ## Release sequence and qualification -The Python candidate is 4.12.0. Its standalone CI uses the published Core 1.4 -minor series, pinned with `~=1.4.0`. Core 2.0.0 is prepared separately because -safe mask-reveal and automatic-overlap transformation semantics are incompatible -changes under Core's version policy. Both use capability contract 1. Before -shipping the combined fixes, publish the qualified Core candidate, then raise -both Python dependency declarations to `~=2.0.0`, update the reviewed native -contract/corpus, and run Python's installed-wheel and release qualification -against the published artifacts. Source-candidate integration is additional +The Python 4.12.0 candidate declares `datafog-core~=2.0.0` in its base +requirements and `rust` extra. Its reviewed native contract is +[`tests/contracts/rust-2.0.0.json`](../tests/contracts/rust-2.0.0.json). +Core 2.0.0 is prepared separately because safe mask-reveal and automatic-overlap +transformation semantics are incompatible changes under Core's version policy. +Both packages continue to use capability contract 1. + +This branch prepares the dependent package without publishing either project. +Its local source wheel is qualified with the source-built Core 2.0.0 wheel. +Merge and publication remain blocked until Core 2.0.0 is published and this +exact Python candidate passes installed-wheel and release qualification against +the actual published Core artifacts. Source-candidate integration is additional evidence; it does not substitute for that post-publication gate. Scheduled Python jobs only qualify candidates. Manual `release.yml` runs default @@ -82,7 +86,8 @@ build requirements are documented in Core's platform policy. All 35 Python findings have a concrete resolution below. Core-owned fixes are separately tracked in Core's `docs/reviews/2026-11-core-findings.md`; their combined -release still requires the Core publication and Python dependency update above. +release still requires Core publication and the published-artifact qualification +above. | Finding | Resolution and proof | | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | diff --git a/setup.py b/setup.py index bffb7ff5..62154455 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ # Core lightweight dependencies only core_deps = [ # Core minor releases may change findings; stay on this reviewed minor. - "datafog-core~=1.4.0", + "datafog-core~=2.0.0", "pydantic>=2.1,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", @@ -89,7 +89,7 @@ extras_require = { # Retained for explicitness; the base install already pulls Core. - "rust": ["datafog-core~=1.4.0"], + "rust": ["datafog-core~=2.0.0"], "nlp": nlp_deps, "nlp-advanced": nlp_deps + nlp_advanced_deps, "ocr": ocr_deps, From e4e751ecd28feb4b6c5e8469664936754b08050e Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:25:19 -0800 Subject: [PATCH 15/20] fix(engine): retain structured findings when NER spans overlap --- datafog/engine.py | 4 +++- tests/corpus/unstructured_pii.json | 4 ++-- tests/test_review_engine.py | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/datafog/engine.py b/datafog/engine.py index 0d4a787e..c8041acf 100644 --- a/datafog/engine.py +++ b/datafog/engine.py @@ -789,7 +789,9 @@ def scan( filtered = _filter_entity_types(combined, entity_types) filtered = _apply_allowlist(filtered, allowlist, allowlist_patterns) - deduped = _suppress_overlapping_entities(_dedupe_entities(filtered)) + # Optional NER adds candidates without suppressing structured findings. + # Transformation resolves overlaps after selection, preserving scan labels. + deduped = _dedupe_entities(filtered) return ScanResult( entities=deduped, text=text, diff --git a/tests/corpus/unstructured_pii.json b/tests/corpus/unstructured_pii.json index ad91c35b..1e5ddf79 100644 --- a/tests/corpus/unstructured_pii.json +++ b/tests/corpus/unstructured_pii.json @@ -168,8 +168,8 @@ { "type": "LOCATION", "text": "Jordan", - "start": 0, - "end": 6 + "start": 33, + "end": 39 } ] }, diff --git a/tests/test_review_engine.py b/tests/test_review_engine.py index b65342a7..5e44c461 100644 --- a/tests/test_review_engine.py +++ b/tests/test_review_engine.py @@ -342,3 +342,19 @@ def test_shared_hash_helper_retains_legacy_widths_and_full_hmac(algorithm): engine._hash_value(value, algorithm, key, plain_length=8) == hmac.new(key, value.encode(), algorithm).hexdigest() ) + + +@pytest.mark.parametrize("mode", ["smart", "spacy", "gliner"]) +def test_optional_ner_cannot_remove_overlapping_structured_scan_findings( + monkeypatch, mode +): + text = "Ping 127.0.0.1" + candidates = [engine.Entity("PERSON", text, 0, len(text), 0.8, mode)] + monkeypatch.setattr(engine, "_spacy_entities", lambda _: candidates) + monkeypatch.setattr(engine, "_gliner_entities", lambda _: candidates) + result = engine.scan(text, engine=mode) + assert any( + item.type == "IP_ADDRESS" and item.text == "127.0.0.1" + for item in result.entities + ) + assert any(item.type == "PERSON" for item in result.entities) From 260b04f76c78b18485f0182d8b78a2b860c001bd Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:40:16 -0800 Subject: [PATCH 16/20] fix(anonymizer): preserve protection with legacy empty entity filters --- datafog/models/anonymizer.py | 2 +- tests/test_review_entrypoints.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datafog/models/anonymizer.py b/datafog/models/anonymizer.py index 30d079bb..3ec36664 100644 --- a/datafog/models/anonymizer.py +++ b/datafog/models/anonymizer.py @@ -55,7 +55,7 @@ def _selected_annotations(self, text, annotations): selected = ( None - if self.entities is None + if not self.entities else { CANONICAL_TYPE_MAP.get(entity.name, entity.name) for entity in self.entities diff --git a/tests/test_review_entrypoints.py b/tests/test_review_entrypoints.py index c471bf06..519c9cf6 100644 --- a/tests/test_review_entrypoints.py +++ b/tests/test_review_entrypoints.py @@ -143,6 +143,16 @@ def test_enum_filter_and_date_replacement(): ) +@pytest.mark.parametrize("strategy", list(AnonymizerType)) +def test_legacy_empty_entity_selection_keeps_protecting_all(strategy): + annotation = AnnotationResult(start=0, end=len(EMAIL), entity_type="EMAIL") + result = Anonymizer(anonymizer_type=strategy, entities=[]).anonymize( + EMAIL, [annotation] + ) + assert EMAIL not in result.anonymized_text + assert len(result.anonymized_entities) == 1 + + @pytest.mark.parametrize("strategy", list(AnonymizerType)) def test_legacy_overlaps_never_leave_partial_value(strategy): text = "abcdefghij" From d6b8a2da8799a23560b35b9f0ad6ad488e90337a Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:41:39 -0800 Subject: [PATCH 17/20] test: recognize exact restored GLiNER URL label coverage --- tests/corpus/negative_cases.json | 22 ++++++++++- tests/test_detection_accuracy.py | 64 ++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/tests/corpus/negative_cases.json b/tests/corpus/negative_cases.json index 44eb1fcf..c5b67b76 100644 --- a/tests/corpus/negative_cases.json +++ b/tests/corpus/negative_cases.json @@ -49,7 +49,16 @@ { "id": "markdown-link", "input": "See [RFC 1918](https://example.com/rfc1918)", - "expected_entities": [] + "expected_entities": [], + "permitted_gliner_entities": [ + { + "type": "URL", + "text": "https://example.com/rfc1918", + "start": 15, + "end": 42 + } + ], + "permitted_gliner_reason": "PY-M14 restores the existing GLiNER URL label. This exact public URL is valid URL detection; it must not become EMAIL, PHONE or IP_ADDRESS." }, { "id": "code-symbol", @@ -74,6 +83,15 @@ { "id": "url-with-at", "input": "https://example.com/@user/profile is a URL path", - "expected_entities": [] + "expected_entities": [], + "permitted_gliner_entities": [ + { + "type": "URL", + "text": "https://example.com/@user/profile", + "start": 0, + "end": 33 + } + ], + "permitted_gliner_reason": "PY-M14 restores the existing GLiNER URL label. This exact public URL is valid URL detection; it must not become EMAIL, PHONE or IP_ADDRESS." } ] diff --git a/tests/test_detection_accuracy.py b/tests/test_detection_accuracy.py index 6e7d8dfe..8cff1e82 100644 --- a/tests/test_detection_accuracy.py +++ b/tests/test_detection_accuracy.py @@ -453,6 +453,33 @@ def _assert_expected_found( return actual, expected +def _unexpected_negative_entities( + case: dict[str, Any], actual: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Keep legacy negative PII checks while recognizing two reviewed URL spans. + + PY-M14 restores GLiNER's URL output. Only exact, source-aligned URL candidates + from that model are permitted; other labels and malformed offsets still fail. + These added label checks are outside the historical PII metric inventory. + """ + permitted = case.get("permitted_gliner_entities", []) + return [ + entity + for entity in actual + if not ( + entity.get("engine") == "gliner" + and any( + all( + entity.get(key) == span[key] + for key in ("type", "text", "start", "end") + ) + and case["input"][span["start"] : span["end"]] == span["text"] + for span in permitted + ) + ) + ] + + def _compute_metrics( engines: list[str], corpora: list[tuple[str, list[dict[str, Any]]]] ) -> dict[str, Any]: @@ -471,6 +498,8 @@ def _compute_metrics( locales=case.get("locales"), backend=_corpus_backend(engine), ) + if corpus_kind == "negative": + actual = _unexpected_negative_entities(case, actual) expected = _required_expected( case["expected_entities"], engine, corpus_kind ) @@ -569,6 +598,7 @@ def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None: for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) } actual = [e for e in actual if (e["type"], e["text"]) not in added] + actual = _unexpected_negative_entities(case, actual) assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -591,6 +621,7 @@ def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None: for pair in _native_deltas()["extra"].get(case["id"], {}).get("added", []) } actual = [e for e in actual if (e["type"], e["text"]) not in added] + actual = _unexpected_negative_entities(case, actual) assert not actual, f"{case['id']} ({engine}) false positives: {actual}" @@ -666,3 +697,36 @@ def test_accuracy_metrics_snapshot() -> None: output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(json.dumps(metrics, indent=2), encoding="utf-8") assert "overall" in metrics and metrics["overall"] + + +def test_reviewed_url_candidates_require_exact_label_origin_and_offsets(): + case = next( + case + for case in load_corpus("negative_cases.json") + if case["id"] == "markdown-link" + ) + permitted = dict(case["permitted_gliner_entities"][0], engine="gliner") + assert _unexpected_negative_entities(case, [permitted]) == [] + for altered in [ + dict(permitted, type="EMAIL"), + dict(permitted, type="PHONE"), + dict(permitted, type="IP_ADDRESS"), + dict(permitted, start=14), + dict(permitted, end=41), + dict(permitted, engine="spacy"), + dict(permitted, text="example.com"), + ]: + assert _unexpected_negative_entities(case, [altered]) == [altered] + + +def test_reviewed_url_permission_is_confined_to_the_two_named_cases(): + permitted_cases = [ + case + for case in load_corpus("negative_cases.json") + if case.get("permitted_gliner_entities") + ] + assert {case["id"] for case in permitted_cases} == {"markdown-link", "url-with-at"} + for case in permitted_cases: + assert case["permitted_gliner_reason"].startswith("PY-M14") + assert len(case["permitted_gliner_entities"]) == 1 + assert case["permitted_gliner_entities"][0]["type"] == "URL" From 5476c74716ac27356d7029f3b4007058e41b3859 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:55:19 -0800 Subject: [PATCH 18/20] chore: prepare Core 1.5 compatibility qualification --- README.md | 2 +- docs/review-remediation.md | 14 +++++++------- .../contracts/{rust-2.0.0.json => rust-1.5.0.json} | 4 ++-- tests/contracts/wrapper-4.12.0.json | 2 +- ...0_integration.py => test_core15_integration.py} | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) rename tests/contracts/{rust-2.0.0.json => rust-1.5.0.json} (91%) rename tests/{test_core20_integration.py => test_core15_integration.py} (88%) diff --git a/README.md b/README.md index 375cbc53..73749bed 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ Telemetry does not include input text or detected PII values. The native engine is the default, with a warned Python fallback. The 4.12 review fixes preserve structured detection across every engine and legacy API. The dependency stays within the reviewed Core 1.4 minor series (`~=1.4.0`); -Core 2.0 is a separate release candidate that must be published and qualified +Core 1.5 is a separate release candidate that must be published and qualified before this SDK raises its floor. See [release preparation](docs/review-remediation.md). Entity labels, locales and activation settings come from the installed Core. diff --git a/docs/review-remediation.md b/docs/review-remediation.md index 04e010ea..7618b363 100644 --- a/docs/review-remediation.md +++ b/docs/review-remediation.md @@ -51,11 +51,11 @@ result = datafog.redact("mail jane@example.com", strategy="hash", hash_key=key) ## Release sequence and qualification The Python candidate is 4.12.0. Its standalone CI uses the published Core 1.4 -minor series, pinned with `~=1.4.0`. Core 2.0.0 is prepared separately because -safe mask-reveal and automatic-overlap transformation semantics are incompatible -changes under Core's version policy. Both use capability contract 1. Before +minor series, pinned with `~=1.4.0`. Core 1.5.0 is prepared separately as a minor +release correcting detection and transformation behavior through the existing +APIs. Public APIs remain available and capability contract 1 is unchanged. Before shipping the combined fixes, publish the qualified Core candidate, then raise -both Python dependency declarations to `~=2.0.0`, update the reviewed native +both Python dependency declarations to `~=1.5.0`, update the reviewed native contract/corpus, and run Python's installed-wheel and release qualification against the published artifacts. Source-candidate integration is additional evidence; it does not substitute for that post-publication gate. @@ -106,9 +106,9 @@ release still requires the Core publication and Python dependency update above. | PY-M13 | Optional Pydantic fields have defaults, labels use canonical names, enum filters work and overlaps merge before replacement. Model/anonymizer regressions pass. | | PY-M14 | Preserve supported GLiNER labels, use exact offsets, exclude relative spaCy DATE mentions, propagate model failures and reject oversized annotator input explicitly. | | PY-M15 | Embedded model calls use bounded overlapping chunks. Large PERSON/ADDRESS plus structured-PII tests succeed with published and candidate Core. Native model limits remain enforced. | -| PY-M16 | Core 2 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | +| PY-M16 | Core 1.5 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | | PY-L1 | Console bootstrap prints the CLI-extra install hint on a base install. A clean installed-wheel check confirms no traceback. | -| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 2 suppresses the labelled-tax-ID PHONE false positive. | +| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 1.5 suppresses the labelled-tax-ID PHONE false positive. | | PY-L3 | Retirement notices derive the current supported release version, and the native install hint no longer names an older release. | | PY-L4 | Supported-entity reporting follows the selected native/Python capability inventory, including opt-in labels. Tests separate supported from default-active labels. | | PY-L5 | Document actual token/pseudonym outputs and reject invalid methods in both process APIs. | @@ -116,7 +116,7 @@ release still requires the Core publication and Python dependency update above. | PY-L7 | Tokens reuse values and avoid existing markers; invalid explicit ranges/text raise errors. Collision, repeated-value and validation tests pass. | | PY-L8 | Remove unused implementation copies, declare an isolated build, include test packages/corpora/scripts in sdists, align formatter pins and extend tox through 3.14. Optional OCR imports skip cleanly when absent. | | PY-L9 | CLI configuration hides API keys; import and commands preserve host logging levels. CLI/logger regressions pass. | -| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 2 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | +| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 1.5 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | | PY-L11 | TextService validates positive chunk lengths and engine names with explicit exceptions, including optimized Python execution. | | PY-L12 | Remove incompatible Torch upper bound and include spaCy in nlp-advanced. Resolving it with Core GLiNER2 succeeds with Torch 2.12.0. | | PY-L13 | Declare Pydantic 2.1 minimum. A clean Python 3.10 base install with Pydantic 2.1 imports and scans successfully. | diff --git a/tests/contracts/rust-2.0.0.json b/tests/contracts/rust-1.5.0.json similarity index 91% rename from tests/contracts/rust-2.0.0.json rename to tests/contracts/rust-1.5.0.json index 324f9cfa..13f39961 100644 --- a/tests/contracts/rust-2.0.0.json +++ b/tests/contracts/rust-1.5.0.json @@ -1,5 +1,5 @@ { - "core_version": "2.0.0", + "core_version": "1.5.0", "cases": { "scan-observation-invalid-card": { "classification": "detector-difference", @@ -73,5 +73,5 @@ } } }, - "review": "Core 2.0 candidate qualifications retain the four reviewed historical differences below. Wrapper 4.12 fixes are layered separately in wrapper-4.12.0.json." + "review": "Core 1.5 candidate qualification retains reviewed historical differences and documents the contextual tax-ID correction. Wrapper 4.12 fixes are layered separately in wrapper-4.12.0.json." } diff --git a/tests/contracts/wrapper-4.12.0.json b/tests/contracts/wrapper-4.12.0.json index 3f47b3d6..cd78ae0d 100644 --- a/tests/contracts/wrapper-4.12.0.json +++ b/tests/contracts/wrapper-4.12.0.json @@ -1,6 +1,6 @@ { "wrapper_version": "4.12.0", - "core_versions": ["1.4.0", "2.0.0"], + "core_versions": ["1.4.0", "1.5.0"], "source_oracle": "4.8.1.json (unchanged)", "review": "2026-11-15 PDF remediation: explicit reviewed wrapper changes, not a regenerated historical oracle", "cases": { diff --git a/tests/test_core20_integration.py b/tests/test_core15_integration.py similarity index 88% rename from tests/test_core20_integration.py rename to tests/test_core15_integration.py index 290f8e0d..ed9bcdf4 100644 --- a/tests/test_core20_integration.py +++ b/tests/test_core15_integration.py @@ -1,4 +1,4 @@ -"""Core 2 candidate seam regressions; exercised before the dependency floor moves.""" +"""Core 1.5 candidate seam regressions; exercised before the dependency floor moves.""" import importlib.metadata @@ -7,8 +7,8 @@ import datafog pytestmark = pytest.mark.skipif( - importlib.metadata.version("datafog-core").split(".")[0] != "2", - reason="qualifies the separately built Core 2 candidate before publication", + importlib.metadata.version("datafog-core").split(".")[:2] != ["1", "5"], + reason="qualifies the separately built Core 1.5 candidate before publication", ) From e55df684b06b698175b883847dbe3f8ee7be4ac9 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:55:54 -0800 Subject: [PATCH 19/20] chore(release): target Core 1.5 for DataFog 4.12 --- CHANGELOG.MD | 4 ++-- README.md | 8 ++++---- RELEASE_NOTES_4.12.0.md | 8 ++++---- docs/review-remediation.md | 21 +++++++++++---------- setup.py | 4 ++-- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 62acaddb..31b49807 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -23,8 +23,8 @@ - Reuse tokens for repeated values, avoid existing token collisions, reject invalid explicit ranges and unsupported entity selections, and report backend entity capabilities. Async decorators await results before filtering. -- Require the reviewed Core 2.0 minor series (`datafog-core~=2.0.0`) in the base - package and `rust` extra; review native differences in `rust-2.0.0.json`. +- Require the reviewed Core 1.5 minor series (`datafog-core~=1.5.0`) in the base + package and `rust` extra; review native differences in `rust-1.5.0.json`. Core's complete mapped-IPv6 spans, email boundaries, Unicode mailboxes and phone-plus-ZIP classification are available through the wrapper. Published Core artifacts must pass dependent qualification before this candidate ships. diff --git a/README.md b/README.md index 7ee0f892..d60ba71c 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ and `ADDRESS` from an embedded model, so name and address detection needs no spaCy and no model download. Findings keep `engine="regex"` provenance and `confidence=1.0` as compatibility values, not native confidence estimates. Reviewed differences from the Python engine are listed in -[tests/contracts/rust-2.0.0.json](tests/contracts/rust-2.0.0.json). +[tests/contracts/rust-1.5.0.json](tests/contracts/rust-1.5.0.json). `datafog.native` exposes the native schema directly. Python 3.13 support is certified for the core SDK, CLI, `nlp`, @@ -288,10 +288,10 @@ Telemetry does not include input text or detected PII values. The native engine is the default, with a warned Python fallback. The 4.12 review fixes preserve structured detection across every engine and legacy API. -This release candidate declares the reviewed Core 2.0 minor series -(`datafog-core~=2.0.0`) in the base package and the `rust` extra. Core 2.0 is a +This release candidate declares the reviewed Core 1.5 minor series +(`datafog-core~=1.5.0`) in the base package and the `rust` extra. Core 1.5 is a separate release candidate; this branch does not publish it. Merge and release -require Core 2.0 publication followed by qualification against its actual +require Core 1.5 publication followed by qualification against its actual published artifacts. Local source-wheel results are additional evidence. See [release preparation](docs/review-remediation.md). diff --git a/RELEASE_NOTES_4.12.0.md b/RELEASE_NOTES_4.12.0.md index e392b15c..2682fb84 100644 --- a/RELEASE_NOTES_4.12.0.md +++ b/RELEASE_NOTES_4.12.0.md @@ -6,11 +6,11 @@ native card, IP and credential findings as the regex mode, then add NER spans. TextService scans complete documents, and spaCy/GLiNER character offsets prevent an earlier substring or repeated name from receiving the wrong redaction. -This candidate requires `datafog-core~=2.0.0` in the base package and the `rust` -extra. Core 2.0 supplies complete IPv4-mapped IPv6 redaction, corrected email +This candidate requires `datafog-core~=1.5.0` in the base package and the `rust` +extra. Core 1.5 supplies complete IPv4-mapped IPv6 redaction, corrected email field boundaries and Unicode mailboxes, and phone-plus-ZIP classification that does not suppress the phone behind a false card finding. Native differences -are recorded in `tests/contracts/rust-2.0.0.json`. Capability contract 1 remains +are recorded in `tests/contracts/rust-1.5.0.json`. Capability contract 1 remains unchanged; Python and Core have independent version numbers. Lone surrogate characters no longer disable native scans. Dense overlap @@ -51,7 +51,7 @@ Scheduled release jobs qualify candidates without publishing. Manual jobs default to dry runs, reject stale or occupied versions, and record immutable release refs before uploading artifacts. -This is a release candidate prepared for review. Core 2.0 publication and +This is a release candidate prepared for review. Core 1.5 publication and qualification of this Python candidate against the actual published Core artifacts remain mandatory before merge and release. Local source-wheel qualification does not claim either package has been published. diff --git a/docs/review-remediation.md b/docs/review-remediation.md index adfd9933..2592ab59 100644 --- a/docs/review-remediation.md +++ b/docs/review-remediation.md @@ -50,16 +50,17 @@ result = datafog.redact("mail jane@example.com", strategy="hash", hash_key=key) ## Release sequence and qualification -The Python 4.12.0 candidate declares `datafog-core~=2.0.0` in its base +The Python 4.12.0 candidate declares `datafog-core~=1.5.0` in its base requirements and `rust` extra. Its reviewed native contract is -[`tests/contracts/rust-2.0.0.json`](../tests/contracts/rust-2.0.0.json). -Core 2.0.0 is prepared separately because safe mask-reveal and automatic-overlap -transformation semantics are incompatible changes under Core's version policy. -Both packages continue to use capability contract 1. +[`tests/contracts/rust-1.5.0.json`](../tests/contracts/rust-1.5.0.json). +Core 1.5.0 is prepared separately with the detector and transformation safety +fixes from the review, including bounded mask reveal and complete scanner +coverage for automatic overlap handling. Both packages continue to use +capability contract 1. This branch prepares the dependent package without publishing either project. -Its local source wheel is qualified with the source-built Core 2.0.0 wheel. -Merge and publication remain blocked until Core 2.0.0 is published and this +Its local source wheel is qualified with the source-built Core 1.5.0 wheel. +Merge and publication remain blocked until Core 1.5.0 is published and this exact Python candidate passes installed-wheel and release qualification against the actual published Core artifacts. Source-candidate integration is additional evidence; it does not substitute for that post-publication gate. @@ -111,9 +112,9 @@ above. | PY-M13 | Optional Pydantic fields have defaults, labels use canonical names, enum filters work and overlaps merge before replacement. Model/anonymizer regressions pass. | | PY-M14 | Preserve supported GLiNER labels, use exact offsets, exclude relative spaCy DATE mentions, propagate model failures and reject oversized annotator input explicitly. | | PY-M15 | Embedded model calls use bounded overlapping chunks. Large PERSON/ADDRESS plus structured-PII tests succeed with published and candidate Core. Native model limits remain enforced. | -| PY-M16 | Core 2 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | +| PY-M16 | Core 1.5 excludes phone-plus-ZIP card candidates. Candidate seam tests assert PHONE/ZIP labels and no false card. | | PY-L1 | Console bootstrap prints the CLI-extra install hint on a base install. A clean installed-wheel check confirms no traceback. | -| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 2 suppresses the labelled-tax-ID PHONE false positive. | +| PY-L2 | Remove README BOM and stale backend statements, repair contract links and make the German example explicitly select its locale/entity. Core 1.5 suppresses the labelled-tax-ID PHONE false positive. | | PY-L3 | Retirement notices derive the current supported release version, and the native install hint no longer names an older release. | | PY-L4 | Supported-entity reporting follows the selected native/Python capability inventory, including opt-in labels. Tests separate supported from default-active labels. | | PY-L5 | Document actual token/pseudonym outputs and reject invalid methods in both process APIs. | @@ -121,7 +122,7 @@ above. | PY-L7 | Tokens reuse values and avoid existing markers; invalid explicit ranges/text raise errors. Collision, repeated-value and validation tests pass. | | PY-L8 | Remove unused implementation copies, declare an isolated build, include test packages/corpora/scripts in sdists, align formatter pins and extend tox through 3.14. Optional OCR imports skip cleanly when absent. | | PY-L9 | CLI configuration hides API keys; import and commands preserve host logging levels. CLI/logger regressions pass. | -| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 2 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | +| PY-L10 | Unicode emails are supported in fallback and candidate Core; Core 1.5 trims embedded PERSON trailing punctuation. Candidate seam tests cover both. | | PY-L11 | TextService validates positive chunk lengths and engine names with explicit exceptions, including optimized Python execution. | | PY-L12 | Remove incompatible Torch upper bound and include spaCy in nlp-advanced. Resolving it with Core GLiNER2 succeeds with Torch 2.12.0. | | PY-L13 | Declare Pydantic 2.1 minimum. A clean Python 3.10 base install with Pydantic 2.1 imports and scans successfully. | diff --git a/setup.py b/setup.py index 62154455..76049bfd 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ # Core lightweight dependencies only core_deps = [ # Core minor releases may change findings; stay on this reviewed minor. - "datafog-core~=2.0.0", + "datafog-core~=1.5.0", "pydantic>=2.1,<3.0", "pydantic-settings>=2.0.0", "typing-extensions>=4.0", @@ -89,7 +89,7 @@ extras_require = { # Retained for explicitness; the base install already pulls Core. - "rust": ["datafog-core~=2.0.0"], + "rust": ["datafog-core~=1.5.0"], "nlp": nlp_deps, "nlp-advanced": nlp_deps + nlp_advanced_deps, "ocr": ocr_deps, From 3e48c34165514f6095c18dfdb9afd59153b495da Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:25:39 -0800 Subject: [PATCH 20/20] docs(release): finalize 4.12 notes after published Core qualification --- CHANGELOG.MD | 4 +++- README.md | 11 +++++------ RELEASE_NOTES_4.12.0.md | 9 ++++----- docs/review-remediation.md | 27 ++++++++++++++++----------- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 31b49807..86116707 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -2,6 +2,8 @@ ## [Unreleased] +## [4.12.0] + #### Security - Preserve native structured PII detection in smart, spaCy, GLiNER, legacy and @@ -27,7 +29,7 @@ package and `rust` extra; review native differences in `rust-1.5.0.json`. Core's complete mapped-IPv6 spans, email boundaries, Unicode mailboxes and phone-plus-ZIP classification are available through the wrapper. Published - Core artifacts must pass dependent qualification before this candidate ships. + Core artifacts passed dependent installed-wheel qualification. - Raise the Pydantic floor to 2.1, include spaCy in `nlp-advanced`, and allow Torch 2.x versions compatible with `datafog-core[gliner2]`. Add an isolated build declaration and runnable sdist diff --git a/README.md b/README.md index d60ba71c..8c647a1e 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,7 @@ scan/redact helpers, or guardrail helpers. DataFog deprecates OCR and Spark with visible use-time warnings; their APIs and extras will be removed in 5.0. The last supported -version in this release train is 4.12.0 — after publication, pin it with +version in this release train is 4.12.0 — pin it with `pip install "datafog==4.12.0"` — and every retired surface is tracked in the [retirement schedule](docs/retirement.rst). See the [4.9 migration guide](docs/migration-4.9.md) for the transition plan, @@ -288,11 +288,10 @@ Telemetry does not include input text or detected PII values. The native engine is the default, with a warned Python fallback. The 4.12 review fixes preserve structured detection across every engine and legacy API. -This release candidate declares the reviewed Core 1.5 minor series -(`datafog-core~=1.5.0`) in the base package and the `rust` extra. Core 1.5 is a -separate release candidate; this branch does not publish it. Merge and release -require Core 1.5 publication followed by qualification against its actual -published artifacts. Local source-wheel results are additional evidence. +DataFog 4.12 requires the reviewed Core 1.5 minor series +(`datafog-core~=1.5.0`) in the base package and the `rust` extra. The published +Core 1.5.0 wheel passed installed integration and compatibility qualification +with this release. Python and Core have independent release versions. See [release preparation](docs/review-remediation.md). Entity labels, locales and activation settings come from the installed Core. diff --git a/RELEASE_NOTES_4.12.0.md b/RELEASE_NOTES_4.12.0.md index 2682fb84..ad598fd8 100644 --- a/RELEASE_NOTES_4.12.0.md +++ b/RELEASE_NOTES_4.12.0.md @@ -6,7 +6,7 @@ native card, IP and credential findings as the regex mode, then add NER spans. TextService scans complete documents, and spaCy/GLiNER character offsets prevent an earlier substring or repeated name from receiving the wrong redaction. -This candidate requires `datafog-core~=1.5.0` in the base package and the `rust` +This release requires `datafog-core~=1.5.0` in the base package and the `rust` extra. Core 1.5 supplies complete IPv4-mapped IPv6 redaction, corrected email field boundaries and Unicode mailboxes, and phone-plus-ZIP classification that does not suppress the phone behind a false card finding. Native differences @@ -51,7 +51,6 @@ Scheduled release jobs qualify candidates without publishing. Manual jobs default to dry runs, reject stale or occupied versions, and record immutable release refs before uploading artifacts. -This is a release candidate prepared for review. Core 1.5 publication and -qualification of this Python candidate against the actual published Core -artifacts remain mandatory before merge and release. Local source-wheel -qualification does not claim either package has been published. +The published Core 1.5.0 dependency passed installed-wheel qualification with +this Python release, including the native bridge and compatibility contracts. +The Python and Core versions are published independently. diff --git a/docs/review-remediation.md b/docs/review-remediation.md index 6190c821..bd7a67dd 100644 --- a/docs/review-remediation.md +++ b/docs/review-remediation.md @@ -1,6 +1,6 @@ # November 2026 review remediation -This candidate addresses the review of DataFog 4.11.0 and Core 1.4.0. The +DataFog 4.12.0 addresses the review of DataFog 4.11.0 and Core 1.4.0. The review's suggested fixes were assessed against the local implementations; regression tests cover the implemented behavior. @@ -50,20 +50,26 @@ result = datafog.redact("mail jane@example.com", strategy="hash", hash_key=key) ## Release sequence and qualification -The Python 4.12.0 candidate declares `datafog-core~=1.5.0` in its base +Python 4.12.0 declares `datafog-core~=1.5.0` in its base requirements and `rust` extra. Its reviewed native contract is [`tests/contracts/rust-1.5.0.json`](../tests/contracts/rust-1.5.0.json). -Core 1.5.0 is prepared separately as a minor release with the detector and +Core 1.5.0 is published separately as a minor release with the detector and transformation safety fixes from the review, including bounded mask reveal and complete scanner coverage for automatic overlap handling. Public APIs remain available and both packages continue to use capability contract 1. -This branch prepares the dependent package without publishing either project. -Its local source wheel is qualified with the source-built Core 1.5.0 wheel. -Merge and publication remain blocked until Core 1.5.0 is published and this -exact Python candidate passes installed-wheel and release qualification against -the actual published Core artifacts. Source-candidate integration is additional -evidence; it does not substitute for that post-publication gate. +The actual PyPI Core 1.5.0 macOS ARM wheel was downloaded and verified against +its registry SHA-256, +`b3ebaca3c7751ec0288ffebec154ffbb6ba73e54b340f3c4e287e1946bbe5e87`. +All five runtime package files, including the native library, exactly match +the final qualified Core build from source `b183bede189ac35413120b995efcb9a29256b4b0`; +only the build SBOM and RECORD differ. Installing that published wheel into +the isolated Python 4.12 environment passes `pip check`, the installed native +bridge smoke test, and the complete non-slow suite: 1,143 passed, 21 skipped, +19 expected failures. The 726-pass real-model/native corpus qualification +continues to apply through the verified byte-identical runtime payload. +Hosted release gates independently qualify published Core on all required +Python and platform combinations before publishing Python 4.12. Scheduled Python jobs only qualify candidates. Manual `release.yml` runs default to `dry_run: true`; explicitly selecting false is the publication boundary. @@ -87,8 +93,7 @@ build requirements are documented in Core's platform policy. All 35 Python findings have a concrete resolution below. Core-owned fixes are separately tracked in Core's `docs/reviews/2026-11-core-findings.md`; their combined -release still requires Core publication and the published-artifact qualification -above. +release includes the published-artifact qualification above. | Finding | Resolution and proof | | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |