diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index ebaf1b65..3916e8de 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -22,12 +22,12 @@ "docs/operations.md", "src/inferrsi/apps/api.py", "src/inferrsi/apps/archive.py", - "src/inferrsi/apps/workflows/benchmark_compare.py", "src/inferrsi/apps/delivery/__init__.py", "src/inferrsi/apps/handoff/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", "src/inferrsi/apps/storage/artifact_reader.py", "src/inferrsi/apps/views/attempt.py", + "src/inferrsi/apps/workflows/benchmark_compare.py", "tests/check_installed.py", "tests/test_apps_assess.py", "tests/test_apps_module_migrations.py", @@ -37,6 +37,7 @@ "tests/test_apps_publication.py", "tests/test_apps_replay.py", "tests/test_benchmark_preparation.py", + "tests/test_harness_entry_verification.py", "tests/test_result_delivery.py" ] }, @@ -60,7 +61,8 @@ "tests/test_apps_artifact_reader.py", "tests/test_apps_assess.py", "tests/test_apps_generation_plan.py", - "tests/test_apps_module_migrations.py" + "tests/test_apps_module_migrations.py", + "tests/test_harness_entry_verification.py" ] }, { @@ -139,7 +141,9 @@ "tests/test_apps_execution_binding.py", "tests/test_apps_generation_plan.py", "tests/test_apps_module_migrations.py", - "tests/test_benchmark_preparation.py" + "tests/test_benchmark_preparation.py", + "tests/test_harness_entry_verification.py", + "tests/test_harness_prepared_binding.py" ] }, { diff --git a/docs/contracts/harness-entry-verification.md b/docs/contracts/harness-entry-verification.md new file mode 100644 index 00000000..5d3fe199 --- /dev/null +++ b/docs/contracts/harness-entry-verification.md @@ -0,0 +1,65 @@ +# Harness source-entry verification + +The local CPU target can opt into the first frozen evaluation task with +`LocalCpuTarget(..., module_root=)`. The host owns +this configuration. Requests cannot supply the root. Existing targets with no +module root retain their ordinary-command behavior and refuse evaluation mode +before registration. + +The first target supports a standalone source entry named +`module_root/.py` with `entrypoint_kind="python_module"`. It reads a +regular file inside that root, without importing worker code or package +initializers. Dotted packages, bytecode/native modules and native executables +remain unsupported on this evaluation target. No ambient module/path fallback +is attempted. Ordinary-command module/executable resolution is unchanged. + +## Ordering and retained facts + +1. Static shape and fixed-preparation declaration checks run before registration. + Malformed bindings still produce no attempt or record. +2. RunStore registers the request and ordinary admission runs. +3. Harness copies the actual entry into its private attempt capture directory, + independently checks the retained bytes, and compares their SHA-256 with the + expected digest already bound to the frozen preparation. +4. A mismatch yields a fixed `not_started` record. Its + `entry_code_identity` check is `failed`, the runtime binding contains the + observed actual digest, and the check references the retained source bytes. +5. An unavailable/unsupported entry yields a fixed `not_started` record with a + `not_performed` check and no source reference. A source-module binding keeps + the actual digest null and the unresolved argv empty. No mismatch is invented. +6. A matching standalone module executes from its captured source file using the + current Python interpreter with `-I -B`. The workspace and `PYTHONPATH` cannot + replace the bound entry, and subsequent changes to its original source file + do not change the bytes being executed. Package execution is outside this + target's supported range. The runtime binding records the resolved command. + +Runtime bindings describe resolution, not proof of a process launch. On an N1 +mismatch, `argv` is the would-be command over the captured source and the step +remains `not_started`; `started` and step facts determine whether anything ran. +On N2 it is empty because the source command could not be resolved. This target +intentionally binds the frozen recipe's single worker step; declaration checks +reject multiple steps, so each private entry directory holds one snapshot. + +Both refusal cases launch zero worker processes, acquire no process lease, and +retain the preparation/protocol/candidate identity in the fixed record. The +source proof uses `python-source@1`; its SHA is the observed byte identity, not +the frozen expectation copied into an observation. The complete interpreter is +not archived. Re-reading the fixed revision preserves those facts. + +The entry capture is a Harness write. A capture failure uses the existing +`ExecutionPersistenceError` path with `started=false`. Record commit failure +also preserves that known start state and cannot produce a committed RecordRef. +There is no process fallback after an entry failure. + +## Delivery boundary + +These checks implement the post-registration T37-N1/N2 boundary separately from +the pre-registration binding tests. A real matching source entry is a positive +control. CPU execution facts and exit zero are not a Benchmark verdict. + +The production preparation/Apps archive cross-tests use the accepted D023 / +`t108-v4` family and actual source bytes. Their small software worker does not +claim complete F/C/M evidence, a service gain, or whole-Issue production +acceptance. Existing ordinary CPU supervision is reused; evaluation deadline, +budget and T40 acceptance remain the later Harness unit, including its separate +launch-boundary prerequisite. diff --git a/docs/contracts/harness-prepared-binding.md b/docs/contracts/harness-prepared-binding.md new file mode 100644 index 00000000..0f4d7580 --- /dev/null +++ b/docs/contracts/harness-prepared-binding.md @@ -0,0 +1,53 @@ +# Harness preparation binding before registration + +The Harness evaluation path checks declarations before registering an attempt. +`inferrsi.harness.prepared_binding.verify(spec, artifact_reader)` returns a +`FixedPreparation` projection or raises a Harness `PublicError`. The spec must +first pass `execution_service.validate`'s static shape checks; `Harness.execute` +performs both checks in that order. Ordinary command mode reads no preparation. + +The projection's `read(ref, reader)` port obtains a binary stream through +`reader.open(ArtifactRef)`, closes it, bounds reads by the declared size plus one +byte, and independently checks size and SHA-256. Missing bytes or a missing +reader produce `ArtifactUnavailable`; changed bytes or noncanonical JSON +produce `ArtifactIntegrityError`. Unknown host programming errors propagate. +`project(bytes)` is pure: it parses the canonical v1 envelope and validates the +fields used by Harness. It does not read a protocol, resolve an entry or start a +process. A schema version other than v1 is explicitly unsupported. + +Binding compares the semantic prepared ID, protocol digest, operation and scope +with the fixed preparation. Disagreement is `IdentityMismatch`. It also checks: + +- Candidate kind and role, a frozen expected candidate ID/hash/format when + present, and the supported `captured-sha256@1` or `expected-sha256@1` identity + rule. An unknown rule/revision is `UnsupportedRuleVersion`. The expected rule + requires a declared expected input hash even without a frozen candidate. +- A non-null expected code digest on every evaluation step, matching the frozen + recipe. This is an expected declaration, not an observed entry-code hash. +- One worker step preserving runtime role, entry kind/name/revision, all output + declarations, and arguments with whole-argument `{input:role}` / `{output:role}` + expansion. Inputs may use caller-chosen materialized paths; arguments must + match those paths. Paths are relative to the attempt directory (`cwd="."`), + and input/output paths must not overlap. Required frozen capabilities cannot + be dropped; additional declared capabilities remain subject to admission. + +Malformed or conflicting declarations are `InvalidExecutionSpec`. All of these +refusals precede `RunStore.claim`: neither attempt nor record is registered. +The public projection does not interpret candidate formats, grading rules, +evidence requirements or scores, and does not recompute Benchmark's semantic +prepared ID. The fixed ArtifactRef authenticates the preparation bytes; Apps +performs the separate archived-protocol binding through its canonical +`inferrsi.apps.handoff.bridge.verify_protocol` port. + +The D023 conformance vector is kept separately under +`tests/fixtures/harness-prepared-binding/`. It adds the frozen code declaration +and a null step timeout without rewriting historical fixtures. Its canonical +bytes must agree between the Apps binder and Harness `request_digest`. +The vector is documentary; its historical code pin is not an available worker. + +Targets without an explicit `module_root` keep valid evaluation-mode execution +unsupported before registration. A configured source target performs the separate +[post-registration entry check](harness-entry-verification.md) (T37). Attempt +deadlines/budget acceptance with a null step timeout and evaluation replay (T40) +remain separate delivery work. Declaration checks alone do not establish +production CPU acceptance or completion of the governing Harness or Apps Issues. diff --git a/docs/operations.md b/docs/operations.md index 9115ca36..1868dfb2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -52,6 +52,8 @@ Effect classes used below: | Rebuild scoped legacy evidence views | `python tools/rebuild_model_evidence.py --archive --output ` | CPU-only; creates a new output directory | Authenticates original bytes and pinned projection/code; no extraction, network, archived-code execution or campaign qualification | | Inspect evidence inventory / generate budget input | `python tools/validate_evidence.py [--budget-assets]` | Read-only | Offline layout/hash/schema checks; optional disposable #328 metadata view derived from campaign authority; no downloads | | Execute one ordinary local CPU command with a fixed record | Python `inferrsi.harness.api.Harness.allocate` / `execute` / `read` | Experiment-executing / record-writing | One explicit RunStore root and one trusted local CPU target; ordinary command mode only; a record holds execution facts, so exit 0 is not a verdict; no device, model or network fallback | +| Verify a frozen preparation binding before registration | Python `inferrsi.harness.prepared_binding.verify` / `read` / `project` | Read-only / pure projection | Fixed ArtifactRef and read-only stream capability; no registration, entry resolution or execution; [binding contract](contracts/harness-prepared-binding.md) | +| Verify and execute one frozen source evaluation entry | Python `inferrsi.harness.api.Harness.execute` with trusted `LocalCpuTarget(module_root=...)` | Experiment-executing / record-writing | Fixed preparation and expected code digest; actual source captured after registration; T37 refusals start no worker; [source-entry contract](contracts/harness-entry-verification.md) | | Freeze a configured P1 preparation | Python `benchmark.preparation.PreparationService.prepare`; [production prepare](contracts/production-prepare.md) | Read-only resource access / pure encoding | Verifies supplied frozen worker/resource bytes and returns a preparation, protocol and role-keyed closure; Apps separately archives them before publishing a Ref | | Visualize a recorded run or campaign | CLI `dashboard`; [dashboard](dashboard.md) | Report-generating / local read-only serving | Writes an offline HTML snapshot or serves an optional live view on loopback; does not run an agent or validate a claim | | Capture a knowledge context | CLI `campaign capture-context`; [offline dashboard](dashboard.md#record-the-optimization-story-campaign-trace) | Local evidence-writing | Saves assembled context and resource hashes in a new `knowledge/.json`. Supports ordinary questions and explicit IDs. The only accepted explicit `--relevance-method` value is `find-lexical-v1` | diff --git a/src/inferrsi/harness/api.py b/src/inferrsi/harness/api.py index cc69e8f5..3643dc7f 100644 --- a/src/inferrsi/harness/api.py +++ b/src/inferrsi/harness/api.py @@ -43,7 +43,7 @@ def allocate( ) def execute(self, spec: Mapping, *, cancellation=None) -> dict: - """The flat ExecutionRecord of one ordinary command request, or a PublicError. + """The flat ExecutionRecord of one supported local CPU request, or a PublicError. ``cancellation`` is a host handle with ``is_set()``, such as ``threading.Event``. """ diff --git a/src/inferrsi/harness/entry_code.py b/src/inferrsi/harness/entry_code.py new file mode 100644 index 00000000..ef8ed3e6 --- /dev/null +++ b/src/inferrsi/harness/entry_code.py @@ -0,0 +1,60 @@ +"""Capture a trusted local source entry without importing or executing worker code.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +from . import input_binding + +_MODULE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*", re.ASCII) + + +class EntryUnavailable(Exception): + """The supported entry bytes cannot be resolved; no actual digest is known.""" + + def __init__(self, reason: str): + super().__init__(reason) + self.reason = reason + + +class EntryCaptureError(Exception): + """A resolved entry could not be retained safely before execution.""" + + +@dataclass(frozen=True) +class CapturedEntry: + path: Path + sha256: str + size_bytes: int + + +def capture_module(root: Path, module: str, directory: Path) -> CapturedEntry: + """Snapshot one source module from an explicit trusted root. + + The first CPU binding supports a standalone ``.py`` entry. Dotted + package names, bytecode/native modules and import hooks are not resolved by + this port. In particular, neither a package initializer nor a worker module + is imported to locate its code. The caller owns the private capture directory. + """ + if not isinstance(module, str) or _MODULE.fullmatch(module) is None: + raise EntryUnavailable("entry_kind_unsupported") + name = module + ".py" + try: + stream = input_binding.open_regular(root, name) + except (OSError, input_binding.BindingError): + raise EntryUnavailable("entry_bytes_unavailable") from None + destination = directory / name + with stream: + # A capture directory belongs to one registered attempt. Never replace + # an existing entry snapshot on an interrupted or repeated operation. + if destination.exists(): + raise EntryCaptureError("entry snapshot already exists") + try: + digest, size = input_binding.copy_stream(stream, destination) + if input_binding.file_sha256(destination) != (digest, size): + raise EntryCaptureError("entry snapshot differs from the captured bytes") + except OSError as error: + raise EntryCaptureError("entry snapshot could not be retained") from error + return CapturedEntry(destination, digest, size) diff --git a/src/inferrsi/harness/execution_service.py b/src/inferrsi/harness/execution_service.py index edc40a3f..38d756b0 100644 --- a/src/inferrsi/harness/execution_service.py +++ b/src/inferrsi/harness/execution_service.py @@ -12,11 +12,12 @@ from pathlib import Path from inferrsi.contracts import json_v1 -from inferrsi.harness import input_binding, local_process, run_store +from inferrsi.harness import entry_code, input_binding, local_process, prepared_binding, run_store from inferrsi.harness.errors import PublicError -# H02 single local-CPU execute in ordinary command mode (V05 null set). Evaluation mode, -# entry-code digests (T37) and generation chaining are later slices and are refused here. +# The ordinary CPU path preserves V05 null mode. Evaluation requires an explicit trusted +# module root, a fixed preparation binding and an actual source-entry snapshot before start. +# Generation chaining and the remaining evaluation replay/limit acceptance are later slices. ATTEMPT_FIELDS = ( "request_id", "run_id", "attempt_id", "owner", "target_request", "deadline_utc", "resources", "budget", "retry_of", "retry_reason", @@ -42,6 +43,10 @@ ) _OUTPUT_FIELDS = frozenset({"role", "path", "required", "media_type", "format_schema"}) _REF_FIELDS = frozenset({"location", "sha256", "size_bytes", "media_type"}) +_PREPARED_REF_FIELDS = frozenset({"prepared_id", "schema_version", "artifact"}) +_CANDIDATE_FIELDS = frozenset( + {"candidate_id", "kind", "input_role", "parent_id", "parent_digest", "context_refs"} +) _DIMENSION_FIELDS = frozenset({"dimension", "unit", "limit", "reserve_estimate"}) _DEFINITION_FIELDS = frozenset({"name", "revision"}) _DECIMAL = re.compile(r"(?:0|-?[1-9][0-9]*|-?(?:0|[1-9][0-9]*)\.[0-9]*[1-9])", re.ASCII) @@ -52,6 +57,7 @@ _STREAM_MEDIA_TYPE = "application/octet-stream" _INPUT_MEDIA_TYPE = "application/octet-stream" _INPUT_SCHEMA = {"name": "input-identity", "revision": "1"} +_ENTRY_SCHEMA = {"name": "python-source", "revision": "1"} # The one snapshot format this slice captures: the bytes of one regular file. SNAPSHOT_FORMAT = "file-v1" # The only capability this target provides: one command tree in an OS-owned domain. @@ -68,7 +74,10 @@ class LocalCpuTarget: ``work_root`` holds per-attempt workspaces (``workspaces/``) and private capture snapshots. ``controller_root`` / ``target_root`` authorize input sources of - those namespaces; ``None`` leaves that namespace unauthorized. + those namespaces; ``None`` leaves that namespace unauthorized. ``module_root`` + explicitly enables evaluation through standalone source entries under that trusted + root. The retained source is executed directly in an isolated Python interpreter; + package imports and ambient cwd/PYTHONPATH entry resolution are not used. """ target_id: str @@ -76,6 +85,7 @@ class LocalCpuTarget: work_root: Path controller_root: Path | None = None target_root: Path | None = None + module_root: Path | None = None def __post_init__(self) -> None: errors = [ @@ -83,7 +93,7 @@ def __post_init__(self) -> None: for name in ("target_id", "config_revision") if not run_store._is_id(getattr(self, name)) ] - for name in ("work_root", "controller_root", "target_root"): + for name in ("work_root", "controller_root", "target_root", "module_root"): value = getattr(self, name) if value is None: continue @@ -222,6 +232,9 @@ def _step_errors(steps: object, ordinary: bool) -> list[str]: elif ordinary and digest is not None: # V32-N1: ordinary command mode carries no expected entry digest. errors.append(f"{where}.expected_code_sha256: must be null in ordinary command mode") + elif not ordinary and digest is None: + # V32-N2: null never opts an evaluation step out of the frozen code digest. + errors.append(f"{where}.expected_code_sha256: must be set in evaluation mode (V32)") if not (isinstance(step["args"], list) and all(isinstance(a, str) for a in step["args"])): errors.append(f"{where}.args: not a list of Text") if step["cwd"] != "." and not _is_path(step["cwd"]): @@ -265,6 +278,53 @@ def _output_errors(outputs: object, where: str) -> list[str]: return errors +def _evaluation_errors(spec: Mapping) -> list[str]: + """Static shape of the evaluation fields; the fixed preparation is read separately.""" + errors: list[str] = [] + ref = spec["prepared_ref"] + if not run_store._has_fields(ref, _PREPARED_REF_FIELDS): + errors.append("prepared_ref: fields must be exactly the PreparedRef fields") + else: + if not prepared_binding._is_semantic_id(ref["prepared_id"]): + errors.append("prepared_ref.prepared_id: not a SemanticId") + if type(ref["schema_version"]) is not int or ref["schema_version"] != 1: + errors.append("prepared_ref.schema_version: must be the integer 1") + if not _is_artifact_ref(ref["artifact"]): + errors.append("prepared_ref.artifact: not an ArtifactRef") + elif ref["artifact"]["media_type"] != prepared_binding.PREPARED_MEDIA_TYPE: + errors.append("prepared_ref.artifact.media_type: must be application/json") + if not prepared_binding._is_semantic_id(spec["protocol_digest"]): + errors.append("protocol_digest: not a SemanticId") + for name in ("operation", "validation_scope"): + if not run_store._is_id(spec[name]): + errors.append(f"{name}: not an Id") + candidate = spec["candidate"] + if not run_store._has_fields(candidate, _CANDIDATE_FIELDS): + errors.append("candidate: fields must be exactly the CandidateSubmission fields") + return errors + for name in ("candidate_id", "kind", "input_role"): + if not run_store._is_id(candidate[name]): + errors.append(f"candidate.{name}: not an Id") + # V02: parent id and digest are both null or both set. + if (candidate["parent_id"] is None) != (candidate["parent_digest"] is None): + errors.append("candidate.parent_id: parent_id and parent_digest must be paired") + elif candidate["parent_id"] is not None and not ( + run_store._is_id(candidate["parent_id"]) and prepared_binding._is_sha256( + candidate["parent_digest"] + ) + ): + errors.append("candidate.parent_id: not an Id with a Sha256 parent_digest") + refs = candidate["context_refs"] + if not isinstance(refs, list) or not all(_is_artifact_ref(item) for item in refs): + errors.append("candidate.context_refs: not a list of ArtifactRef") + # V03: the candidate role names exactly one InputSpec. + inputs = spec["inputs"] if isinstance(spec["inputs"], list) else [] + roles = [item["role"] for item in inputs if isinstance(item, Mapping) and "role" in item] + if roles.count(candidate["input_role"]) != 1: + errors.append("candidate.input_role: must name exactly one input role (V03)") + return errors + + def _request_errors(spec: Mapping) -> list[str]: """Shape and resource/budget field errors beyond the AttemptContext rules.""" errors: list[str] = [] @@ -304,8 +364,12 @@ def _request_errors(spec: Mapping) -> list[str]: return errors -def validate(spec: object, target: LocalCpuTarget) -> None: - """Refuse a request before registration (D021 priority: version, fields, support).""" +def validate(spec: object, target: LocalCpuTarget, *, artifact_reader=None) -> None: + """Refuse a request before registration (D021 priority: version, fields, support). + + An evaluation-mode spec is also checked against its fixed preparation, read through + ``artifact_reader`` (V05, V28, V32). Nothing here registers an attempt or writes a record. + """ if isinstance(spec, Mapping): version = spec.get("schema_version") if type(version) is int and version != 1: @@ -323,8 +387,11 @@ def validate(spec: object, target: LocalCpuTarget) -> None: errors += run_store._attempt_errors({name: spec[name] for name in ATTEMPT_FIELDS}) errors += _request_errors(spec) evaluation = [name for name in _EVALUATION_FIELDS if spec[name] is not None] - if evaluation and len(evaluation) != len(_EVALUATION_FIELDS): + complete = len(evaluation) == len(_EVALUATION_FIELDS) + if evaluation and not complete: errors.append("prepared_ref: evaluation fields must be all null or all set (V05)") + elif complete: + errors += _evaluation_errors(spec) errors += _input_errors(spec["inputs"]) errors += _step_errors(spec["steps"], ordinary=not evaluation) try: @@ -336,9 +403,13 @@ def validate(spec: object, target: LocalCpuTarget) -> None: "InvalidExecutionSpec", stage="validate", message="illegal ExecutionSpec", field_errors=errors, ) - unsupported = [] if evaluation: - unsupported.append("evaluation mode is not implemented in this slice") + # Malformed or inconsistent bindings are refused here, before the RunStore sees the + # request. The reader is the read-only port; H does not write or register anything. + prepared_binding.verify(spec, artifact_reader) + unsupported = [] + if evaluation and target.module_root is None: + unsupported.append("this target has no configured evaluation module root") if spec["generation_record_ref"] is not None: unsupported.append("generation chaining is not implemented in this slice") request = spec["target_request"] @@ -510,8 +581,8 @@ def _capture_input(ctx: _Context, index: int, item: Mapping, target: LocalCpuTar "checker_revision": "1", "subject": item["role"], "phase": "preparing", - "operation": None, - "validation_scope": None, + "operation": ctx.spec["operation"], + "validation_scope": ctx.spec["validation_scope"], "attempt_id": ctx.attempt_id, "status": {"matched": "passed", "mismatched": "failed", "not_performed": "not_performed", "error": "execution_error"}[status], @@ -532,6 +603,48 @@ def _argv(step: Mapping) -> list[str] | None: return None if executable is None else [executable, *step["args"]] +def _bind_entry(ctx: _Context, step: Mapping, target: LocalCpuTarget): + """Observe/capture the supported entry after registration, before workspace execution.""" + actual, ref, argv = None, None, [] + reason = "entry_kind_unsupported" + if step["entrypoint_kind"] == "python_module": + try: + captured = entry_code.capture_module( + target.module_root, step["entrypoint"], ctx.captures / "entry", + ) + except entry_code.EntryUnavailable as error: + reason = error.reason + except entry_code.EntryCaptureError as error: + raise _RetentionFailure("entry capture failed") from error + else: + actual = captured.sha256 + ref = ctx.object_ref(captured.path, actual, captured.size_bytes, "text/x-python") + # Execute exactly the retained source, with no cwd/PYTHONPATH module fallback. + # This first target supports standalone source modules, not Python packages. + argv = [sys.executable, "-I", "-B", str(captured.path), *step["args"]] + reason = ("entry_code_matched" if actual == step["expected_code_sha256"] + else "entry_code_sha256_differs") + status = ("not_performed" if actual is None else + "passed" if actual == step["expected_code_sha256"] else "failed") + runtime = { + "step_id": step["step_id"], "runtime_role": step["runtime_role"], + "executable": sys.executable, "entrypoint": step["entrypoint"], + "entrypoint_revision": step["entrypoint_revision"], "code_sha256": actual, + "argv": argv, "environment_identity": None, + } if step["entrypoint_kind"] == "python_module" else None + check = { + "check_id": "entry_code_identity", "checker_revision": "1", "subject": step["step_id"], + "phase": "preparing", "operation": ctx.spec["operation"], + "validation_scope": ctx.spec["validation_scope"], "attempt_id": ctx.attempt_id, + "status": status, "producer": { + "kind": "harness_capture", "identity": _HARNESS, "revision": "1", + "control_domain": _CONTROL_DOMAIN, "capture_route": "entry-source-snapshot", + }, "evidence_refs": [] if ref is None else [ref], "artifact_schema": _ENTRY_SCHEMA, + "reasons": [reason], + } + return argv, runtime, check + + def _retain_stream(ctx: _Context, stream, name: str, media_type: str) -> dict | None: """Snapshot one produced file from its open stream; None when the copy fails.""" target = ctx.captures / name @@ -780,12 +893,12 @@ def _payload(ctx: _Context, **facts) -> dict: "target_binding": facts["target_binding"], "retry_of": spec["retry_of"], "retry_reason": spec["retry_reason"], - "generation_record_ref": None, - "prepared_ref": None, - "protocol_digest": None, - "operation": None, - "validation_scope": None, - "candidate": None, + "generation_record_ref": copy.deepcopy(spec["generation_record_ref"]), + "prepared_ref": copy.deepcopy(spec["prepared_ref"]), + "protocol_digest": spec["protocol_digest"], + "operation": spec["operation"], + "validation_scope": spec["validation_scope"], + "candidate": copy.deepcopy(spec["candidate"]), "record_version": 1, "recorded_at": run_store.utc_now(), "bound_inputs": facts["bound_inputs"], @@ -839,7 +952,22 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: binding = _target_binding(target, capability_checks, []) if errors: return _refusal(ctx, digest, binding, admission, [], [], errors, "admission_rejected") - argvs = [_argv(step) for step in ctx.spec["steps"]] + evaluation = ctx.spec["prepared_ref"] is not None + entry_checks = [] + if evaluation: + entries = [_bind_entry(ctx, step, target) for step in ctx.spec["steps"]] + argvs = [entry[0] for entry in entries] + binding["runtime_bindings_by_step"] = [ + entry[1] for entry in entries if entry[1] is not None + ] + entry_checks = [entry[2] for entry in entries] + if any(check["status"] != "passed" for check in entry_checks): + code = ("entry_code_mismatch" if any(check["status"] == "failed" + for check in entry_checks) else "entrypoint_unavailable") + return _refusal(ctx, digest, binding, admission, [], entry_checks, + [_error(code, "binding", "evaluation entry binding refused")], code) + else: + argvs = [_argv(step) for step in ctx.spec["steps"]] if None in argvs: errors = [_error("entrypoint_unavailable", "binding", "an executable entry is not found")] return _refusal(ctx, digest, binding, admission, [], [], errors, "entrypoint_unavailable") @@ -847,7 +975,9 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: ctx.workspace.mkdir(parents=True) except FileExistsError: errors = [_error("workspace_exists", "binding", "the attempt workspace already exists")] - return _refusal(ctx, digest, binding, admission, [], [], errors, "workspace_exists") + return _refusal( + ctx, digest, binding, admission, [], entry_checks, errors, "workspace_exists", + ) except OSError as exc: raise _RetentionFailure("workspace creation failed") from exc captured = [ @@ -855,7 +985,7 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: for index, item in enumerate(ctx.spec["inputs"]) ] bound = [item[0] for item in captured] - checks = [item[1] for item in captured] + checks = entry_checks + [item[1] for item in captured] failed = [b for b in bound if b["identity_status"] in ("mismatched", "error")] if failed: # H02-3: the actual binding is refused after registration and before any start. @@ -878,7 +1008,7 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: outcome, exit_code = _outcome(steps) cleanup, leases = _cleanup(runs) launched = {step["step_id"] for step, run in runs if run["pid"] is not None} - binding["runtime_bindings_by_step"] = [ + ordinary_runtime = [ { "step_id": step["step_id"], "runtime_role": step["runtime_role"], @@ -893,6 +1023,8 @@ def _run_claimed(ctx, target, digest, reader, deadline, cancellation) -> dict: for step, argv in zip(ctx.spec["steps"], argvs) if step["step_id"] in launched ] + if not evaluation: + binding["runtime_bindings_by_step"] = ordinary_runtime codes = {"timed_out": "timeout", "cancelled": "cancelled", "execution_error": "execution_error", "not_started": "deadline_exhausted"} run_errors = ( @@ -920,7 +1052,7 @@ def execute( artifact_reader=None, ) -> dict: """H ``execute``: register, admit, bind, run and commit one ordinary command request.""" - validate(spec, target) + validate(spec, target, artifact_reader=artifact_reader) digest = run_store.request_digest(spec) attempt = {name: copy.deepcopy(spec[name]) for name in ATTEMPT_FIELDS} # Re-entry is decided by the store before any binding or start (V14 / D021 section 3). diff --git a/src/inferrsi/harness/prepared_binding.py b/src/inferrsi/harness/prepared_binding.py new file mode 100644 index 00000000..dbeb6709 --- /dev/null +++ b/src/inferrsi/harness/prepared_binding.py @@ -0,0 +1,363 @@ +"""H's read-only binding projection of a fixed PreparedEvaluation (D021 section 1, V28, V32). + +An evaluation-mode ExecutionSpec names its preparation by PreparedRef. Before any +registration, H reads the fixed bytes through the read-only artifact port, verifies them +against the Ref and compares the fields it needs with the spec: ids, digests, operation and +scope, the candidate declaration and the frozen worker declarations. The projection is defined +here. It imports nothing from Benchmark or Apps, and it never interprets task rules, formats +or scores. + +Nothing in this module registers an attempt, writes a record or starts a process. Every +refusal is raised before the RunStore sees the request. Verifying the actual entry bytes +against the frozen digest (T37) happens after registration and is not part of this module. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Mapping +from dataclasses import dataclass + +from inferrsi.contracts import json_v1 +from inferrsi.harness import input_binding, run_store +from inferrsi.harness.errors import PublicError + +STAGE = "prepared_binding" +PREPARED_MEDIA_TYPE = "application/json" +# The candidate identity rules this H supports (V28). Any other is refused, never downgraded. +CAPTURED_SHA256 = {"name": "captured-sha256", "revision": "1"} +EXPECTED_SHA256 = {"name": "expected-sha256", "revision": "1"} + +# Exact v1 field sets of the objects the projection reads. An unknown field is refused and +# never dropped (V01). +_PREPARED_FIELDS = frozenset( + { + "schema_version", "prepared_id", "task", "grader", "operation", "validation_scope", + "protocol", "protocol_digest", "candidate_contract", "expected_candidate", + "required_evidence", "recipe", "resources", "created_at", "display_notes", + } +) +_CONTRACT_FIELDS = frozenset( + { + "kind", "input_role", "format_schema", "entrypoint", "required_paths", + "allowed_content", "identity_rule", "modification_receipt_required", + } +) +_RECIPE_FIELDS = frozenset( + { + "recipe_revision", "runtime_role", "entrypoint_kind", "entrypoint", + "entrypoint_revision", "code_sha256", "args", "input_roles", "outputs", + "required_capabilities", + } +) +_EXPECTED_FIELDS = frozenset({"candidate_id", "sha256", "format_schema"}) +_DEFINITION_FIELDS = frozenset({"name", "revision"}) +_OUTPUT_FIELDS = frozenset({"role", "path", "required", "media_type", "format_schema"}) +_ARTIFACT_FIELDS = frozenset({"location", "sha256", "size_bytes", "media_type"}) +_PLACEHOLDER = re.compile(r"\{(input|output):([^{}]+)\}") +_SEMANTIC_ID = re.compile(r"sha256:[0-9a-f]{64}", re.ASCII) +_SHA256 = re.compile(r"[0-9a-f]{64}", re.ASCII) +_READ_CHUNK = 1024 * 1024 + + +@dataclass(frozen=True) +class FixedPreparation: + """H's mechanical identity/binding projection; task rules and scores are not retained.""" + + prepared_id: str + operation: str + validation_scope: str + protocol_digest: str + candidate_kind: str + candidate_input_role: str + candidate_format: dict + identity_rule: dict + expected_candidate: dict | None + code_sha256: str + recipe: dict + + +def _refuse(code: str, message: str, field_errors: list[str] | None = None) -> PublicError: + return PublicError(code, stage=STAGE, message=message, field_errors=field_errors or []) + + +def _is_definition(value: object) -> bool: + return run_store._has_fields(value, _DEFINITION_FIELDS) and all( + run_store._is_id(value[name]) for name in _DEFINITION_FIELDS + ) + + +def _is_sha256(value: object) -> bool: + return isinstance(value, str) and _SHA256.fullmatch(value) is not None + + +def _is_semantic_id(value: object) -> bool: + return isinstance(value, str) and _SEMANTIC_ID.fullmatch(value) is not None + + +def _is_ids(value: object) -> bool: + return (isinstance(value, list) and all(run_store._is_id(item) for item in value) + and len(set(value)) == len(value)) + + +def _is_path(value: object) -> bool: + return isinstance(value, str) and input_binding.RELATIVE_PATH.fullmatch(value) is not None + + +def _recipe_errors(recipe: object) -> list[str]: + """Validate only mechanical worker declarations, without interpreting formats or rules.""" + if not run_store._has_fields(recipe, _RECIPE_FIELDS): + return ["prepared.recipe: not the OperationRecipe fields"] + errors = [] + for name in ("recipe_revision", "runtime_role", "entrypoint_revision"): + if not run_store._is_id(recipe[name]): + errors.append(f"prepared.recipe.{name}: not an Id") + if recipe["entrypoint_kind"] not in ("python_module", "executable"): + errors.append("prepared.recipe.entrypoint_kind: not a supported entry kind") + if not isinstance(recipe["entrypoint"], str) or not recipe["entrypoint"]: + errors.append("prepared.recipe.entrypoint: not a non-empty Text") + if not _is_sha256(recipe["code_sha256"]): + errors.append("prepared.recipe.code_sha256: not a Sha256") + for name in ("input_roles", "required_capabilities"): + if not _is_ids(recipe[name]): + errors.append(f"prepared.recipe.{name}: not a list of unique Ids") + outputs = recipe["outputs"] + if not isinstance(outputs, list) or not all( + run_store._has_fields(item, _OUTPUT_FIELDS) + and run_store._is_id(item["role"]) and _is_path(item["path"]) + and type(item["required"]) is bool + and isinstance(item["media_type"], str) and bool(item["media_type"]) + and _is_definition(item["format_schema"]) for item in outputs + ): + errors.append("prepared.recipe.outputs: not a list of OutputSpec") + elif (len({item["role"] for item in outputs}) != len(outputs) + or len({item["path"] for item in outputs}) != len(outputs)): + errors.append("prepared.recipe.outputs: roles and paths must be unique") + args = recipe["args"] + if not isinstance(args, list) or not all(isinstance(arg, str) for arg in args): + errors.append("prepared.recipe.args: not a list of Text") + elif not errors: + roles = {"input": set(recipe["input_roles"]), + "output": {item["role"] for item in outputs}} + for arg in args: + match = _PLACEHOLDER.fullmatch(arg) + if match: + if match[2] not in roles[match[1]]: + errors.append("prepared.recipe.args: unknown placeholder role") + elif "{input:" in arg or "{output:" in arg: + errors.append("prepared.recipe.args: placeholders must occupy a whole argument") + return errors + + +def read(ref: Mapping, reader) -> bytes: + """The fixed bytes of ``ref`` from the read-only port, verified against the Ref. + + The port's own verification is not trusted: size and hash are checked here again + (V06). At most ``size_bytes + 1`` bytes are read, so a wrong Ref cannot make this + allocate more than the stream actually holds. + """ + if not (run_store._has_fields(ref, _ARTIFACT_FIELDS) and _is_path(ref["location"]) + and _is_sha256(ref["sha256"]) and type(ref["size_bytes"]) is int + and 0 <= ref["size_bytes"] <= json_v1.MAX_SAFE_INTEGER + and ref["media_type"] == PREPARED_MEDIA_TYPE): + raise _refuse("InvalidExecutionSpec", "illegal fixed preparation ArtifactRef") + if reader is None: + raise _refuse( + "ArtifactUnavailable", "no read-only artifact reader is configured for the " + "fixed preparation", + ) + size = ref["size_bytes"] + expected_sha256 = ref["sha256"] + digest = hashlib.sha256() + parts: list[bytes] = [] + total = 0 + try: + stream = reader.open(dict(ref)) + with stream: + while total <= size: + chunk = stream.read(min(_READ_CHUNK, size + 1 - total)) + if not chunk: + break + digest.update(chunk) + parts.append(chunk) + total += len(chunk) + except PublicError: + raise + except OSError as exc: + raise _refuse( + "ArtifactUnavailable", + f"the fixed preparation could not be read: {run_store._describe(exc)}", + ) from exc + if total != size or digest.hexdigest() != expected_sha256: + raise _refuse( + "ArtifactIntegrityError", "the fixed preparation bytes differ from their ArtifactRef" + ) + return b"".join(parts) + + +def project(data: bytes) -> FixedPreparation: + """Parse the identity fields of canonical PreparedEvaluation bytes.""" + try: + prepared = json_v1.decode(data) + except json_v1.CanonicalError as exc: + raise _refuse( + "ArtifactIntegrityError", + f"the fixed preparation is not canonical inferrsi-json-v1 ({exc.stage})", + ) from exc + version = prepared.get("schema_version") if isinstance(prepared, dict) else None + if type(version) is int and version != 1: + raise _refuse("UnsupportedSchema", "only PreparedEvaluation v1 is supported") + errors: list[str] = [] + if not run_store._has_fields(prepared, _PREPARED_FIELDS): + raise _refuse( + "InvalidExecutionSpec", "illegal fixed preparation", + ["prepared: fields must be exactly the PreparedEvaluation fields"], + ) + if type(version) is not int: + errors.append("prepared.schema_version: must be the integer 1") + if not _is_semantic_id(prepared["prepared_id"]): + errors.append("prepared.prepared_id: not a SemanticId") + if not _is_semantic_id(prepared["protocol_digest"]): + errors.append("prepared.protocol_digest: not a SemanticId") + for name in ("operation", "validation_scope"): + if not run_store._is_id(prepared[name]): + errors.append(f"prepared.{name}: not an Id") + contract, recipe = prepared["candidate_contract"], prepared["recipe"] + expected = prepared["expected_candidate"] + if not run_store._has_fields(contract, _CONTRACT_FIELDS): + errors.append("prepared.candidate_contract: not the CandidateContract fields") + else: + for name in ("kind", "input_role"): + if not run_store._is_id(contract[name]): + errors.append(f"prepared.candidate_contract.{name}: not an Id") + for name in ("format_schema", "identity_rule"): + if not _is_definition(contract[name]): + errors.append(f"prepared.candidate_contract.{name}: not a DefinitionRef") + errors.extend(_recipe_errors(recipe)) + if expected is not None: + if not run_store._has_fields(expected, _EXPECTED_FIELDS): + errors.append( + "prepared.expected_candidate: not null or the CandidateExpectation fields" + ) + else: + if not run_store._is_id(expected["candidate_id"]): + errors.append("prepared.expected_candidate.candidate_id: not an Id") + if not _is_sha256(expected["sha256"]): + errors.append("prepared.expected_candidate.sha256: not a Sha256") + if not _is_definition(expected["format_schema"]): + errors.append("prepared.expected_candidate.format_schema: not a DefinitionRef") + if errors: + raise _refuse("InvalidExecutionSpec", "illegal fixed preparation", errors) + return FixedPreparation( + prepared_id=prepared["prepared_id"], + operation=prepared["operation"], + validation_scope=prepared["validation_scope"], + protocol_digest=prepared["protocol_digest"], + candidate_kind=contract["kind"], + candidate_input_role=contract["input_role"], + candidate_format=contract["format_schema"], + identity_rule=contract["identity_rule"], + expected_candidate=expected, + code_sha256=recipe["code_sha256"], + recipe=recipe, + ) + + +def _check_identity(spec: Mapping, fixed: FixedPreparation) -> None: + """V17: top-level identity fields must agree before any declaration is compared.""" + pairs = ( + ("prepared_ref.prepared_id", spec["prepared_ref"]["prepared_id"], fixed.prepared_id), + ("protocol_digest", spec["protocol_digest"], fixed.protocol_digest), + ("operation", spec["operation"], fixed.operation), + ("validation_scope", spec["validation_scope"], fixed.validation_scope), + ) + differing = [f"{name}: differs from the fixed preparation" for name, a, b in pairs if a != b] + if differing: + raise _refuse( + "IdentityMismatch", "the execution spec names a different preparation", differing + ) + + +def _check_declarations(spec: Mapping, fixed: FixedPreparation) -> None: + """V28 and V32: declaration conflicts are refused before registration, never recorded.""" + errors = [ + f"steps[{index}].expected_code_sha256: must equal the frozen recipe code_sha256 (V32)" + for index, step in enumerate(spec["steps"]) + if step["expected_code_sha256"] != fixed.code_sha256 + ] + errors.extend(_worker_errors(spec, fixed.recipe)) + candidate = spec["candidate"] + if candidate["input_role"] != fixed.candidate_input_role: + errors.append("candidate.input_role: differs from the frozen candidate contract") + if candidate["kind"] != fixed.candidate_kind: + errors.append("candidate.kind: differs from the frozen candidate contract") + declared = next( + (item["expected_sha256"] for item in spec["inputs"] + if item["role"] == candidate["input_role"]), + None, + ) + if fixed.identity_rule == EXPECTED_SHA256 and declared is None: + errors.append( + "inputs: the candidate input needs expected_sha256 under expected-sha256@1 (V28)" + ) + frozen = fixed.expected_candidate + if frozen is not None: + if candidate["candidate_id"] != frozen["candidate_id"]: + errors.append("candidate.candidate_id: differs from the frozen expected candidate") + if declared != frozen["sha256"]: + errors.append("inputs: the candidate expected_sha256 differs from the frozen one") + if fixed.candidate_format != frozen["format_schema"]: + errors.append("prepared: expected candidate format differs from the candidate contract") + if errors: + raise _refuse( + "InvalidExecutionSpec", "the execution spec conflicts with its fixed preparation", + errors, + ) + + +def _worker_errors(spec: Mapping, recipe: dict) -> list[str]: + """Preserve the frozen worker's entry, paths and placeholder-expanded arguments.""" + errors = [] + if len(spec["steps"]) != 1: + return ["steps: the frozen recipe binds exactly one worker step"] + step = spec["steps"][0] + for name in ("runtime_role", "entrypoint_kind", "entrypoint", "entrypoint_revision", "outputs"): + if step[name] != recipe[name]: + errors.append(f"steps[0].{name}: differs from the frozen worker declaration") + if step["cwd"] != ".": + errors.append("steps[0].cwd: frozen recipe paths are relative to the attempt directory") + if not set(recipe["required_capabilities"]) <= set(spec["required_capabilities"]): + errors.append("required_capabilities: missing a frozen worker requirement") + inputs = {item["role"]: item["materialized_path"] for item in spec["inputs"]} + if not set(recipe["input_roles"]) <= inputs.keys(): + errors.append("inputs: missing a frozen worker input role") + else: + outputs = {item["role"]: item["path"] for item in recipe["outputs"]} + paths = {"input": inputs, "output": outputs} + args = [] + for arg in recipe["args"]: + match = _PLACEHOLDER.fullmatch(arg) + args.append(paths[match[1]][match[2]] if match else arg) + if step["args"] != args: + errors.append("steps[0].args: differs from the frozen recipe binding") + ordered = sorted(tuple(path.split("/")) for path in [*inputs.values(), *outputs.values()]) + if any(b[:len(a)] == a for a, b in zip(ordered, ordered[1:])): + errors.append("inputs/outputs: materialized paths must not overlap") + return errors + + +def verify(spec: Mapping, reader) -> FixedPreparation: + """Check an evaluation-mode spec against its fixed preparation before registration. + + ``spec`` has passed the static ExecutionSpec checks, so its evaluation fields are all + set and well formed. Raises a PublicError for any malformed or inconsistent binding. + """ + fixed = project(read(spec["prepared_ref"]["artifact"], reader)) + if fixed.identity_rule not in (CAPTURED_SHA256, EXPECTED_SHA256): + raise _refuse( + "UnsupportedRuleVersion", "the frozen candidate identity rule is unsupported" + ) + _check_identity(spec, fixed) + _check_declarations(spec, fixed) + return fixed diff --git a/tests/check_installed.py b/tests/check_installed.py index 5b02b0d9..f6da835c 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -27,6 +27,16 @@ from inferrsi.resources import repository_packs from inferrsi.resources.repository_packs import REPO_TYPES, repository_pack +# The new Harness binding port ships independently of Apps and Benchmark. +subprocess.run([ + sys.executable, "-I", "-B", "-c", + "import sys; from inferrsi.harness import prepared_binding as p; " + "from inferrsi.harness import entry_code; assert callable(entry_code.capture_module); " + "assert all(callable(getattr(p, n)) for n in ('read', 'project', 'verify')); " + "assert not any(n == root or n.startswith(root + '.') for n in sys.modules " + "for root in ('inferrsi.apps', 'inferrsi.benchmark', 'inferrsi.knowledge'))", +], check=True, capture_output=True, text=True) + assert Path(inferrsi.__file__).resolve().parent.parent.name != "src", ( "This check requires an installed package" ) diff --git a/tests/fixtures/harness-prepared-binding/README.md b/tests/fixtures/harness-prepared-binding/README.md new file mode 100644 index 00000000..5d6e1789 --- /dev/null +++ b/tests/fixtures/harness-prepared-binding/README.md @@ -0,0 +1,16 @@ +# D023 null-timeout binding vector + +`d023-null-timeout.json` is derived from the unchanged canonical +`docs/contracts/fixtures/t108-v4/execution-spec.json`: add the frozen preparation's +`recipe.code_sha256` as `steps[0].expected_code_sha256`, and set +`steps[0].timeout_seconds` to null. The accepted #429 PR-1 vector is 2069 bytes, +with request digest +`sha256:62ce11e6d458b5c581245d0988189e1bd834f2ed1e1880d15242faf70c5b6fe9`. + +This is immutable contract-conformance input, not observed execution evidence or +an available worker implementation. Tests compare the canonical Apps binder, +Harness digest and read-only prepared projection. No process is started by these +binding tests. Post-registration T37 is covered separately in +`tests/test_harness_entry_verification.py`; deadline/budget and production T40 +replay remain PR-3 work. +The original documentary fixtures are not rewritten. diff --git a/tests/fixtures/harness-prepared-binding/d023-null-timeout.json b/tests/fixtures/harness-prepared-binding/d023-null-timeout.json new file mode 100644 index 00000000..da758ae6 --- /dev/null +++ b/tests/fixtures/harness-prepared-binding/d023-null-timeout.json @@ -0,0 +1 @@ +{"attempt_id":"att-1","budget":{"declared_by":"fixture","dimensions":[],"policy_ref":null,"required_enforcement":[],"scope_id":"campaign-1"},"candidate":{"candidate_id":"cand-1","context_refs":[],"input_role":"candidate","kind":"text","parent_digest":null,"parent_id":null},"deadline_utc":"2099-01-01T00:00:00Z","generation_record_ref":null,"inputs":[{"expected_sha256":"e39494a6eed78161414f7afdef602f86ff19de6efa9ca9053407d548981fe01f","materialized_path":"inputs/candidate.txt","role":"candidate","snapshot_format":"file-v1","source":"fixtures/candidate.txt","source_namespace":"artifact","source_ref":{"location":"fixtures/candidate.txt","media_type":"text/plain","sha256":"e39494a6eed78161414f7afdef602f86ff19de6efa9ca9053407d548981fe01f","size_bytes":18}}],"operation":"evaluate","output_root":"attempts/att-1","owner":"fixture","prepared_ref":{"artifact":{"location":"fixtures/prepared.json","media_type":"application/json","sha256":"8450cf590172a785282fed31480eb82bb192f4e9a72c73dd70d0b5ea9a3d30ff","size_bytes":2690},"prepared_id":"sha256:7f481e7e61c98343015118d7dcd98806abe863398b2c16dacac125d2d00ed486","schema_version":1},"protocol_digest":"sha256:497290081c4f3d3cab0c50c42a0be1c467e1a840dcf680eaace602311c46a80c","request_id":"att-1","required_capabilities":["local-process-tree"],"resources":{"cpu_count":1,"memory_bytes":null,"required_capabilities":["local-process-tree"]},"retry_of":null,"retry_reason":null,"run_id":"run-1","schema_version":1,"steps":[{"args":["inputs/candidate.txt","outputs/grader.json"],"cwd":".","entrypoint":"fixture_worker","entrypoint_kind":"python_module","entrypoint_revision":"1","expected_code_sha256":"ba351e8f96c162ed9c32309f24b1d2d75d11a7369168571ebbb0a79d13315332","outputs":[{"format_schema":{"name":"fixture-grader","revision":"1"},"media_type":"application/json","path":"outputs/grader.json","required":true,"role":"grader"}],"runtime_role":"local-python","step_id":"main","timeout_seconds":null}],"target_request":{"config_revision":"1","executor_kind":"local_cpu","target_id":"fixture-cpu"},"validation_scope":"p1"} \ No newline at end of file diff --git a/tests/test_harness_entry_code.py b/tests/test_harness_entry_code.py new file mode 100644 index 00000000..9af659f8 --- /dev/null +++ b/tests/test_harness_entry_code.py @@ -0,0 +1,39 @@ +"""Actual entry-byte capture is separate from preparation declaration checks.""" + +from hashlib import sha256 + +import pytest + +from inferrsi.harness import entry_code + + +def test_capture_reads_actual_source_without_importing_or_executing_it(tmp_path): + root = tmp_path / "trusted" + root.mkdir() + source = b"raise RuntimeError('this worker must never be imported during binding')\n" + (root / "fixture_worker.py").write_bytes(source) + result = entry_code.capture_module(root, "fixture_worker", tmp_path / "captures") + assert result.path.read_bytes() == source + assert (result.sha256, result.size_bytes) == (sha256(source).hexdigest(), len(source)) + (root / "fixture_worker.py").write_bytes(b"changed after capture\n") + assert result.path.read_bytes() == source + + +@pytest.mark.parametrize("module", ["missing", "package.worker", "../outside", "worker.py"]) +def test_unresolvable_entry_has_no_actual_digest_or_capture(tmp_path, module): + with pytest.raises(entry_code.EntryUnavailable): + entry_code.capture_module(tmp_path, module, tmp_path / "captures") + assert not (tmp_path / "captures").exists() + + +def test_existing_snapshot_is_preserved_instead_of_replaced(tmp_path): + root = tmp_path / "trusted" + root.mkdir() + (root / "worker.py").write_bytes(b"new bytes") + captures = tmp_path / "captures" + captures.mkdir() + retained = captures / "worker.py" + retained.write_bytes(b"already retained") + with pytest.raises(entry_code.EntryCaptureError): + entry_code.capture_module(root, "worker", captures) + assert retained.read_bytes() == b"already retained" diff --git a/tests/test_harness_entry_verification.py b/tests/test_harness_entry_verification.py new file mode 100644 index 00000000..d47c3302 --- /dev/null +++ b/tests/test_harness_entry_verification.py @@ -0,0 +1,240 @@ +"""T37 happens after registration; declaration-only refusals remain in their own test file.""" + +import json +import sqlite3 +import subprocess +import sys +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from inferrsi.apps import api +from inferrsi.apps.handoff.bridge import bind_execution +from inferrsi.apps.storage.archive import read_prepared +from inferrsi.apps.storage.artifact_reader import ArtifactReader +from inferrsi.benchmark import preparation as benchmark +from inferrsi.contracts.json_v1 import decode +from inferrsi.harness import execution_service, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from inferrsi.harness.errors import PublicError +from tests.test_benchmark_preparation import preparation as preparation + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def evaluation(preparation, tmp_path): + service, request, sources, _, _, config, _ = preparation + output = api.prepare( + {"schema_version": 1, "request": request}, workspace=tmp_path, + storage_config=config, prepare_benchmark=service.prepare, + encode_prepared=benchmark.encode, validate_prepared=benchmark.validate_prepared, + ) + assert output.result["exit_code"] == 0 + ref = output.result["prepared_ref"] + prepared = read_prepared(config, ref) + vector = decode((ROOT / "tests/fixtures/harness-prepared-binding/d023-null-timeout.json") + .read_bytes()) + controller, modules = tmp_path / "controller", tmp_path / "modules" + controller.mkdir() + modules.mkdir() + source = sources["worker-code"].read_bytes() + (modules / "fixture_worker.py").write_bytes(source) + candidate = b"raise RuntimeError('candidate input must never replace the trusted worker')\n" + (controller / "candidate.txt").write_bytes(candidate) + proposal = {key: deepcopy(vector[key]) + for key in ("candidate", "inputs", "generation_record_ref")} + proposal["inputs"][0].update( + source="candidate.txt", source_namespace="controller", source_ref=None, + expected_sha256=sha256(candidate).hexdigest(), + ) + request = { + "schema_version": 1, "prepared_ref": ref, + "attempt": {key: vector[key] for key in execution_service.ATTEMPT_FIELDS}, + "proposal": proposal, "output_root": vector["output_root"], + } + spec = bind_execution(request, prepared) + store = run_store.RunStore(tmp_path / "store", "test") + target = LocalCpuTarget( + "fixture-cpu", "1", work_root=tmp_path / "work", controller_root=controller, + module_root=modules, + ) + reader = ArtifactReader(config) + return {"spec": spec, "store": store, "target": target, "modules": modules, + "source": source, "reader": reader, "harness": Harness(store, target, + artifact_reader=reader)} + + +def assert_fixed_refusal(case, record, status, actual): + schema = json.loads((ROOT / "docs/contracts/schemas/contract-v1.schema.json") + .read_text(encoding="utf-8")) + Draft202012Validator({**schema, "$ref": "#/$defs/ExecutionRecord"}).validate(record) + assert record["started"] is False and record["phase"] == "finished" + assert record["process_outcome"] == "not_started" and record["exit_code"] is None + assert record["prepared_ref"] == case["spec"]["prepared_ref"] + assert record["candidate"] == case["spec"]["candidate"] + check = record["checks"][0] + assert check["check_id"] == "entry_code_identity" and check["status"] == status + assert check["operation"] == "evaluate" and check["validation_scope"] == "p1" + runtime = record["target_binding"]["runtime_bindings_by_step"][0] + assert runtime["code_sha256"] == actual + assert all(step["status"] == "not_started" for step in record["steps"]) + if actual is not None: + assert runtime["argv"][1:3] == ["-I", "-B"] + assert record["resource_leases"]["leases"] == [] + saved = case["store"].read(record["attempt_id"], record["record_version"]) + assert saved["record_ref"] == record["record_ref"] + assert saved["checks"] == record["checks"] + with sqlite3.connect(case["store"].root / run_store.INDEX_NAME) as connection: + assert connection.execute("SELECT count(*) FROM attempts").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM records").fetchone()[0] == 1 + + +@pytest.mark.parametrize("failure", ["mismatch", "missing"]) +def test_t37_entry_failure_commits_a_not_started_record(evaluation, monkeypatch, failure): + case = evaluation + source = case["modules"] / "fixture_worker.py" + if failure == "mismatch": + observed = b"raise RuntimeError('changed worker must not start')\n" + source.write_bytes(observed) + actual = sha256(observed).hexdigest() + else: + source.unlink() + actual = None + + def forbidden(*args, **kwargs): + pytest.fail("T37-N1/N2 launched a process") + + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + record = case["harness"].execute(case["spec"]) + assert_fixed_refusal( + case, record, "failed" if failure == "mismatch" else "not_performed", actual, + ) + refs = record["checks"][0]["evidence_refs"] + if failure == "mismatch": + assert len(refs) == 1 and refs[0]["sha256"] == actual + with case["reader"].open(refs[0]) as stream: + assert stream.read() == observed + else: + assert refs == [] + assert record["target_binding"]["runtime_bindings_by_step"][0]["argv"] == [] + + +@pytest.mark.parametrize("interference", ["workspace-shadow", "changed-origin"]) +def test_matching_entry_executes_its_snapshot_and_retains_the_observed_digest( + evaluation, monkeypatch, interference, +): + case = evaluation + supervise = execution_service.local_process._supervise + observed_pids = [] + + def counted(*args, **kwargs): + result = supervise(*args, **kwargs) + observed_pids.append(result["pid"]) + return result + + monkeypatch.setattr(execution_service.local_process, "_supervise", counted) + if interference == "workspace-shadow": + case["spec"]["inputs"][0]["materialized_path"] = "fixture_worker.py" + case["spec"]["steps"][0]["args"][0] = "fixture_worker.py" + else: + original = execution_service.entry_code.capture_module + + def capture_then_change(*args, **kwargs): + result = original(*args, **kwargs) + (case["modules"] / "fixture_worker.py").write_bytes( + b"raise RuntimeError('mutable origin is not the bound entry')\n", + ) + return result + + monkeypatch.setattr(execution_service.entry_code, "capture_module", capture_then_change) + record = case["harness"].execute(case["spec"]) + assert record["started"] is True and record["process_outcome"] == "exited" + assert record["exit_code"] == 0 + assert len(observed_pids) == 1 and observed_pids[0] is not None + check = record["checks"][0] + assert check["status"] == "passed" + actual = sha256(case["source"]).hexdigest() + assert record["target_binding"]["runtime_bindings_by_step"][0]["code_sha256"] == actual + assert record["target_binding"]["runtime_bindings_by_step"][0]["argv"][1:3] == ["-I", "-B"] + with case["reader"].open(check["evidence_refs"][0]) as stream: + assert stream.read() == case["source"] + assert "verdict" not in record + + +@pytest.mark.parametrize("failure", ["mismatch", "missing"]) +def test_t37_commit_failure_preserves_not_started_and_no_committed_record( + evaluation, monkeypatch, failure, +): + case = evaluation + source = case["modules"] / "fixture_worker.py" + if failure == "missing": + source.unlink() + else: + source.write_bytes(b"changed worker\n") + + def fail_commit(*args, **kwargs): + raise sqlite3.OperationalError("fixture commit failure") + + monkeypatch.setattr(case["store"], "_append", fail_commit) + monkeypatch.setattr(execution_service.local_process, "_supervise", + lambda *a, **k: pytest.fail("failed T37 commit started a process")) + with pytest.raises(PublicError) as caught: + case["harness"].execute(case["spec"]) + assert caught.value.code == "ExecutionPersistenceError" + assert caught.value.started is False + assert caught.value.latest_record_ref is None + assert caught.value.partial_record is not None + assert caught.value.partial_record.started is False + with pytest.raises(PublicError) as missing: + case["store"].read(case["spec"]["attempt_id"]) + assert missing.value.code == "NoCommittedRecord" + + +def test_t37_fixed_mismatch_and_source_bytes_can_be_read_in_a_fresh_process( + evaluation, monkeypatch, +): + case = evaluation + changed = b"raise RuntimeError('mismatching worker never executes')\n" + source = case["modules"] / "fixture_worker.py" + source.write_bytes(changed) + monkeypatch.setattr(execution_service.local_process, "_supervise", + lambda *a, **k: pytest.fail("mismatching worker started")) + record = case["harness"].execute(case["spec"]) + source.unlink() + script = ( + "import sys,json,hashlib; from inferrsi.harness.run_store import RunStore,RunStoreReader; " + "s=RunStore(sys.argv[1],'test'); r=s.read(sys.argv[2],1); " + "ref=r['checks'][0]['evidence_refs'][0]; " + "stream=RunStoreReader(s.root,'test').open(ref); raw=stream.read(); stream.close(); " + "print(json.dumps([r['record_ref'],r['started'],r['checks'][0]['status']," + "r['target_binding']['runtime_bindings_by_step'][0]['code_sha256']," + "hashlib.sha256(raw).hexdigest()]))" + ) + child = subprocess.run( + [sys.executable, "-c", script, str(case["store"].root), record["attempt_id"]], + check=True, capture_output=True, text=True, timeout=30, + ) + actual = sha256(changed).hexdigest() + assert json.loads(child.stdout) == [record["record_ref"], False, "failed", actual, actual] + + +def test_entry_capture_write_failure_is_not_an_unresolvable_entry(evaluation, monkeypatch): + case = evaluation + + def fail_capture(*args, **kwargs): + raise execution_service.entry_code.EntryCaptureError("fixture capture failure") + + monkeypatch.setattr(execution_service.entry_code, "capture_module", fail_capture) + monkeypatch.setattr(execution_service.local_process, "_supervise", + lambda *a, **k: pytest.fail("failed capture started a worker")) + with pytest.raises(PublicError) as caught: + case["harness"].execute(case["spec"]) + assert caught.value.code == "ExecutionPersistenceError" + assert caught.value.started is False and caught.value.latest_record_ref is None + with pytest.raises(PublicError) as missing: + case["store"].read(case["spec"]["attempt_id"]) + assert missing.value.code == "NoCommittedRecord" diff --git a/tests/test_harness_execute.py b/tests/test_harness_execute.py index efd05451..94e01b96 100644 --- a/tests/test_harness_execute.py +++ b/tests/test_harness_execute.py @@ -37,6 +37,8 @@ NEW_H_CLOSURE = { "inferrsi.harness.api", "inferrsi.harness.execution_service", "inferrsi.harness.input_binding", "inferrsi.harness.local_process", + "inferrsi.harness.prepared_binding", + "inferrsi.harness.entry_code", "inferrsi.harness.atomic_json", "inferrsi.harness.run_store", "inferrsi.harness.errors", "inferrsi.harness.execution_spec", "inferrsi.contracts.json_v1", @@ -436,7 +438,8 @@ def test_spec_time_refusals_register_nothing(h: Harness) -> None: candidate={"candidate_id": "cand", "context_refs": [], "input_role": "candidate", "kind": "text", "parent_digest": None, "parent_id": None}, ) - assert error_of(h.execute, evaluation).code == "UnsupportedOperation" + # The malformed binding is rejected before support/entry checks (PR-1 / V32-N2). + assert error_of(h.execute, evaluation).code == "InvalidExecutionSpec" assert error_of(h.execute, dict(spec, schema_version=2)).code == "UnsupportedSchema" float_timeout = dict(spec, steps=[dict(spec["steps"][0], timeout_seconds=1.5)]) assert error_of(h.execute, float_timeout).code == "InvalidExecutionSpec" diff --git a/tests/test_harness_prepared_binding.py b/tests/test_harness_prepared_binding.py new file mode 100644 index 00000000..61027fac --- /dev/null +++ b/tests/test_harness_prepared_binding.py @@ -0,0 +1,356 @@ +"""PR-1: bind frozen preparation before registration; never perform T37 entry checks.""" + +import sqlite3 +from copy import deepcopy +from hashlib import sha256 +from io import BytesIO +from pathlib import Path + +import pytest + +from inferrsi.apps.handoff.bridge import bind_execution, verify_protocol +from inferrsi.contracts.json_v1 import decode, encode +from inferrsi.harness import execution_service, prepared_binding, run_store +from inferrsi.harness.api import Harness, LocalCpuTarget +from inferrsi.harness.errors import PublicError + +ROOT = Path(__file__).resolve().parents[1] +GOLDEN = ROOT / "docs/contracts/fixtures/t108-v4" +VECTOR = ROOT / "tests/fixtures/harness-prepared-binding/d023-null-timeout.json" +DIGEST = "sha256:62ce11e6d458b5c581245d0988189e1bd834f2ed1e1880d15242faf70c5b6fe9" + + +def test_d023_null_timeout_bytes_match_apps_and_harness(): + data = VECTOR.read_bytes() + spec = decode(data) + prepared = decode((GOLDEN / "prepared.json").read_bytes()) + verify_protocol(prepared, (GOLDEN / "protocol.json").read_bytes()) + request = { + "schema_version": 1, "prepared_ref": spec["prepared_ref"], + "attempt": {key: spec[key] for key in execution_service.ATTEMPT_FIELDS}, + "proposal": {key: spec[key] for key in ("candidate", "inputs", "generation_record_ref")}, + "output_root": spec["output_root"], + } + assert len(data) == 2069 + assert "sha256:" + sha256(data).hexdigest() == DIGEST + assert run_store.request_digest(spec) == DIGEST + assert encode(bind_execution(request, prepared)) == data + assert spec["steps"][0]["timeout_seconds"] is None + + +@pytest.fixture +def binding(tmp_path, monkeypatch): + spec = decode(VECTOR.read_bytes()) + prepared = (GOLDEN / "prepared.json").read_bytes() + + class Reader: + def __init__(self): + self.opens = [] + self.data = prepared + + def open(self, ref): + assert ref == spec["prepared_ref"]["artifact"] + self.opens.append(deepcopy(ref)) + return BytesIO(self.data) + + reader = Reader() + store = run_store.RunStore(tmp_path / "store", "binding-tests") + target = LocalCpuTarget("fixture-cpu", "1", work_root=tmp_path / "work") + harness = Harness(store, target, artifact_reader=reader) + + def forbidden(*args, **kwargs): + pytest.fail("a PR-1 refusal reached registration or process execution") + + monkeypatch.setattr(store, "claim", forbidden) + monkeypatch.setattr(execution_service.local_process, "_supervise", forbidden) + monkeypatch.setattr(execution_service, "_argv", forbidden) + return spec, reader, store, harness + + +def assert_empty(store): + with sqlite3.connect(store.root / run_store.INDEX_NAME) as connection: + assert connection.execute("SELECT count(*) FROM attempts").fetchone()[0] == 0 + assert connection.execute("SELECT count(*) FROM records").fetchone()[0] == 0 + + +def test_projection_reads_the_fixed_preparation_without_resolving_an_entry(binding): + spec, reader, store, harness = binding + fixed = prepared_binding.verify(spec, reader) + assert fixed.prepared_id == spec["prepared_ref"]["prepared_id"] + assert fixed.code_sha256 == spec["steps"][0]["expected_code_sha256"] + assert len(reader.opens) == 1 + # T37 is a later delivery: valid evaluation bindings still cannot start. + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "UnsupportedOperation" + assert_empty(store) + + +@pytest.mark.parametrize("failure", ["null-code", "wrong-code", "protocol", "candidate", "input"]) +def test_malformed_or_inconsistent_binding_registers_no_attempt(binding, failure): + spec, reader, store, harness = binding + if failure == "null-code": + spec["steps"][0]["expected_code_sha256"] = None + elif failure == "wrong-code": + spec["steps"][0]["expected_code_sha256"] = "0" * 64 + elif failure == "protocol": + spec["protocol_digest"] = "sha256:" + "0" * 64 + elif failure == "candidate": + spec["candidate"]["candidate_id"] = "different-candidate" + else: + spec["inputs"][0]["expected_sha256"] = "0" * 64 + spec["inputs"][0]["source_ref"]["sha256"] = "0" * 64 + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == ("IdentityMismatch" if failure == "protocol" + else "InvalidExecutionSpec") + assert_empty(store) + assert len(reader.opens) == (0 if failure == "null-code" else 1) + + +def test_corrupt_preparation_bytes_cannot_reach_registration(binding): + spec, reader, store, harness = binding + reader.data = b"corrupt retained preparation" + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "ArtifactIntegrityError" + assert_empty(store) + + +def replace_prepared(binding, path, value): + spec, reader, _, _ = binding + prepared = decode(reader.data) + parent = prepared + for key in path[:-1]: + parent = parent[key] + parent[path[-1]] = value + reader.data = encode(prepared) + spec["prepared_ref"]["artifact"].update( + sha256=sha256(reader.data).hexdigest(), size_bytes=len(reader.data), + ) + + +@pytest.mark.parametrize("path,value", [ + (("prepared_ref", "prepared_id"), "sha256:" + "0" * 64), + (("operation",), "different"), (("validation_scope",), "different"), +]) +def test_all_preparation_identity_disagreements_precede_registration(binding, path, value): + spec, reader, store, harness = binding + parent = spec + for key in path[:-1]: + parent = parent[key] + parent[path[-1]] = value + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "IdentityMismatch" + assert len(reader.opens) == 1 + assert_empty(store) + + +@pytest.mark.parametrize("field,value", [ + ("runtime_role", "another-python"), ("entrypoint_kind", "executable"), + ("entrypoint", "another_worker"), ("entrypoint_revision", "2"), + ("args", ["inputs/candidate.txt", "outputs/other.json"]), ("cwd", "elsewhere"), + ("outputs", []), +]) +def test_worker_declarations_must_preserve_the_frozen_recipe(binding, field, value): + spec, reader, store, harness = binding + spec["steps"][0][field] = value + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "InvalidExecutionSpec" + assert caught.value.stage == "prepared_binding" + assert len(reader.opens) == 1 + assert_empty(store) + + +@pytest.mark.parametrize("field,value", [ + ("path", "inputs/candidate.txt"), ("required", False), ("role", "other"), + ("media_type", "text/plain"), ("format_schema", {"name": "fixture-grader", "revision": "2"}), +]) +def test_output_relabeling_cannot_change_the_frozen_worker_contract(binding, field, value): + spec, _, store, harness = binding + spec["steps"][0]["outputs"][0][field] = value + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "InvalidExecutionSpec" + assert_empty(store) + + +@pytest.mark.parametrize("failure", ["capability", "missing-role", "overlap", "extra-step"]) +def test_incomplete_or_ambiguous_worker_binding_is_refused(binding, failure): + spec, _, store, harness = binding + if failure == "capability": + spec["required_capabilities"] = [] + elif failure == "missing-role": + replace_prepared(binding, ("recipe", "input_roles"), ["candidate", "config"]) + elif failure == "extra-step": + extra = deepcopy(spec["steps"][0]) + extra["step_id"] = "extra" + spec["steps"].append(extra) + else: + # Keep argument binding internally consistent while overlapping an output. + spec["inputs"][0]["materialized_path"] = "outputs" + spec["steps"][0]["args"][0] = "outputs" + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "InvalidExecutionSpec" + assert_empty(store) + + +@pytest.mark.parametrize("path,value", [ + (("schema_version",), True), (("schema_version",), 2), + (("candidate_contract",), None), (("candidate_contract", "identity_rule"), {}), + (("expected_candidate",), {}), (("expected_candidate", "sha256"), None), + (("recipe",), None), (("recipe", "code_sha256"), None), + (("recipe", "runtime_role"), None), (("recipe", "entrypoint_kind"), "unknown"), + (("recipe", "entrypoint"), None), (("recipe", "entrypoint_revision"), ""), + (("recipe", "args"), [None]), (("recipe", "args"), ["{input:unknown}"]), + (("recipe", "args"), ["prefix{input:candidate}"]), + (("recipe", "input_roles"), ["candidate", "candidate"]), + (("recipe", "required_capabilities"), [None]), + (("recipe", "outputs"), [{}]), (("recipe", "outputs", 0, "path"), "../outside"), + (("recipe", "outputs", 0, "format_schema"), {}), +]) +def test_malformed_fixed_projection_is_refused_with_a_declared_error(binding, path, value): + _, _, store, harness = binding + replace_prepared(binding, path, value) + with pytest.raises(PublicError) as caught: + harness.execute(binding[0]) + assert caught.value.code == ("UnsupportedSchema" if value == 2 else "InvalidExecutionSpec") + assert_empty(store) + + +@pytest.mark.parametrize("revision", ["2", "unknown"]) +def test_unknown_candidate_identity_revision_is_never_downgraded(binding, revision): + spec, _, store, harness = binding + replace_prepared(binding, ("candidate_contract", "identity_rule", "revision"), revision) + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == "UnsupportedRuleVersion" + assert_empty(store) + + +@pytest.mark.parametrize("rule", ["captured-sha256", "expected-sha256"]) +def test_expected_hash_requirement_depends_on_the_frozen_identity_rule(binding, rule): + spec, _, store, harness = binding + replace_prepared(binding, ("candidate_contract", "identity_rule", "name"), rule) + replace_prepared(binding, ("expected_candidate",), None) + spec["inputs"][0].update( + expected_sha256=None, source_namespace="controller", source_ref=None, + source="candidate.txt", + ) + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == ("InvalidExecutionSpec" if rule == "expected-sha256" + else "UnsupportedOperation") + assert_empty(store) + + +@pytest.mark.parametrize("mode", ["noncanonical", "duplicate-key", "array", "extra-field"]) +def test_unparseable_or_unknown_preparation_fields_never_register(binding, mode): + spec, reader, store, harness = binding + if mode == "noncanonical": + reader.data += b"\n" + elif mode == "duplicate-key": + reader.data = reader.data[:-1] + b',"schema_version":1}' + elif mode == "array": + reader.data = b"[]" + else: + replace_prepared(binding, ("unknown",), True) + spec["prepared_ref"]["artifact"].update( + sha256=sha256(reader.data).hexdigest(), size_bytes=len(reader.data), + ) + with pytest.raises(PublicError) as caught: + harness.execute(spec) + expected = ("ArtifactIntegrityError" if mode in {"noncanonical", "duplicate-key"} + else "InvalidExecutionSpec") + assert caught.value.code == expected + assert_empty(store) + + +@pytest.mark.parametrize("path,value", [ + (("schema_version",), True), (("schema_version",), 2), (("unknown",), None), + (("prepared_ref",), None), (("prepared_ref", "schema_version"), 2), + (("prepared_ref", "artifact", "location"), "../outside"), + (("prepared_ref", "artifact", "size_bytes"), True), + (("prepared_ref", "artifact", "sha256"), "A" * 64), + (("prepared_ref", "artifact", "media_type"), "text/plain"), + (("protocol_digest",), "invalid"), (("candidate",), {}), + (("candidate", "kind"), None), (("candidate", "parent_id"), "parent"), + (("candidate", "context_refs"), [None]), (("candidate", "input_role"), "absent"), + (("steps", 0, "expected_code_sha256"), "invalid"), +]) +def test_static_shape_errors_never_open_the_preparation(binding, path, value): + spec, reader, store, harness = binding + parent = spec + for key in path[:-1]: + parent = parent[key] + parent[path[-1]] = value + with pytest.raises(PublicError) as caught: + harness.execute(spec) + assert caught.value.code == ("UnsupportedSchema" if path == ("schema_version",) and value == 2 + else "InvalidExecutionSpec") + assert reader.opens == [] + assert_empty(store) + + +@pytest.mark.parametrize("mode", ["undersize", "oversize", "wrong-hash", "missing", "os-error"]) +def test_fixed_byte_reader_closes_streams_and_bounds_reads(binding, mode): + spec, reader, store, harness = binding + original = spec["prepared_ref"]["artifact"] + sizes = [] + + class Tracking(BytesIO): + def read(self, size=-1): + sizes.append(size) + return super().read(size) + + stream = Tracking(reader.data + (b"x" if mode == "oversize" else b"")) + + class Tracked: + def open(self, ref): + if mode == "os-error": + raise FileNotFoundError("fixture missing") + return stream + + if mode == "undersize": + original["size_bytes"] += 1 + elif mode == "wrong-hash": + original["sha256"] = "0" * 64 + with pytest.raises(PublicError) as caught: + prepared_binding.verify(spec, None if mode == "missing" else Tracked()) + assert caught.value.code == ("ArtifactUnavailable" if mode in {"missing", "os-error"} + else "ArtifactIntegrityError") + if mode not in {"missing", "os-error"}: + assert stream.closed + assert max(sizes) <= original["size_bytes"] + 1 + assert_empty(store) + + +def test_reader_cannot_rewrite_the_ref_it_is_meant_to_verify(binding): + spec, _, store, _ = binding + original = deepcopy(spec["prepared_ref"]["artifact"]) + + class Mutating: + def open(self, ref): + ref.update(sha256=sha256(b"{}").hexdigest(), size_bytes=2) + return BytesIO(b"{}") + + with pytest.raises(PublicError) as caught: + prepared_binding.verify(spec, Mutating()) + assert caught.value.code == "ArtifactIntegrityError" + assert spec["prepared_ref"]["artifact"] == original + assert_empty(store) + + +def test_reader_programming_errors_propagate_without_registration(binding): + spec, _, store, _ = binding + + class Broken: + def open(self, ref): + raise TypeError("broken host reader") + + with pytest.raises(TypeError, match="broken host reader"): + prepared_binding.verify(spec, Broken()) + assert_empty(store)