diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9765a0d..894544e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,12 +33,40 @@ jobs: - name: Lint (ruff) run: uv run ruff check . - - name: Type check (mypy) - run: uv run mypy reportnet_api + - name: Format (ruff) + run: uv run ruff format --check . - - name: Test (pytest) + - name: Type check (mypy, strict; package, tests, scripts, tools) + run: uv run mypy + + - name: Test (pytest; warnings are errors, no network) run: uv run pytest + generated: + # Generated code must match its generator and spec exactly. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: true + + - name: Install dependencies (exactly as locked) + run: uv sync --all-groups --locked + + - name: Regenerate models, resources and reference docs + run: | + scripts/regen_models.sh + scripts/regen_resources.sh + scripts/regen_docs.sh + + - name: Fail if anything changed + run: git diff --exit-code + package: runs-on: ubuntu-latest steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..73872de --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +# Run before each commit: `uvx pre-commit install` once, then it runs on `git commit`. +# Run on everything by hand: `uvx pre-commit run --all-files`. +# The hooks call the project's own locked tools through uv, so they match CI exactly. +repos: + - repo: local + hooks: + - id: ruff-check + name: ruff check + entry: uv run ruff check --fix + language: system + types: [python] + - id: ruff-format + name: ruff format + entry: uv run ruff format + language: system + types: [python] + - id: mypy + name: mypy (strict, whole project) + entry: uv run mypy + language: system + types: [python] + pass_filenames: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3953722..370576b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,13 +5,15 @@ Thanks for helping. This is a small library, so the process is light. ## Set up ``` -uv sync --all-groups # install deps + dev tools -uv run ruff check . # lint -uv run mypy reportnet_api -uv run pytest # no network needed: HTTP is mocked +uv sync --all-groups # install deps + dev tools +uv run ruff check . # lint +uv run ruff format --check . +uv run mypy # strict: package, tests, scripts, tools +uv run pytest # warnings are errors; real network access fails the test +uvx pre-commit install # optional: run ruff and mypy on every commit ``` -All three must pass; CI runs the same commands on Python 3.11 to 3.14 and also builds the +All of these must pass; CI runs the same commands on Python 3.11 to 3.14 and also builds the wheel. The docs site is built with `uv run --group docs mkdocs build --strict`. ## Generated files: do not hand-edit diff --git a/README.md b/README.md index 67e88fc..b3db733 100644 --- a/README.md +++ b/README.md @@ -217,8 +217,8 @@ the code here is MIT-licensed (see `LICENSE`). ``` uv sync --all-groups # install deps + dev tools -uv run ruff check . # lint -uv run mypy reportnet_api +uv run ruff check . && uv run ruff format --check . +uv run mypy # strict, whole project uv run pytest ``` diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 0c35f32..48eb415 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -24,7 +24,7 @@ repository's workflow, so no API token is created or stored. ## Each release 1. Update `version` in `pyproject.toml` and the entry in `CHANGELOG.md`. -2. `uv run ruff check . && uv run mypy reportnet_api && uv run pytest` must pass (CI runs +2. `uv run ruff check . && uv run ruff format --check . && uv run mypy && uv run pytest` must pass (CI runs them on Python 3.11 to 3.14). 3. **Rehearse:** run the *Release* workflow by hand (Actions tab → Release → Run workflow). It uploads to TestPyPI. Install from there in a clean environment and import it: diff --git a/pyproject.toml b/pyproject.toml index 6f86a5a..c72de97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,20 +67,51 @@ target-version = "py311" line-length = 100 [tool.ruff.lint] -select = ["E", "F", "I", "UP", "B"] +select = [ + "E", "F", "I", "B", "UP", "RUF", "S", "PL", "DTZ", "PTH", "FBT", "PT", "SIM", "C4", "RET", +] +# ANN is left out on purpose: mypy strict already enforces annotations. +ignore = [ + "PLR0913", # resource methods mirror the endpoint's parameters + "PLC0415", # pandas is imported lazily so that it stays optional + "FBT001", # boolean parameters are public API; changing them is a breaking change + "FBT002", # same + "PT018", # splitting compound asserts is churn; pytest already shows both sides + "SIM117", # nested `with` blocks read fine; merging them is style only + "PLR0911", # complexity counts: splitting long, linear functions risks behaviour + "PLR0912", # changes for no correctness gain; revisit case by case + "PLR0915", +] [tool.ruff.lint.per-file-ignores] -"reportnet_api/models/generated.py" = ["E501", "UP"] +"reportnet_api/models/generated.py" = ["ALL"] "reportnet_api/resources/*.py" = ["E501"] +"tests/**" = ["S101", "S105", "S106", "PLR2004", "FBT003"] [tool.mypy] python_version = "3.11" strict = true -warn_unused_ignores = true +files = ["reportnet_api", "tests", "scripts", "tools"] +warn_unreachable = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool", "possibly-undefined"] [tool.pytest.ini_options] testpaths = ["tests"] +addopts = ["--strict-markers", "--strict-config", "-ra"] +filterwarnings = ["error"] +xfail_strict = true [[tool.mypy.overrides]] module = ["pandas", "pandas.*"] ignore_missing_imports = true + +[[tool.mypy.overrides]] +# numpy's stubs use Python 3.12 syntax, which python_version 3.11 cannot parse. +module = ["numpy", "numpy.*"] +follow_imports = "skip" +follow_imports_for_stubs = true + +[[tool.mypy.overrides]] +# Only scripts/make_logo.py uses these, installed ad hoc with `uv run --with`. +module = ["cairosvg", "fontTools.*"] +ignore_missing_imports = true diff --git a/reportnet_api/_core.py b/reportnet_api/_core.py index 6d9126b..88a06c9 100644 --- a/reportnet_api/_core.py +++ b/reportnet_api/_core.py @@ -9,7 +9,8 @@ from dataclasses import dataclass from datetime import UTC, datetime from email.utils import parsedate_to_datetime -from types import MappingProxyType +from pathlib import Path +from types import FrameType, MappingProxyType from typing import Any, BinaryIO import httpx @@ -39,11 +40,15 @@ def caller_stacklevel() -> int: """The `stacklevel` that points a warning at the first frame outside this package.""" - package_dir = os.path.dirname(os.path.abspath(__file__)) + os.sep - level, frame = 1, sys._getframe(1) - while frame is not None and os.path.abspath(frame.f_code.co_filename).startswith(package_dir): + # os.path on purpose: Path.resolve() follows symlinks, co_filename does not. + package_dir = os.path.dirname(os.path.abspath(__file__)) + os.sep # noqa: PTH100, PTH120 + level = 1 + frame: FrameType | None = sys._getframe(1) + while frame is not None and os.path.abspath( # noqa: PTH100 - same reason + frame.f_code.co_filename + ).startswith(package_dir): level += 1 - frame = frame.f_back # type: ignore[assignment] + frame = frame.f_back return level @@ -218,5 +223,6 @@ def stream_upload_file( httpx reads from the file handle lazily while sending the request, so the file is never fully buffered in memory. """ - with open(file, "rb") as fh: - yield {field_name: (os.path.basename(str(file)), fh)} + path = Path(file) + with path.open("rb") as fh: + yield {field_name: (path.name, fh)} diff --git a/reportnet_api/_exceptions.py b/reportnet_api/_exceptions.py index 77710ed..c9510ae 100644 --- a/reportnet_api/_exceptions.py +++ b/reportnet_api/_exceptions.py @@ -1,7 +1,9 @@ from __future__ import annotations import json +import math import re +from http import HTTPStatus import httpx @@ -59,11 +61,12 @@ class TooManyRequestsError(ReportnetAPIError): @property def retry_after(self) -> float | None: - """Seconds from a numeric `Retry-After` header, or None if absent or not a number.""" + """Seconds from a numeric `Retry-After` header, or None if absent or not a finite number.""" try: - return float(self.response.headers["Retry-After"]) + seconds = float(self.response.headers["Retry-After"]) except (KeyError, ValueError): return None + return seconds if math.isfinite(seconds) else None class ServerError(ReportnetAPIError): @@ -107,14 +110,14 @@ class ServerError(ReportnetAPIError): def _hint(response: httpx.Response) -> str | None: """Short tip for 401/403 (tailored to the Authorization scheme used), 423 and 429.""" - if response.status_code == 423: + if response.status_code == HTTPStatus.LOCKED: return _LOCKED_HINT - if response.status_code == 429: + if response.status_code == HTTPStatus.TOO_MANY_REQUESTS: return _TOO_MANY_HINT - if response.status_code not in (401, 403): + if response.status_code not in (HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN): return None scheme = response.request.headers.get("Authorization", "").split(" ", 1)[0].lower() - if response.status_code == 401: + if response.status_code == HTTPStatus.UNAUTHORIZED: return _APIKEY_401_HINT if scheme == "apikey" else _BEARER_401_HINT return _APIKEY_HINT if scheme == "apikey" else _BEARER_FORBIDDEN_HINT @@ -166,7 +169,9 @@ def _explain(err: ReportnetAPIError) -> list[str]: lines = [f"{request.method} {path} failed with HTTP {status}."] if reason: lines.append(f"The server said: {reason}") - if status == 500 and _is_rejected_credential_500(err.response_body): + if status == HTTPStatus.INTERNAL_SERVER_ERROR and _is_rejected_credential_500( + err.response_body + ): lines.append( "Likely cause: the credential was rejected upstream (a revoked or expired ApiKey " "shows up as a 500 like this, even for plain reads), or it was sent with the " @@ -177,26 +182,28 @@ def _explain(err: ReportnetAPIError) -> list[str]: "A Bearer token sent as an ApiKey fails the same way: Bearer tokens are bare " 'UUIDs here, not JWTs, so connect(token, scheme="Bearer").' ) - elif status == 500 and "accessReferenceEntity" in err.response_body: + elif ( + status == HTTPStatus.INTERNAL_SERVER_ERROR and "accessReferenceEntity" in err.response_body + ): lines.append( "Likely cause: the dataflow id does not exist, or is not visible to this " "credential (the gateway reports this as a 500, not a 404)." ) lines.append("Try: check the dataflow id (list_dataflows) and that it is not a dataset id.") - elif status == 423: + elif status == HTTPStatus.LOCKED: lines.append( "Likely cause: another operation (often an import job still QUEUED or " "IN_PROGRESS) holds a lock on this dataset or dataflow; the server's own text " "can be vague." ) lines.append("Try: wait for that job to finish (see wait_for_job), then repeat the call.") - elif status == 403 and scheme == "bearer" and "/etl" in path: + elif status == HTTPStatus.FORBIDDEN and scheme == "bearer" and "/etl" in path: lines.append( "Likely cause: the ETL endpoints are gated separately from dataflow roles; a " "Bearer user can get 403 here even as Custodian/Steward." ) lines.append("Try: use a dataflow ApiKey for ETL calls, or ask the dataflow admins.") - elif status == 403 and scheme == "apikey": + elif status == HTTPStatus.FORBIDDEN and scheme == "apikey": lines.append( "Likely cause: ApiKeys are Reporter-level and work only for the dataflow they " "were issued for, so Custodian/Steward-only routes refuse them." @@ -204,10 +211,10 @@ def _explain(err: ReportnetAPIError) -> list[str]: lines.append("Try: use a Bearer token of a user holding that role on this dataflow.") elif (hint := _hint(err.response)) is not None: lines.append(f"Hint: {hint}.") - elif status >= 500: + elif status >= HTTPStatus.INTERNAL_SERVER_ERROR: lines.append("Likely cause: a server-side failure; the request itself may be fine.") lines.append("Try: check the server's message above, then retry later.") - elif status == 404: + elif status == HTTPStatus.NOT_FOUND: lines.append("Likely cause: the id in the request does not exist for this credential.") lines.append("Try: check dataflow and dataset ids are not mixed up.") elif status in (400, 422): @@ -222,7 +229,7 @@ def error_for_response(response: httpx.Response) -> ReportnetAPIError | None: return None exc_cls = _STATUS_EXCEPTIONS.get(response.status_code) - if exc_cls is None and response.status_code >= 500: + if exc_cls is None and response.status_code >= HTTPStatus.INTERNAL_SERVER_ERROR: exc_cls = ServerError if exc_cls is None: exc_cls = ReportnetAPIError diff --git a/reportnet_api/async_client.py b/reportnet_api/async_client.py index 4e971dd..4a2fb88 100644 --- a/reportnet_api/async_client.py +++ b/reportnet_api/async_client.py @@ -2,6 +2,7 @@ import asyncio import os +from pathlib import Path from typing import Any import httpx @@ -36,7 +37,7 @@ def __init__( base_url: str, token: TokenLike, *, - token_scheme: str = "Bearer", + token_scheme: str = "Bearer", # noqa: S107 - an auth scheme name, not a secret timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, retry: RetryPolicy | None = None, ) -> None: @@ -99,7 +100,7 @@ async def download( if not response.is_success: await response.aread() self._core.handle_response(response) - with open(dest, "wb") as fh: + with Path(dest).open("wb") as fh: async for chunk in response.aiter_bytes(): fh.write(chunk) diff --git a/reportnet_api/cli.py b/reportnet_api/cli.py index 62c203f..2ed79e0 100644 --- a/reportnet_api/cli.py +++ b/reportnet_api/cli.py @@ -24,6 +24,7 @@ import sys import tempfile from collections.abc import Sequence +from pathlib import Path from typing import Any, TextIO import httpx @@ -165,15 +166,17 @@ def main(argv: Sequence[str] | None = None, out: TextIO | None = None) -> int: elif args.command == "read": if args.out: # Write next to the target and rename, so a failure leaves no empty file. - folder = os.path.dirname(os.path.abspath(args.out)) + # abspath normalises '..' without touching the disk; Path.absolute() does not. + folder = os.path.dirname(os.path.abspath(args.out)) # noqa: PTH100, PTH120 fd, tmp = tempfile.mkstemp(dir=folder, suffix=".part") try: with os.fdopen(fd, "w", newline="", encoding="utf-8") as fh: _read(client, args, fh) - os.chmod(tmp, 0o666 & ~_UMASK) # mkstemp makes 0600; use the usual mode - os.replace(tmp, args.out) + # mkstemp makes 0600; use the usual mode + Path(tmp).chmod(0o666 & ~_UMASK) + Path(tmp).replace(args.out) except BaseException: - os.unlink(tmp) + Path(tmp).unlink() raise else: _read(client, args, out) diff --git a/reportnet_api/client.py b/reportnet_api/client.py index 6827865..fb48309 100644 --- a/reportnet_api/client.py +++ b/reportnet_api/client.py @@ -2,6 +2,7 @@ import os import time +from pathlib import Path from typing import Any import httpx @@ -36,7 +37,7 @@ def __init__( base_url: str, token: TokenLike, *, - token_scheme: str = "Bearer", + token_scheme: str = "Bearer", # noqa: S107 - an auth scheme name, not a secret timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, retry: RetryPolicy | None = None, ) -> None: @@ -100,7 +101,7 @@ def download( if not response.is_success: response.read() self._core.handle_response(response) - with open(dest, "wb") as fh: + with Path(dest).open("wb") as fh: for chunk in response.iter_bytes(): fh.write(chunk) diff --git a/reportnet_api/easy.py b/reportnet_api/easy.py index d488eb7..a375670 100644 --- a/reportnet_api/easy.py +++ b/reportnet_api/easy.py @@ -23,6 +23,7 @@ import zipfile from collections.abc import Callable, Iterable from decimal import Decimal +from http import HTTPStatus from pathlib import Path from typing import TYPE_CHECKING, Any, Literal @@ -587,8 +588,8 @@ def apply_schema_dtypes( it, so a wrong guess never silently changes data. Returns new frames. """ out = {} - for name, df in tables.items(): - df = df.copy() + for name, frame in tables.items(): + df = frame.copy() types = field_types.get(name, {}) for col in df.columns: ftype = types.get(col) @@ -1177,7 +1178,8 @@ def check_import_file( return [] if best is None or len(missing) + len(extra) < len(best[0]) + len(best[1]): best = (missing, extra) - assert best is not None + if best is None: # unreachable: the loop runs at least once + raise RuntimeError("no delimiter was tried") missing, extra = best parts = [] if missing: @@ -1196,7 +1198,7 @@ def _format_cell(value: Any, field_type: str) -> str: if value is None or value is pd.NA: return "" is_real = isinstance(value, numbers.Real) and not isinstance(value, numbers.Integral | bool) - if is_real and value != value: # NaN, also numpy's + if is_real and value != value: # noqa: PLR0124 - NaN, also numpy's and Decimal's return "" if value is pd.NaT: return "" @@ -1346,8 +1348,9 @@ def check_access( rows.append({"check": "credential alive", "result": "yes", "detail": "list dataflows ok"}) except ReportnetAPIError as err: verdict, detail = _probe_failure(err) - if err.status_code == 401 or ( - err.status_code == 500 and _is_rejected_credential_500(err.response_body) + if err.status_code == HTTPStatus.UNAUTHORIZED or ( + err.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + and _is_rejected_credential_500(err.response_body) ): verdict = "no (rejected)" detail = "expired, revoked, or sent with the wrong scheme" @@ -1367,7 +1370,7 @@ def check_access( try: flow = client.dataflow.dataflow_datasets(dataflow_id) except ReportnetAPIError as err: - if err.status_code == 403: + if err.status_code == HTTPStatus.FORBIDDEN: verdict, detail = "no access", "the server refuses this credential (403)" elif "accessReferenceEntity" in err.response_body: verdict, detail = "not found", "unknown id, or not visible (reported as 500)" diff --git a/reportnet_api/handles.py b/reportnet_api/handles.py index 9fa8b9c..bcccce0 100644 --- a/reportnet_api/handles.py +++ b/reportnet_api/handles.py @@ -25,17 +25,21 @@ import pandas as pd +_COUNTRY_CODE_LENGTH = 2 # ISO 3166-1 alpha-2, as Reportnet uses + + def _int_or_none(value: Any) -> int | None: """Ints come back from DataFrames as floats with NaN for missing values.""" try: - return None if value is None or value != value else int(value) + return None if value is None or value != value else int(value) # noqa: PLR0124 - NaN except (TypeError, ValueError): return None def _as_id(key: object) -> int | None: """An integer key (numpy integers included; bool is not an id), else None.""" - if isinstance(key, numbers.Integral) and not isinstance(key, bool): + # mypy thinks no bool is Integral; at runtime isinstance(True, numbers.Integral) is True. + if isinstance(key, numbers.Integral) and not isinstance(key, bool): # type: ignore[unreachable] return int(key) return None @@ -197,7 +201,7 @@ def dataset(self, key: int | str, *, kind: str | None = None) -> Dataset: k = str(key).strip().lower() if not k: raise ValueError("The dataset key is empty: give an id, a country code or a name") - if codes is None and len(k) <= 2: + if codes is None and len(k) <= _COUNTRY_CODE_LENGTH: # Looks like a country code, which cannot be told from a name fragment. raise LookupError( f"Cannot resolve {key!r} in dataflow {self.id}: provider codes could not " @@ -320,7 +324,7 @@ def find_dataflow(client: ReportnetClient, query: int | str) -> Dataflow: return Dataflow( id=flow_id, name=str(row["name"]), - status=None if status is None or status != status else str(status), + status=None if status is None or status != status else str(status), # noqa: PLR0124 - NaN big_data=easy.is_data_lake(client, flow_id), client=client, ) diff --git a/reportnet_api/jobs.py b/reportnet_api/jobs.py index 814e056..f6eaf6d 100644 --- a/reportnet_api/jobs.py +++ b/reportnet_api/jobs.py @@ -54,7 +54,7 @@ class JobError(Exception): job itself reported the failure (`FAILED`, `CANCELED`, ...), else None. """ - def __init__(self, message: str, *, status: str | None = None, info: str | None = None): + def __init__(self, message: str, *, status: str | None = None, info: str | None = None) -> None: super().__init__(message) self.status = status self.info = info @@ -237,6 +237,7 @@ def _wait( poll_interval: float, timeout: float, progress: Progress, + *, announce_interrupt: bool, ) -> dict[str, Any]: url = polling_url_of(job) @@ -278,7 +279,7 @@ def wait_for_job( callable receives an event dict per poll (see `progress_event()`). With progress on, Ctrl-C prints the job id and that the job keeps running. """ - return _wait(client, job, poll_interval, timeout, progress, False) + return _wait(client, job, poll_interval, timeout, progress, announce_interrupt=False) class Job: @@ -332,7 +333,14 @@ def wait( self, *, poll_interval: float = 2.0, timeout: float = 300.0, progress: Progress = None ) -> dict[str, Any]: """Like `wait_for_job()` for this job; Ctrl-C always reports the job id.""" - answer = _wait(self._client, self.polling_url, poll_interval, timeout, progress, True) + answer = _wait( + self._client, + self.polling_url, + poll_interval, + timeout, + progress, + announce_interrupt=True, + ) self.last_status = answer.get("status") or self.last_status return answer diff --git a/reportnet_api/resources/dataset_schema.py b/reportnet_api/resources/dataset_schema.py index 1d37a2d..afc2a9e 100644 --- a/reportnet_api/resources/dataset_schema.py +++ b/reportnet_api/resources/dataset_schema.py @@ -129,7 +129,6 @@ def import_field_schemas( url = f"/dataschema/v1/{dataset_schema_id}/importFieldSchemas" with stream_upload_file("file", file) as files: self._client.request("POST", url, params=params, files=files) - return None def update_manually_editable_by_dataset_id( self, @@ -138,10 +137,8 @@ def update_manually_editable_by_dataset_id( ) -> None: """updateManuallyEditableByDatasetId""" params = {"manuallyEditable": manually_editable} - params = {k: v for k, v in params.items() if v is not None} url = f"/dataschema/updateManuallyEditable/{dataset_id}" self._client.request("PUT", url, params=params) - return None class AsyncDatasetSchemaResource: @@ -259,7 +256,6 @@ async def import_field_schemas( url = f"/dataschema/v1/{dataset_schema_id}/importFieldSchemas" with stream_upload_file("file", file) as files: await self._client.request("POST", url, params=params, files=files) - return None async def update_manually_editable_by_dataset_id( self, @@ -268,7 +264,5 @@ async def update_manually_editable_by_dataset_id( ) -> None: """updateManuallyEditableByDatasetId""" params = {"manuallyEditable": manually_editable} - params = {k: v for k, v in params.items() if v is not None} url = f"/dataschema/updateManuallyEditable/{dataset_id}" await self._client.request("PUT", url, params=params) - return None diff --git a/reportnet_api/resources/dataset_snapshot.py b/reportnet_api/resources/dataset_snapshot.py index 1138828..641a7e0 100644 --- a/reportnet_api/resources/dataset_snapshot.py +++ b/reportnet_api/resources/dataset_snapshot.py @@ -34,7 +34,6 @@ def delete_snapshot( params = None url = f"/snapshot/v1/{id_snapshot}/dataset/{id_dataset}/delete" self._client.request("DELETE", url, params=params) - return None def download_historic_releases_csv( self, @@ -54,7 +53,6 @@ def download_historic_releases_csv( params = {k: v for k, v in params.items() if v is not None} url = f"/snapshot/downloadHistoricReleases/{dataset_id}" self._client.download("GET", url, params=params, dest=dest) - return None def export_historic_releases_csv( self, @@ -68,7 +66,6 @@ def export_historic_releases_csv( `/snapshot/downloadHistoricReleases/{id}?dataflowId=..&fileName=..&processId=..`. Pass its `fileName`/`processId` to `download_historic_releases_csv`.""" params = {"dataflowId": dataflow_id} - params = {k: v for k, v in params.items() if v is not None} url = f"/snapshot/exportHistoricReleases/{dataset_id}" response = self._client.request("POST", url, params=params) return response.text @@ -114,7 +111,6 @@ async def delete_snapshot( params = None url = f"/snapshot/v1/{id_snapshot}/dataset/{id_dataset}/delete" await self._client.request("DELETE", url, params=params) - return None async def download_historic_releases_csv( self, @@ -134,7 +130,6 @@ async def download_historic_releases_csv( params = {k: v for k, v in params.items() if v is not None} url = f"/snapshot/downloadHistoricReleases/{dataset_id}" await self._client.download("GET", url, params=params, dest=dest) - return None async def export_historic_releases_csv( self, @@ -148,7 +143,6 @@ async def export_historic_releases_csv( `/snapshot/downloadHistoricReleases/{id}?dataflowId=..&fileName=..&processId=..`. Pass its `fileName`/`processId` to `download_historic_releases_csv`.""" params = {"dataflowId": dataflow_id} - params = {k: v for k, v in params.items() if v is not None} url = f"/snapshot/exportHistoricReleases/{dataset_id}" response = await self._client.request("POST", url, params=params) return response.text diff --git a/reportnet_api/resources/datasets.py b/reportnet_api/resources/datasets.py index a13aed7..9353e62 100644 --- a/reportnet_api/resources/datasets.py +++ b/reportnet_api/resources/datasets.py @@ -56,7 +56,6 @@ def convert_iceberg_to_parquet_tables( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/convertIcebergToParquetTables/{dataset_id}" self._client.request("POST", url, params=params) - return None def convert_parquet_to_iceberg_tables( self, @@ -74,7 +73,6 @@ def convert_parquet_to_iceberg_tables( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/convertParquetToIcebergTables/{dataset_id}" self._client.request("POST", url, params=params) - return None def create_empty_tables_v3( self, @@ -86,7 +84,6 @@ def create_empty_tables_v3( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/{dataset_id}/createEmptyTablesV3" self._client.request("POST", url, params=params) - return None def create_public_files( self, @@ -103,7 +100,6 @@ def create_public_files( params = {k: v for k, v in params.items() if v is not None} url = "/dataset/createPublicFiles" self._client.request("POST", url, params=params) - return None def delete_attachment( self, @@ -140,7 +136,6 @@ def delete_attachment( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/v1/{dataset_id}/field/{field_id}/attachment" self._client.request("DELETE", url, params=params) - return None def delete_dataset_data( self, @@ -216,10 +211,8 @@ def download_imported_file( ) -> None: """Downloads a specific imported file for a given dataset and a given filename""" params = {"dataflowId": dataflow_id, "datasetId": dataset_id, "fileName": file_name} - params = {k: v for k, v in params.items() if v is not None} url = "/dataset/download-imported-file" self._client.download("GET", url, params=params, dest=dest) - return None def duplicate_field_value_exists( self, @@ -229,7 +222,6 @@ def duplicate_field_value_exists( ) -> bool: """duplicateFieldValueExists""" params = {"tableSchemaId": table_schema_id} - params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/duplicateFieldValueExists/{dataset_id}" response = self._client.request( "POST", url, params=params, json=field_vo.model_dump(mode="json", exclude_none=True) @@ -290,7 +282,6 @@ def etl_export_dataset( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/v1/{dataset_id}/etlExport" self._client.download("GET", url, params=params, dest=dest) - return None def etl_export_dataset_v2( self, @@ -346,7 +337,6 @@ def etl_export_dataset_v2( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/v2/etlExport/{dataset_id}" self._client.download("GET", url, params=params, dest=dest) - return None def etl_export_dataset_with_job( self, @@ -631,7 +621,6 @@ def get_attachment( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/v1/{dataset_id}/field/{field_id}/attachment" self._client.download("GET", url, params=params, dest=dest) - return None def get_available_for_manual_editing_tables( self, @@ -712,7 +701,6 @@ def get_released_dataset_data_info( "providerCode": provider_code, "tableSchemaId": table_schema_id, } - params = {k: v for k, v in params.items() if v is not None} url = "/dataset/getReleasedDatasetDataInfo" response = self._client.request("GET", url, params=params) return generated.ReleasedDatasetDataInfoVO.model_validate(response.json()) @@ -809,7 +797,6 @@ def import_file_data( url = f"/dataset/v1/{dataset_id}/importFileData" with stream_upload_file("file", file) as files: self._client.request("POST", url, params=params, files=files) - return None def is_iceberg_table_created( self, @@ -833,7 +820,6 @@ def list_imported_files( ) -> list[generated.ImportedFilesDirectoriesVO]: """Returns all imported files for a given dataset""" params = {"datasetId": dataset_id} - params = {k: v for k, v in params.items() if v is not None} url = "/dataset/list-imported-files" response = self._client.request("GET", url, params=params) return [ @@ -850,7 +836,6 @@ def restore_prefilled_tables( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/restorePrefilledTables/{dataset_id}" self._client.request("POST", url, params=params) - return None def update_attachment( self, @@ -889,7 +874,6 @@ def update_attachment( url = f"/dataset/v1/{dataset_id}/field/{field_id}/attachment" with stream_upload_file("file", file) as files: self._client.request("PUT", url, params=params, files=files) - return None def update_geometry( self, @@ -899,7 +883,6 @@ def update_geometry( params = None url = f"/dataset/{dataset_id}/updateGeometry" self._client.request("PUT", url, params=params) - return None def update_geometry_v2( self, @@ -909,7 +892,6 @@ def update_geometry_v2( params = None url = f"/dataset/v2/{dataset_id}/updateGeometry" self._client.request("PUT", url, params=params) - return None class AsyncDatasetsResource: @@ -954,7 +936,6 @@ async def convert_iceberg_to_parquet_tables( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/convertIcebergToParquetTables/{dataset_id}" await self._client.request("POST", url, params=params) - return None async def convert_parquet_to_iceberg_tables( self, @@ -972,7 +953,6 @@ async def convert_parquet_to_iceberg_tables( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/convertParquetToIcebergTables/{dataset_id}" await self._client.request("POST", url, params=params) - return None async def create_empty_tables_v3( self, @@ -984,7 +964,6 @@ async def create_empty_tables_v3( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/{dataset_id}/createEmptyTablesV3" await self._client.request("POST", url, params=params) - return None async def create_public_files( self, @@ -1001,7 +980,6 @@ async def create_public_files( params = {k: v for k, v in params.items() if v is not None} url = "/dataset/createPublicFiles" await self._client.request("POST", url, params=params) - return None async def delete_attachment( self, @@ -1038,7 +1016,6 @@ async def delete_attachment( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/v1/{dataset_id}/field/{field_id}/attachment" await self._client.request("DELETE", url, params=params) - return None async def delete_dataset_data( self, @@ -1114,10 +1091,8 @@ async def download_imported_file( ) -> None: """Downloads a specific imported file for a given dataset and a given filename""" params = {"dataflowId": dataflow_id, "datasetId": dataset_id, "fileName": file_name} - params = {k: v for k, v in params.items() if v is not None} url = "/dataset/download-imported-file" await self._client.download("GET", url, params=params, dest=dest) - return None async def duplicate_field_value_exists( self, @@ -1127,7 +1102,6 @@ async def duplicate_field_value_exists( ) -> bool: """duplicateFieldValueExists""" params = {"tableSchemaId": table_schema_id} - params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/duplicateFieldValueExists/{dataset_id}" response = await self._client.request( "POST", url, params=params, json=field_vo.model_dump(mode="json", exclude_none=True) @@ -1188,7 +1162,6 @@ async def etl_export_dataset( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/v1/{dataset_id}/etlExport" await self._client.download("GET", url, params=params, dest=dest) - return None async def etl_export_dataset_v2( self, @@ -1244,7 +1217,6 @@ async def etl_export_dataset_v2( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/v2/etlExport/{dataset_id}" await self._client.download("GET", url, params=params, dest=dest) - return None async def etl_export_dataset_with_job( self, @@ -1529,7 +1501,6 @@ async def get_attachment( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/v1/{dataset_id}/field/{field_id}/attachment" await self._client.download("GET", url, params=params, dest=dest) - return None async def get_available_for_manual_editing_tables( self, @@ -1610,7 +1581,6 @@ async def get_released_dataset_data_info( "providerCode": provider_code, "tableSchemaId": table_schema_id, } - params = {k: v for k, v in params.items() if v is not None} url = "/dataset/getReleasedDatasetDataInfo" response = await self._client.request("GET", url, params=params) return generated.ReleasedDatasetDataInfoVO.model_validate(response.json()) @@ -1707,7 +1677,6 @@ async def import_file_data( url = f"/dataset/v1/{dataset_id}/importFileData" with stream_upload_file("file", file) as files: await self._client.request("POST", url, params=params, files=files) - return None async def is_iceberg_table_created( self, @@ -1731,7 +1700,6 @@ async def list_imported_files( ) -> list[generated.ImportedFilesDirectoriesVO]: """Returns all imported files for a given dataset""" params = {"datasetId": dataset_id} - params = {k: v for k, v in params.items() if v is not None} url = "/dataset/list-imported-files" response = await self._client.request("GET", url, params=params) return [ @@ -1748,7 +1716,6 @@ async def restore_prefilled_tables( params = {k: v for k, v in params.items() if v is not None} url = f"/dataset/restorePrefilledTables/{dataset_id}" await self._client.request("POST", url, params=params) - return None async def update_attachment( self, @@ -1787,7 +1754,6 @@ async def update_attachment( url = f"/dataset/v1/{dataset_id}/field/{field_id}/attachment" with stream_upload_file("file", file) as files: await self._client.request("PUT", url, params=params, files=files) - return None async def update_geometry( self, @@ -1797,7 +1763,6 @@ async def update_geometry( params = None url = f"/dataset/{dataset_id}/updateGeometry" await self._client.request("PUT", url, params=params) - return None async def update_geometry_v2( self, @@ -1807,4 +1772,3 @@ async def update_geometry_v2( params = None url = f"/dataset/v2/{dataset_id}/updateGeometry" await self._client.request("PUT", url, params=params) - return None diff --git a/reportnet_api/resources/eu_datasets.py b/reportnet_api/resources/eu_datasets.py index 01e0f94..847e295 100644 --- a/reportnet_api/resources/eu_datasets.py +++ b/reportnet_api/resources/eu_datasets.py @@ -24,7 +24,6 @@ def populate_data_from_data_collection( params = {k: v for k, v in params.items() if v is not None} url = f"/euDataset/v1/populateData/dataflow/{dataflow_id}" self._client.request("POST", url, params=params) - return None class AsyncEuDatasetsResource: @@ -41,4 +40,3 @@ async def populate_data_from_data_collection( params = {k: v for k, v in params.items() if v is not None} url = f"/euDataset/v1/populateData/dataflow/{dataflow_id}" await self._client.request("POST", url, params=params) - return None diff --git a/reportnet_api/resources/release_receipts.py b/reportnet_api/resources/release_receipts.py index dafeda3..b5fe2a9 100644 --- a/reportnet_api/resources/release_receipts.py +++ b/reportnet_api/resources/release_receipts.py @@ -39,7 +39,6 @@ def delete_release_receipt_by_dataflow_id( params = None url = f"/release-receipts/dataflow/{dataflow_id}" self._client.request("DELETE", url, params=params) - return None def get_release_receipt_by_dataflow_id( self, @@ -94,7 +93,6 @@ async def delete_release_receipt_by_dataflow_id( params = None url = f"/release-receipts/dataflow/{dataflow_id}" await self._client.request("DELETE", url, params=params) - return None async def get_release_receipt_by_dataflow_id( self, diff --git a/reportnet_api/resources/representative_ext.py b/reportnet_api/resources/representative_ext.py index 5779cbc..8f67ecf 100644 --- a/reportnet_api/resources/representative_ext.py +++ b/reportnet_api/resources/representative_ext.py @@ -18,6 +18,7 @@ from __future__ import annotations import os +from pathlib import Path from typing import TYPE_CHECKING, Any, cast from .._core import ACCEPT_ANY @@ -57,11 +58,10 @@ def _upload(file: str | os.PathLike[str]) -> dict[str, tuple[str, bytes, str]]: backend silently ignores the email; this includes Reportnet's own import result CSV, which has a third `Imported` column). Raises `ValueError`. """ - name = os.path.basename(os.fspath(file)) + name = Path(file).name if not name.lower().endswith(".csv"): raise ValueError(f"The lead reporters file must be a .csv file: {name!r}") - with open(file, "rb") as fh: - text = fh.read().decode("utf-8-sig") + text = Path(file).read_bytes().decode("utf-8-sig") normalized = text.replace("\r\n", "\n").replace("\r", "\n") lines = [line for line in normalized.split("\n") if line.strip()] if not lines or lines[0].split(",")[0].strip().strip('"').casefold() != "representing": @@ -70,7 +70,7 @@ def _upload(file: str | os.PathLike[str]) -> dict[str, tuple[str, bytes, str]]: "(Reportnet discards the first line whatever it holds)" ) for number, line in enumerate(lines[1:], start=2): - if len(line.split(",")) != 2: + if len(line.split(",")) != _LEAD_REPORTER_COLUMNS: raise ValueError( f"{name}: data line {number} must have exactly two comma-separated values " "(provider code and email; Reportnet does not read quotes, and a result CSV with " @@ -100,6 +100,7 @@ def _check_representative( _EXPORT_PATH = "/representative/export/{}" +_LEAD_REPORTER_COLUMNS = 2 # Representing,Email _IMPORT_PATH = "/representative/import/{}/group/{}" _VALIDATE_PATH = "/representative/validateLeadReporters/dataflow/{}" _CREATE_LEAD_PATH = "/representative/{}/leadReporter/dataflow/{}" diff --git a/scripts/_swagger_defs_to_openapi.py b/scripts/_swagger_defs_to_openapi.py index 3079fd6..3061196 100644 --- a/scripts/_swagger_defs_to_openapi.py +++ b/scripts/_swagger_defs_to_openapi.py @@ -9,6 +9,7 @@ import json import sys +from pathlib import Path def _rewrite_refs(obj: object) -> object: @@ -28,7 +29,7 @@ def _rewrite_refs(obj: object) -> object: def main() -> None: src, dst = sys.argv[1], sys.argv[2] - with open(src) as f: + with Path(src).open() as f: spec = json.load(f) schemas = _rewrite_refs(spec.get("definitions", {})) @@ -38,7 +39,7 @@ def main() -> None: "paths": {}, "components": {"schemas": schemas}, } - with open(dst, "w") as f: + with Path(dst).open("w") as f: json.dump(doc, f) diff --git a/scripts/generate_docs.py b/scripts/generate_docs.py index fe80048..16e8fe3 100644 --- a/scripts/generate_docs.py +++ b/scripts/generate_docs.py @@ -30,7 +30,8 @@ def _load_script(name: str) -> ModuleType: spec = importlib.util.spec_from_file_location(name, SCRIPTS_DIR / f"{name}.py") - assert spec is not None and spec.loader is not None + if spec is None or spec.loader is None: + raise ImportError(f"cannot load scripts/{name}.py") module = importlib.util.module_from_spec(spec) sys.modules[name] = module # dataclasses resolve annotations via sys.modules spec.loader.exec_module(module) @@ -366,7 +367,7 @@ def build_pages() -> tuple[dict[str, str], dict[str, list[MethodDoc]]]: ): continue prop = isinstance(member, property) - func = member.fget if prop else member + func = member.fget if isinstance(member, property) else member d = build_method_doc( f"{cls.__name__}.{n}", func, @@ -437,8 +438,11 @@ def build_pages() -> tuple[dict[str, str], dict[str, list[MethodDoc]]]: """ +_SUMMARY_WIDTH = 110 # index entries longer than this are cut with "..." + + def render_index(docs_by_page: dict[str, list[MethodDoc]]) -> str: - titles = {p: t for p, t in RESOURCE_PAGES} | {p: t for p, _m, t in MODULE_PAGES} + titles = dict(RESOURCE_PAGES) | {p: t for p, _m, t in MODULE_PAGES} titles["handles"] = HANDLES_TITLE lines = [INDEX_INTRO.rstrip("\n"), ""] order = [p for p, _ in RESOURCE_PAGES] + [p for p, _m, _t in MODULE_PAGES] + ["handles"] @@ -446,7 +450,11 @@ def render_index(docs_by_page: dict[str, list[MethodDoc]]) -> str: docs = docs_by_page[page] lines += [f"### [{titles[page]}]({page}.md)", ""] for d in docs: - summary = d.summary if len(d.summary) <= 110 else d.summary[:107].rstrip() + "..." + summary = ( + d.summary + if len(d.summary) <= _SUMMARY_WIDTH + else d.summary[: _SUMMARY_WIDTH - 3].rstrip() + "..." + ) anchor = d.name.lower().replace(".", "") lines.append(f"- [`{d.name}`]({page}.md#{anchor}): {esc(summary)}") lines.append("") diff --git a/scripts/generate_resources.py b/scripts/generate_resources.py index 3f0a965..885ddd8 100644 --- a/scripts/generate_resources.py +++ b/scripts/generate_resources.py @@ -17,6 +17,7 @@ import sys from dataclasses import dataclass, field from pathlib import Path +from typing import Any REPO_ROOT = Path(__file__).resolve().parent.parent SPEC_PATH = REPO_ROOT / "spec" / "api-docs.json" @@ -224,14 +225,17 @@ class Operation: # neither JSON nor `*/*` (Spring 406s the client's JSON default otherwise). accept: str | None = None + def path_fstring(self) -> str: + return _path_fstring(self) -def swagger_param_type(p: dict) -> str: + +def swagger_param_type(p: dict[str, Any]) -> str: if p.get("type") == "array": item_type = SWAGGER_TO_PY.get(p.get("items", {}).get("type"), "str") return f"list[{item_type}]" if p.get("type") == "file": return "str" - return SWAGGER_TO_PY.get(p.get("type"), "str") + return SWAGGER_TO_PY.get(p.get("type", ""), "str") def accept_for(produces: list[str]) -> str | None: @@ -250,7 +254,7 @@ def accept_for(produces: list[str]) -> str | None: return ", ".join([*produces, "*/*;q=0.1"]) -def parse_operation(method: str, path: str, op: dict) -> Operation: +def parse_operation(method: str, path: str, op: dict[str, Any]) -> Operation: method_name = method_name_for(op["operationId"]) operation = Operation( method=method.upper(), @@ -432,7 +436,8 @@ def render_body(op: Operation, *, is_async: bool) -> str: if query_items: pairs = ", ".join(f'"{p.wire_name}": {p.py_name}' for p in query_items) lines.append(f" params = {{{pairs}}}") - lines.append(" params = {k: v for k, v in params.items() if v is not None}") + if any(not p.required or "None" in p.py_type for p in query_items): + lines.append(" params = {k: v for k, v in params.items() if v is not None}") else: lines.append(" params = None") @@ -444,7 +449,6 @@ def render_body(op: Operation, *, is_async: bool) -> str: lines.append( f' {await_kw}self._client.download("{op.method}", url, params=params, dest=dest)' ) - lines.append(" return None") return "\n".join(lines) call_kwargs = ["params=params"] @@ -485,7 +489,7 @@ def finish_kwargs() -> None: ) if op.return_kind == "none": - lines.append(" return None") + pass # the method simply ends (no trailing `return None`) elif op.return_kind == "raw": lines.append(" return response") elif op.return_kind == "model": @@ -513,9 +517,6 @@ def _path_fstring(self: Operation) -> str: return f'f"{result}"' -Operation.path_fstring = _path_fstring # type: ignore[attr-defined] - - def render_method(op: Operation, *, is_async: bool) -> str: return "\n".join( [ @@ -586,8 +587,9 @@ def main() -> None: import subprocess - subprocess.run(["ruff", "check", "--fix", *out_paths], check=True) - subprocess.run(["ruff", "format", *out_paths], check=True) + # Fixed argv, our own output files; `ruff` comes from the project environment (uv run). + subprocess.run(["ruff", "check", "--fix", *out_paths], check=True) # noqa: S603, S607 + subprocess.run(["ruff", "format", *out_paths], check=True) # noqa: S603, S607 if __name__ == "__main__": diff --git a/scripts/live_guard.py b/scripts/live_guard.py index 8f96864..740cb13 100644 --- a/scripts/live_guard.py +++ b/scripts/live_guard.py @@ -41,7 +41,7 @@ def check(dataflow_id: int) -> None: if __name__ == "__main__": - if len(sys.argv) < 2: + if len(sys.argv) < 2: # noqa: PLR2004 - program name plus at least one id raise SystemExit("usage: live_guard.py [...]") for arg in sys.argv[1:]: check(int(arg)) diff --git a/scripts/make_logo.py b/scripts/make_logo.py index a7ec74f..f6fec4f 100644 --- a/scripts/make_logo.py +++ b/scripts/make_logo.py @@ -44,7 +44,7 @@ CELL, GAP, X0, Y0 = 9, 3, 15.5, 3.5 -def mark(c: dict, ghosts: bool = True) -> str: +def mark(c: dict[str, str], ghosts: bool = True) -> str: out = [] for r, row in enumerate(GRID): for k, ch in enumerate(row): @@ -84,7 +84,7 @@ def svg(w: float, h: float, body: str) -> str: ) -def lockup(c: dict, size: float = 36) -> tuple[str, float]: +def lockup(c: dict[str, str], size: float = 36) -> tuple[str, float]: d1, w1 = text_path("reportnet", 78, 36, size) d2, w2 = text_path("-api", 78 + w1, 36, size) d3, _ = text_path("for Python", 80, 59, 14) diff --git a/scripts/smoke_test.py b/scripts/smoke_test.py index 5c5737a..130a11c 100755 --- a/scripts/smoke_test.py +++ b/scripts/smoke_test.py @@ -77,7 +77,7 @@ def run_check(check: Check) -> tuple[str, str]: return "WARN", f"404 — endpoint reached but nothing found for this id ({e.response_body!r})" except ReportnetAPIError as e: return "FAIL", f"{e.status_code} {type(e).__name__} — {e.response_body!r}" - except Exception as e: # noqa: BLE001 - smoke test wants to report, not crash + except Exception as e: # a smoke test reports every failure rather than crash return "FAIL", f"unexpected {type(e).__name__}: {e}" else: return "PASS", repr(result)[:120] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7ac6d5b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,61 @@ +"""Shared test setup: unit tests must never reach a real server.""" + +import ipaddress +import socket +from typing import Any + +import pytest + + +def _is_local(host: object) -> bool: + """True for loopback hosts: any 127.0.0.0/8 or ::1 address, and localhost names.""" + if isinstance(host, bytes): + host = host.decode("ascii", "replace") + if not isinstance(host, str): + return False + name = host.strip("[]").lower() + if name == "localhost" or name.endswith(".localhost"): + return True + try: + return ipaddress.ip_address(name.split("%", 1)[0]).is_loopback + except ValueError: + return False + + +def _check(sock: socket.socket, address: Any) -> None: + if sock.family == socket.AF_UNIX: + return + host = address[0] if isinstance(address, tuple) else address + if not _is_local(host): + raise RuntimeError(f"Unit test tried to open a network connection to {address!r}") + + +@pytest.fixture(autouse=True) +def _no_network(monkeypatch: pytest.MonkeyPatch) -> None: + """Fail any test that reaches beyond loopback (respx mocks never open sockets).""" + real_connect = socket.socket.connect + real_connect_ex = socket.socket.connect_ex + real_sendto = socket.socket.sendto + real_getaddrinfo = socket.getaddrinfo + + def connect(self: socket.socket, address: Any) -> None: + _check(self, address) + real_connect(self, address) + + def connect_ex(self: socket.socket, address: Any) -> int: + _check(self, address) + return real_connect_ex(self, address) + + def sendto(self: socket.socket, data: Any, *args: Any) -> int: + _check(self, args[-1]) # sendto(data, address) or sendto(data, flags, address) + return real_sendto(self, data, *args) + + def getaddrinfo(host: Any, *args: Any, **kwargs: Any) -> Any: + if host is not None and not _is_local(host): + raise RuntimeError(f"Unit test tried to resolve the host name {host!r}") + return real_getaddrinfo(host, *args, **kwargs) + + monkeypatch.setattr(socket.socket, "connect", connect) + monkeypatch.setattr(socket.socket, "connect_ex", connect_ex) + monkeypatch.setattr(socket.socket, "sendto", sendto) + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..07b67bd --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,16 @@ +"""Small shared builders for tests.""" + +from typing import TypeVar + +import httpx + +from reportnet_api import ReportnetAPIError + +E = TypeVar("E", bound=ReportnetAPIError) + + +def api_error(cls: type[E], status: int, body: str = "", message: str = "x") -> E: + """An API error of type `cls` with a real (minimal) request and response attached.""" + request = httpx.Request("GET", "https://api.reportnet.europa.eu/test") + response = httpx.Response(status, text=body, request=request) + return cls(message, status_code=status, response_body=body, request=request, response=response) diff --git a/tests/test_cli.py b/tests/test_cli.py index be76729..d118708 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,17 +1,20 @@ import io +from pathlib import Path +from typing import NoReturn import pytest from reportnet_api import cli +from reportnet_api.jobs import ProgressPrinter -def run(*argv): +def run(*argv: str) -> tuple[int, str]: out = io.StringIO() code = cli.main(list(argv), out=out) return code, out.getvalue() -def test_demo_listings_are_csv(): +def test_demo_listings_are_csv() -> None: code, text = run("--demo", "dataflows") assert code == 0 and text.splitlines()[0] == "id,name,status" assert "DEMO DATA" in text @@ -21,7 +24,7 @@ def test_demo_listings_are_csv(): assert "Stations" in text -def test_demo_read_to_stdout_and_file(tmp_path): +def test_demo_read_to_stdout_and_file(tmp_path: Path) -> None: code, text = run("--demo", "read", "102", "1") lines = text.splitlines() assert code == 0 and lines[0] == "country_code,Id,Station,Latitude,Commissioned" @@ -32,7 +35,7 @@ def test_demo_read_to_stdout_and_file(tmp_path): assert target.read_text().splitlines()[1].startswith("BE,1,Brussels Ring") -def test_find_and_guide_need_no_credential(monkeypatch): +def test_find_and_guide_need_no_credential(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("REPORTNET_API_KEY", raising=False) code, text = run("find", "export", "table") assert code == 0 and "easy.export_table" in text @@ -40,7 +43,9 @@ def test_find_and_guide_need_no_credential(monkeypatch): assert code == 0 and "Reportnet cheat sheet" in text -def test_errors_are_short_messages_not_tracebacks(monkeypatch, capsys): +def test_errors_are_short_messages_not_tracebacks( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: monkeypatch.delenv("REPORTNET_API_KEY", raising=False) assert cli.main(["dataflows"]) == 2 assert "REPORTNET_API_KEY" in capsys.readouterr().err @@ -48,8 +53,10 @@ def test_errors_are_short_messages_not_tracebacks(monkeypatch, capsys): assert "only knows dataflow 1" in capsys.readouterr().err -def test_no_write_commands_and_credential_not_an_argument(): - choices = set(cli.build_parser()._subparsers._group_actions[0].choices) +def test_no_write_commands_and_credential_not_an_argument() -> None: + subparsers = cli.build_parser()._subparsers + assert subparsers is not None + choices = set(subparsers._group_actions[0].choices or ()) assert choices == { "check", "dataflows", @@ -64,49 +71,52 @@ def test_no_write_commands_and_credential_not_an_argument(): cli.build_parser().parse_args(["--token", "x", "dataflows"]) -def test_check_exit_code_reflects_the_probe(monkeypatch): +def test_check_exit_code_reflects_the_probe(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("REPORTNET_API_KEY", "k") monkeypatch.setenv("REPORTNET_BASE_URL", "http://127.0.0.1:9") # nothing listens here code, text = run("check") assert code == 1 and "credential alive: unknown" in text -def test_missing_credential_message_is_for_the_command_line(monkeypatch, capsys): +def test_missing_credential_message_is_for_the_command_line( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: monkeypatch.delenv("REPORTNET_API_KEY", raising=False) assert cli.main(["dataflows"]) == 2 err = capsys.readouterr().err assert "REPORTNET_API_KEY" in err and "--demo" in err -def test_read_out_leaves_no_file_on_failure(tmp_path): +def test_read_out_leaves_no_file_on_failure(tmp_path: Path) -> None: target = tmp_path / "out.csv" code, _ = run("--demo", "read", "102", "1", "--table", "Nope", "--out", str(target)) assert code != 0 and not target.exists() and list(tmp_path.iterdir()) == [] -def test_unwritable_out_is_a_message_not_a_traceback(capsys): +def test_unwritable_out_is_a_message_not_a_traceback(capsys: pytest.CaptureFixture[str]) -> None: assert cli.main(["--demo", "read", "102", "1", "--out", "/nonexistent/dir/x.csv"]) == 2 assert "error:" in capsys.readouterr().err -def test_read_out_file_gets_the_usual_permissions(tmp_path): - import os +def test_read_out_file_gets_the_usual_permissions(tmp_path: Path) -> None: import stat target = tmp_path / "out.csv" assert run("--demo", "read", "102", "1", "--out", str(target))[0] == 0 - mode = stat.S_IMODE(os.stat(target).st_mode) + mode = stat.S_IMODE(Path(target).stat().st_mode) assert mode == 0o666 & ~cli._UMASK -def test_export_closes_the_progress_line_on_failure(monkeypatch, capsys): +def test_export_closes_the_progress_line_on_failure( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: from reportnet_api import easy from reportnet_api.jobs import JobError - closed = [] - monkeypatch.setattr(cli.ProgressPrinter, "close", lambda self: closed.append(True)) + closed: list[bool] = [] + monkeypatch.setattr(ProgressPrinter, "close", lambda self: closed.append(True)) - def fail(*a, **k): + def fail(*a: object, **k: object) -> NoReturn: raise JobError("job CANCELED") monkeypatch.setattr(easy, "export_to_file", fail) diff --git a/tests/test_client.py b/tests/test_client.py index f7c6501..398d945 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,3 +1,5 @@ +from typing import Any + import httpx import pytest import respx @@ -139,6 +141,8 @@ def test_locked_and_rate_limited_get_hints(status_code: int, fragment: str) -> N [ ({"Retry-After": "7"}, 7.0), ({"Retry-After": "Wed, 21 Oct 2026 07:28:00 GMT"}, None), + ({"Retry-After": "inf"}, None), + ({"Retry-After": "nan"}, None), ({}, None), ], ) @@ -151,9 +155,16 @@ def test_retry_after(headers: dict[str, str], expected: float | None) -> None: assert exc_info.value.retry_after == expected -def _error(status_code: int, scheme: str = "Bearer", path: str = "/dataset/1", **kw): +def _error( + status_code: int, + scheme: str = "Bearer", + path: str = "/dataset/1", + *, + json: object = None, + text: str | None = None, +) -> ReportnetAPIError: with respx.mock(base_url=BASE_URL) as mock: - mock.get(path).mock(return_value=httpx.Response(status_code, **kw)) + mock.get(path).mock(return_value=httpx.Response(status_code, json=json, text=text)) with ReportnetClient(base_url=BASE_URL, token="t", token_scheme=scheme) as client: with pytest.raises(ReportnetAPIError) as exc_info: client.request("GET", path) @@ -226,7 +237,9 @@ def test_html_body_is_not_put_in_message() -> None: (500, "Bearer", "/dataset/1", {"text": "x"}, "server-side"), ], ) -def test_explain(status_code, scheme, path, kwargs, fragment) -> None: +def test_explain( + status_code: int, scheme: str, path: str, kwargs: dict[str, Any], fragment: str +) -> None: text = _error(status_code, scheme, path, **kwargs).explain() assert f"HTTP {status_code}" in text assert fragment in text diff --git a/tests/test_demo.py b/tests/test_demo.py index 7caaf77..70e8153 100644 --- a/tests/test_demo.py +++ b/tests/test_demo.py @@ -1,13 +1,16 @@ +import pytest + import reportnet_api from reportnet_api import easy from reportnet_api.demo import DEMO_DATAFLOW_ID, demo_client -def test_version_is_a_string(): - assert isinstance(reportnet_api.__version__, str) and reportnet_api.__version__ +def test_version_is_a_string() -> None: + assert isinstance(reportnet_api.__version__, str) + assert reportnet_api.__version__ -def test_demo_runs_the_notebook_helpers_without_network(capsys): +def test_demo_runs_the_notebook_helpers_without_network(capsys: pytest.CaptureFixture[str]) -> None: c = demo_client() assert "DEMO MODE" in capsys.readouterr().out flows = easy.list_dataflows(c) @@ -25,13 +28,13 @@ def test_demo_runs_the_notebook_helpers_without_network(capsys): assert easy.is_data_lake(c, DEMO_DATAFLOW_ID) is False -def test_demo_pages_and_pandas_free_read(): +def test_demo_pages_and_pandas_free_read() -> None: c = demo_client(quiet=True) rows = easy.read_records(c, 102, DEMO_DATAFLOW_ID, page_size=2) assert [r["Station"] for r in rows] == ["Vienna Centre", "Graz South", "Linz Harbour"] -def test_demo_refuses_anything_it_does_not_fake(): +def test_demo_refuses_anything_it_does_not_fake() -> None: import pytest from reportnet_api import NotFoundError, ReportnetAPIError diff --git a/tests/test_discover.py b/tests/test_discover.py index 602ef26..b1e6f1e 100644 --- a/tests/test_discover.py +++ b/tests/test_discover.py @@ -1,37 +1,39 @@ +import pytest + import reportnet_api from reportnet_api import easy, find -def test_find_exported_and_matches_names(): +def test_find_exported_and_matches_names() -> None: assert reportnet_api.find is find calls = {m.call for m in find("export")} assert "easy.export_table" in calls assert "client.datasets.etl_export_dataset" in calls -def test_find_all_words_and_underscore_equivalence(): +def test_find_all_words_and_underscore_equivalence() -> None: assert {m.call for m in find("import_file")} == {m.call for m in find("import file")} assert all("read" in m.call or "read" in m.summary.lower() for m in find("read table")) -def test_find_is_case_insensitive_and_covers_handles(): +def test_find_is_case_insensitive_and_covers_handles() -> None: calls = {m.call for m in find("UPLOAD")} assert "ds.upload" in calls -def test_find_empty_and_no_match(): +def test_find_empty_and_no_match() -> None: assert find("") == [] assert "No match" in str(find("zzzqqq")) -def test_summary_never_a_roles_table_and_str_is_a_table(): +def test_summary_never_a_roles_table_and_str_is_a_table() -> None: hits = find("etl") assert hits and all(not m.summary.lower().startswith("allowed roles") for m in hits) assert str(hits).splitlines()[0].startswith(hits[0].call) assert hits[0].doc_link.startswith("docs/reference/datasets.md#") -def test_guide_prints_and_returns(capsys): +def test_guide_prints_and_returns(capsys: pytest.CaptureFixture[str]) -> None: text = easy.guide() assert capsys.readouterr().out.strip() == text.strip() assert "find(" in text @@ -39,7 +41,7 @@ def test_guide_prints_and_returns(capsys): assert capsys.readouterr().out == "" -def test_guide_names_real_helpers(): +def test_guide_names_real_helpers() -> None: for name in ( "connect_interactive", "check_access", diff --git a/tests/test_dtypes.py b/tests/test_dtypes.py index 9fc928c..5ba3654 100644 --- a/tests/test_dtypes.py +++ b/tests/test_dtypes.py @@ -1,4 +1,6 @@ import warnings +from collections.abc import Sequence +from typing import Any import pandas as pd import pytest @@ -18,11 +20,11 @@ } -def _frame(**cols): +def _frame(**cols: Sequence[object]) -> dict[str, pd.DataFrame]: return {"T": pd.DataFrame(cols)} -def test_live_formats_convert_and_empty_becomes_na(): +def test_live_formats_convert_and_empty_becomes_na() -> None: # Value text as read live from a sandbox dataset; '' for empty cells. out = easy.apply_schema_dtypes( _frame( @@ -44,14 +46,14 @@ def test_live_formats_convert_and_empty_becomes_na(): assert out["p"].tolist() == ["", ""] # geometry untouched -def test_boolean_true_false_any_case(): +def test_boolean_true_false_any_case() -> None: out = easy.apply_schema_dtypes(_frame(b=["true", "FALSE", ""]), TYPES)["T"] assert str(out["b"].dtype) == "boolean" assert out["b"].tolist()[:2] == [True, False] and out["b"].isna().tolist()[2] @pytest.mark.parametrize( - "col,values", + ("col", "values"), [ ("i", ["1", "x"]), ("i", ["1.5"]), @@ -60,14 +62,14 @@ def test_boolean_true_false_any_case(): ("b", ["yes"]), ], ) -def test_misfit_column_stays_text_with_warning(col, values): +def test_misfit_column_stays_text_with_warning(col: str, values: list[str]) -> None: frames = _frame(**{col: values}) with pytest.warns(RuntimeWarning, match=f"column '{col}'"): out = easy.apply_schema_dtypes(frames, TYPES)["T"] assert out[col].tolist() == values -def test_unknown_table_or_column_untouched_and_input_not_mutated(): +def test_unknown_table_or_column_untouched_and_input_not_mutated() -> None: frames = {"Other": pd.DataFrame({"x": ["1"]}), "T": pd.DataFrame({"i": ["1"], "zz": ["1"]})} with warnings.catch_warnings(): warnings.simplefilter("error") @@ -77,12 +79,12 @@ def test_unknown_table_or_column_untouched_and_input_not_mutated(): assert frames["T"]["i"].tolist() == ["1"] # original still text -def test_read_tables_rejects_unknown_dtypes_value(): +def test_read_tables_rejects_unknown_dtypes_value() -> None: with pytest.raises(ValueError, match="dtypes"): easy.read_tables(easy.connect("k"), 1, 1, dtypes="bogus") # type: ignore[arg-type] -def _rec(rid, **fields): +def _rec(rid: str, **fields: str) -> dict[str, Any]: return { "id_record": rid, "countryCode": "", @@ -90,7 +92,7 @@ def _rec(rid, **fields): } -def test_read_tables_dtypes_schema_and_default(monkeypatch): +def test_read_tables_dtypes_schema_and_default(monkeypatch: pytest.MonkeyPatch) -> None: import httpx import respx diff --git a/tests/test_easy.py b/tests/test_easy.py index 98219b6..a7b3118 100644 --- a/tests/test_easy.py +++ b/tests/test_easy.py @@ -1,15 +1,21 @@ +from collections.abc import Callable +from pathlib import Path +from types import ModuleType +from typing import Any + import httpx import pytest import respx -from reportnet_api import AsyncReportnetClient, RetryPolicy, easy +from reportnet_api import AsyncReportnetClient, ReportnetClient, RetryPolicy, easy +from reportnet_api.jobs import JobError BASE = "https://api.reportnet.europa.eu" -def test_connect_detects_scheme(monkeypatch): +def test_connect_detects_scheme(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("REPORTNET_API_KEY", raising=False) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="No credential"): easy.connect() assert easy.connect("abc")._core.build_headers()["Authorization"] == "ApiKey abc" c = easy.connect("Bearer xyz", base_url=BASE) @@ -19,7 +25,7 @@ def test_connect_detects_scheme(monkeypatch): @respx.mock -def test_list_datasets_flattens(): +def test_list_datasets_flattens() -> None: respx.get(f"{BASE}/dataflow/v1/5").respond( json={ "designDatasets": [{"id": 1, "dataSetName": "d"}], @@ -31,7 +37,9 @@ def test_list_datasets_flattens(): assert df["kind"].tolist() == ["design", "reporting"] -def _row_offset_backend(rows: list[str], *, total: bool = True): +def _row_offset_backend( + rows: list[str], *, total: bool = True +) -> Callable[[httpx.Request], httpx.Response]: """Fake ETL export with the live semantics: `offset` is a 0-based row offset.""" def page(request: httpx.Request) -> httpx.Response: @@ -41,7 +49,7 @@ def page(request: httpx.Request) -> httpx.Response: {"countryCode": "GR", "fields": [{"fieldName": "a", "value": v}]} for v in rows[offset : offset + limit] ] - table: dict = {"tableName": "T", "records": recs} + table: dict[str, Any] = {"tableName": "T", "records": recs} if total: table["totalRecords"] = len(rows) return httpx.Response(200, json={"tables": [table]}) @@ -50,7 +58,7 @@ def page(request: httpx.Request) -> httpx.Response: @respx.mock -def test_read_tables_pages_until_complete(no_schema_check): +def test_read_tables_pages_until_complete(no_schema_check: None) -> None: # Live (sandbox, 2026-10-27): offset is a row offset; offset=1 would skip row 0. route = respx.get(url__regex=rf"{BASE}/dataset/v1/.*").mock( side_effect=_row_offset_backend(["r0", "r1", "r2", "r3", "r4"]) @@ -62,7 +70,7 @@ def test_read_tables_pages_until_complete(no_schema_check): @respx.mock -def test_read_tables_pages_without_total_records(no_schema_check): +def test_read_tables_pages_without_total_records(no_schema_check: None) -> None: # Backend omits totalRecords unless a filter (e.g. tableSchemaId) is given. route = respx.get(url__regex=rf"{BASE}/dataset/v1/.*").mock( side_effect=_row_offset_backend(["a", "b", "c", "d", "e"], total=False) @@ -72,17 +80,17 @@ def test_read_tables_pages_without_total_records(no_schema_check): assert route.call_count == 3 -def test_read_table_needs_name_when_ambiguous(monkeypatch): +def test_read_table_needs_name_when_ambiguous(monkeypatch: pytest.MonkeyPatch) -> None: import pandas as pd monkeypatch.setattr( easy, "read_tables", lambda *a, **k: {"A": pd.DataFrame(), "B": pd.DataFrame()} ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="pass table_name"): easy.read_table(None, 1, 1) # type: ignore[arg-type] -def test_placeholder_ids_give_clear_error(): +def test_placeholder_ids_give_clear_error() -> None: with pytest.raises(ValueError, match="list_dataflows"): easy.list_datasets(easy.connect("k"), None) # type: ignore[arg-type] with pytest.raises(ValueError, match="dataset_id"): @@ -90,38 +98,38 @@ def test_placeholder_ids_give_clear_error(): @respx.mock -def test_forbidden_gets_hint(): +def test_forbidden_gets_hint() -> None: from reportnet_api import ForbiddenError respx.get(f"{BASE}/dataflow/v1/5").respond(403) - with pytest.raises(ForbiddenError, match="Reporter-level only.*mix up"): + with pytest.raises(ForbiddenError, match=r"Reporter-level only.*mix up"): easy.list_datasets(easy.connect("k"), 5) @respx.mock -def test_list_datasets_empty_body_is_not_no_datasets(): +def test_list_datasets_empty_body_is_not_no_datasets() -> None: respx.get(f"{BASE}/dataflow/v1/5").respond(200) with pytest.raises(LookupError, match="unknown or this credential has no access"): easy.list_datasets(easy.connect("k"), 5) @respx.mock -def test_list_datasets_real_empty_dataflow_is_empty_frame(): +def test_list_datasets_real_empty_dataflow_is_empty_frame() -> None: respx.get(f"{BASE}/dataflow/v1/5").respond(json={"id": 5, "reportingDatasets": []}) assert easy.list_datasets(easy.connect("k"), 5).empty @pytest.fixture -def no_schema_check(monkeypatch): +def no_schema_check(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(easy, "_schema_table_names", lambda *a, **k: None) -def _rec(rid: str, value: str) -> dict: +def _rec(rid: str, value: str) -> dict[str, Any]: return {"id_record": rid, "fields": [{"fieldName": "a", "value": value}]} @respx.mock -def test_read_tables_stops_and_dedups_when_offset_is_ignored(no_schema_check): +def test_read_tables_stops_and_dedups_when_offset_is_ignored(no_schema_check: None) -> None: # Backend that ignores `offset`: same full page every time must not loop forever. recs = [_rec("r1", "x"), _rec("r2", "y")] route = respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond( @@ -134,7 +142,7 @@ def test_read_tables_stops_and_dedups_when_offset_is_ignored(no_schema_check): @respx.mock -def test_read_tables_warns_when_total_differs(no_schema_check): +def test_read_tables_warns_when_total_differs(no_schema_check: None) -> None: pages = [ httpx.Response( 200, @@ -151,7 +159,7 @@ def test_read_tables_warns_when_total_differs(no_schema_check): @respx.mock -def test_read_tables_warns_on_table_missing_from_export(monkeypatch): +def test_read_tables_warns_on_table_missing_from_export(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(easy, "_schema_table_names", lambda *a, **k: {"T", "U"}) respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond( json={"tables": [{"tableName": "T", "records": [_rec("r1", "x")]}]} @@ -161,7 +169,7 @@ def test_read_tables_warns_on_table_missing_from_export(monkeypatch): @respx.mock -def test_read_tables_warns_when_export_has_no_rows(no_schema_check): +def test_read_tables_warns_when_export_has_no_rows(no_schema_check: None) -> None: # Live (sandbox, 2026-11-06): a data-lake dataset answers 200 with empty tables. respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond( json={"tables": [{"tableName": "T", "records": []}]} @@ -171,7 +179,7 @@ def test_read_tables_warns_when_export_has_no_rows(no_schema_check): assert out["T"].empty -def _vrow(code, rule): +def _vrow(code: str, rule: str) -> dict[str, Any]: return { "shortCode": code, "levelError": "ERROR", @@ -185,7 +193,7 @@ def _vrow(code, rule): @respx.mock -def test_validation_errors_uses_lake_route_and_pages(): +def test_validation_errors_uses_lake_route_and_pages() -> None: respx.get(f"{BASE}/dataflow/v1/5").respond(json={"bigData": True}) old = respx.get(url__regex=rf"{BASE}/validation/listGroupValidations/.*") @@ -204,7 +212,7 @@ def page(request: httpx.Request) -> httpx.Response: @respx.mock -def test_validation_errors_old_route_and_empty(): +def test_validation_errors_old_route_and_empty() -> None: respx.get(f"{BASE}/dataflow/v1/5").respond(json={"bigData": False}) respx.get(url__regex=rf"{BASE}/validation/listGroupValidations/.*").respond( json={"errors": [], "totalRecords": 0} @@ -214,7 +222,7 @@ def test_validation_errors_old_route_and_empty(): @respx.mock -def test_export_validations_saves_file(tmp_path): +def test_export_validations_saves_file(tmp_path: Path) -> None: respx.post(f"{BASE}/validation/export/9").respond(200) respx.get(url__regex=rf"{BASE}/validation/downloadFile/9.*").respond( content=b"Entity,Table\nTABLE,T\n" @@ -224,7 +232,7 @@ def test_export_validations_saves_file(tmp_path): @respx.mock -def test_validation_summary_groups_by_table_and_level(): +def test_validation_summary_groups_by_table_and_level() -> None: respx.get(f"{BASE}/dataflow/v1/5").respond(json={"bigData": False}) respx.get(url__regex=rf"{BASE}/validation/listGroupValidations/.*").respond( json={"errors": [_vrow("A", "r1"), _vrow("B", "r2")], "totalRecords": 2} @@ -234,8 +242,8 @@ def test_validation_summary_groups_by_table_and_level(): @respx.mock -def test_validate_and_wait_waits_for_new_notification(monkeypatch): - monkeypatch.setattr(easy.time, "sleep", lambda s: None) +def test_validate_and_wait_waits_for_new_notification(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("reportnet_api.easy.time.sleep", lambda s: None) old = {"eventType": "VALIDATION_FINISHED_EVENT", "insertDate": 100, "content": {"datasetId": 9}} other = { "eventType": "VALIDATION_FINISHED_EVENT", @@ -257,22 +265,22 @@ def test_validate_and_wait_waits_for_new_notification(monkeypatch): @respx.mock -def test_validate_and_wait_times_out(monkeypatch): - monkeypatch.setattr(easy.time, "sleep", lambda s: None) +def test_validate_and_wait_times_out(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("reportnet_api.easy.time.sleep", lambda s: None) respx.get(f"{BASE}/notification/findUserNotifications").respond(json={"userNotifications": []}) respx.put(f"{BASE}/orchestrator/jobs/addValidationJob/9").respond(200) - with pytest.raises(easy.JobError): + with pytest.raises(JobError): easy.validate_and_wait(easy.connect("k"), 9, 5, timeout=0) JWT = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ4In0.c2ln-_x" -def _auth(client): +def _auth(client: ReportnetClient) -> str: return client._core.build_headers()["Authorization"] -def test_connect_detects_raw_jwt_and_scheme_override(monkeypatch): +def test_connect_detects_raw_jwt_and_scheme_override(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("REPORTNET_API_KEY", raising=False) assert _auth(easy.connect(JWT)) == f"Bearer {JWT}" assert _auth(easy.connect("a.b")) == "ApiKey a.b" # two parts: not a JWT @@ -283,7 +291,7 @@ def test_connect_detects_raw_jwt_and_scheme_override(monkeypatch): easy.connect("k", scheme="Basic") -def test_connect_token_provider_and_retry(): +def test_connect_token_provider_and_retry() -> None: tokens = iter(["t1", "t2"]) c = easy.connect(token_provider=lambda: next(tokens)) assert _auth(c) == "Bearer t1" @@ -298,15 +306,20 @@ def test_connect_token_provider_and_retry(): assert easy.connect("k", retry=policy)._retry is policy -def test_connect_interactive_uses_getpass(monkeypatch): - prompts = [] - monkeypatch.setattr(easy.getpass, "getpass", lambda p: prompts.append(p) or JWT) +def test_connect_interactive_uses_getpass(monkeypatch: pytest.MonkeyPatch) -> None: + prompts: list[str] = [] + + def getpass(p: str) -> str: + prompts.append(p) + return JWT + + monkeypatch.setattr("reportnet_api.easy.getpass.getpass", getpass) c = easy.connect_interactive(base_url=BASE, retry=True) assert _auth(c) == f"Bearer {JWT}" assert len(prompts) == 1 and JWT not in prompts[0] -def test_client_repr_hides_secret(): +def test_client_repr_hides_secret() -> None: c = easy.connect("sekrit-key", base_url=BASE) assert "sekrit-key" not in repr(c) assert repr(c) == "ReportnetClient(base_url='https://api.reportnet.europa.eu', scheme='ApiKey')" @@ -315,7 +328,7 @@ def test_client_repr_hides_secret(): @respx.mock -def test_check_access_alive_and_dataflow(capsys): +def test_check_access_alive_and_dataflow(capsys: pytest.CaptureFixture[str]) -> None: respx.post(f"{BASE}/dataflow/getDataflows").respond(json={"dataflows": []}) respx.get(f"{BASE}/dataflow/v1/5").respond(json={"designDatasets": [{"id": 1}]}) rows = easy.check_access(easy.connect("secret-key"), 5) @@ -331,7 +344,7 @@ def test_check_access_alive_and_dataflow(capsys): @respx.mock -def test_check_access_dead_key_skips_dataflow(): +def test_check_access_dead_key_skips_dataflow() -> None: respx.post(f"{BASE}/dataflow/getDataflows").respond( 500, text='{"status":500,"message":"401 UNAUTHORIZED"}{"message":"No message available"}' ) @@ -343,7 +356,7 @@ def test_check_access_dead_key_skips_dataflow(): @respx.mock -def test_check_access_access_denied_500_is_rejected_and_hints_scheme(): +def test_check_access_access_denied_500_is_rejected_and_hints_scheme() -> None: # Live (2026-10-26): the listing route answers a dead key, or a Bearer token # sent as ApiKey, with this body. respx.post(f"{BASE}/dataflow/getDataflows").respond( @@ -357,7 +370,7 @@ def test_check_access_access_denied_500_is_rejected_and_hints_scheme(): @respx.mock -def test_check_access_dataflow_verdicts(): +def test_check_access_dataflow_verdicts() -> None: respx.post(f"{BASE}/dataflow/getDataflows").respond(json={}) respx.get(f"{BASE}/dataflow/v1/1").respond( 500, json={"message": "status 500 reading DataFlowControllerZuul#accessReferenceEntity(X)"} @@ -368,18 +381,18 @@ def test_check_access_dataflow_verdicts(): assert easy.check_access(client, 1, show=False)[2]["result"] == "not found" assert easy.check_access(client, 2, show=False)[2]["result"] == "no access" assert easy.check_access(client, 3, show=False)[2]["result"] == "unclear" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="positive integer id"): easy.check_access(client, 0) @respx.mock -def test_check_access_network_error(): +def test_check_access_network_error() -> None: respx.post(f"{BASE}/dataflow/getDataflows").mock(side_effect=httpx.ConnectError("down")) rows = easy.check_access(easy.connect("k"), show=False) assert rows[1]["result"] == "unknown" -def test_connect_scheme_from_env(monkeypatch): +def test_connect_scheme_from_env(monkeypatch: pytest.MonkeyPatch) -> None: uuid = "11111111-2222-3333-4444-555555555555" monkeypatch.setenv("REPORTNET_TOKEN_SCHEME", "bearer") assert "scheme='Bearer'" in repr(easy.connect(uuid)) @@ -392,19 +405,22 @@ def test_connect_scheme_from_env(monkeypatch): assert "scheme='Bearer'" in repr(easy.connect(token_provider=lambda: uuid)) -def test_connect_rejects_bad_env_scheme(monkeypatch): +def test_connect_rejects_bad_env_scheme(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("REPORTNET_TOKEN_SCHEME", "Basic") with pytest.raises(ValueError, match="REPORTNET_TOKEN_SCHEME"): easy.connect("k") @respx.mock -def test_read_records_pages_and_needs_no_pandas(no_schema_check, monkeypatch): +def test_read_records_pages_and_needs_no_pandas( + no_schema_check: None, monkeypatch: pytest.MonkeyPatch +) -> None: import builtins real_import = builtins.__import__ - def no_pandas(name, *a, **k): + # Stands in for builtins.__import__, whose other parameters vary; forwarded as is. + def no_pandas(name: str, *a: Any, **k: Any) -> ModuleType: if name == "pandas" or name.startswith("pandas."): raise ImportError("pandas is blocked in this test") return real_import(name, *a, **k) @@ -419,7 +435,7 @@ def no_pandas(name, *a, **k): @respx.mock -def test_read_records_table_selection(no_schema_check): +def test_read_records_table_selection(no_schema_check: None) -> None: respx.get(url__regex=rf"{BASE}/dataset/v1/.*").mock(side_effect=_row_offset_backend(["x"])) client = easy.connect("k") assert easy.read_records(client, 7, 1, table_name="T")[0]["a"] == "x" @@ -428,7 +444,7 @@ def test_read_records_table_selection(no_schema_check): easy.read_records(client, 7, 1, table_name="Nope") -def test_read_table_without_name_ignores_empty_tables(monkeypatch): +def test_read_table_without_name_ignores_empty_tables(monkeypatch: pytest.MonkeyPatch) -> None: import pandas as pd tables = {"Empty": pd.DataFrame(), "Full": pd.DataFrame({"a": [1]})} @@ -438,13 +454,13 @@ def test_read_table_without_name_ignores_empty_tables(monkeypatch): assert easy.read_records(None, 1, 1) == [{"a": 1}] # type: ignore[arg-type] -def test_page_size_must_be_positive(): +def test_page_size_must_be_positive() -> None: with pytest.raises(ValueError, match="page_size"): easy.read_records(easy.connect("k"), 7, 1, page_size=0) @respx.mock -def test_warnings_point_at_the_callers_line(no_schema_check): +def test_warnings_point_at_the_callers_line(no_schema_check: None) -> None: import warnings respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(json={"tables": []}) @@ -454,7 +470,7 @@ def test_warnings_point_at_the_callers_line(no_schema_check): assert caught and all(w.filename == __file__ for w in caught) -def test_read_table_when_every_table_is_empty(monkeypatch): +def test_read_table_when_every_table_is_empty(monkeypatch: pytest.MonkeyPatch) -> None: import pandas as pd monkeypatch.setattr( @@ -464,13 +480,13 @@ def test_read_table_when_every_table_is_empty(monkeypatch): easy.read_table(None, 1, 1) # type: ignore[arg-type] -def test_page_size_accepts_numpy_ints_and_rejects_bools(no_schema_check): +def test_page_size_accepts_numpy_ints_and_rejects_bools(no_schema_check: None) -> None: import numpy as np client = easy.connect("k") for bad in (True, 2.5, "10", -1): with pytest.raises(ValueError, match="page_size"): - easy.read_records(client, 7, 1, page_size=bad) # type: ignore[arg-type] + easy.read_records(client, 7, 1, page_size=bad) with respx.mock: respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(json={"tables": []}) with pytest.warns(RuntimeWarning): @@ -478,7 +494,7 @@ def test_page_size_accepts_numpy_ints_and_rejects_bools(no_schema_check): @respx.mock -def test_validation_errors_explicit_lake_skips_detection(): +def test_validation_errors_explicit_lake_skips_detection() -> None: flow = respx.get(f"{BASE}/dataflow/v1/5").respond(json={"bigData": False}) old = respx.get(url__regex=rf"{BASE}/validation/listGroupValidations/.*") dl = respx.get(url__regex=rf"{BASE}/validation/listGroupValidationsDL/.*").respond( @@ -490,7 +506,7 @@ def test_validation_errors_explicit_lake_skips_detection(): @respx.mock -def test_validation_errors_explicit_relational_skips_detection(): +def test_validation_errors_explicit_relational_skips_detection() -> None: flow = respx.get(f"{BASE}/dataflow/v1/5").respond(json={"bigData": True}) dl = respx.get(url__regex=rf"{BASE}/validation/listGroupValidationsDL/.*") old = respx.get(url__regex=rf"{BASE}/validation/listGroupValidations/.*").respond( @@ -502,8 +518,8 @@ def test_validation_errors_explicit_relational_skips_detection(): @respx.mock -def test_validate_and_wait_passes_data_lake_through(monkeypatch): - monkeypatch.setattr(easy.time, "sleep", lambda s: None) +def test_validate_and_wait_passes_data_lake_through(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("reportnet_api.easy.time.sleep", lambda s: None) old = {"eventType": "VALIDATION_FINISHED_EVENT", "insertDate": 100, "content": {"datasetId": 9}} new = {"eventType": "VALIDATION_FINISHED_EVENT", "insertDate": 200, "content": {"datasetId": 9}} answers = iter([[old], [new, old]]) diff --git a/tests/test_export_fallback.py b/tests/test_export_fallback.py index c7c6dd6..25f7bea 100644 --- a/tests/test_export_fallback.py +++ b/tests/test_export_fallback.py @@ -8,6 +8,8 @@ import io import json import zipfile +from pathlib import Path +from typing import Any import httpx import pytest @@ -20,8 +22,9 @@ V4_POLL = "/orchestrator/jobs/pollForJobStatus/41" V3_POLL = "/orchestrator/jobs/pollForJobStatus/43" V3_DOWNLOAD = "/orchestrator/jobs/downloadEtlExportedFile/43" +# The backend's exact text, typographic quote and dash included. REFUSED_INFO = ( - "ETL Export v4 isn’t compatible with non–big data dataflows. " + "ETL Export v4 isn’t compatible with non–big data dataflows. " # noqa: RUF001 "Please use a supported export version." ) SCHEMA = { @@ -73,7 +76,7 @@ } -def _zip_of(name: str, payload: dict) -> bytes: +def _zip_of(name: str, payload: dict[str, Any]) -> bytes: buf = io.BytesIO() with zipfile.ZipFile(buf, "w") as z: z.writestr(name, json.dumps(payload)) @@ -99,7 +102,7 @@ def test_job_error_carries_status_and_info() -> None: assert JobError("plain").status is None -def test_non_lake_csv_export_falls_back_to_v3(tmp_path) -> None: +def test_non_lake_csv_export_falls_back_to_v3(tmp_path: Path) -> None: dest = tmp_path / "out.zip" with respx.mock(base_url=BASE_URL) as mock: _mock_refused_v4(mock, "/dataset/v4/etlExport/1", big_data=False) @@ -129,18 +132,18 @@ def test_non_lake_csv_export_falls_back_to_v3(tmp_path) -> None: assert not list(tmp_path.glob("*.part")) -def test_lake_dataflow_failure_is_not_retried_with_v3(tmp_path) -> None: +def test_lake_dataflow_failure_is_not_retried_with_v3(tmp_path: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: _mock_refused_v4(mock, "/dataset/v4/etlExport/1", big_data=True) with ( ReportnetClient(base_url=BASE_URL, token="t") as client, - pytest.raises(JobError, match="isn’t compatible"), + pytest.raises(JobError, match="isn’t compatible"), # noqa: RUF001 - backend text ): easy.export_to_file(client, 1, 2, tmp_path / "out.zip") assert not (tmp_path / "out.zip").exists() -def test_non_lake_parquet_export_explains(tmp_path) -> None: +def test_non_lake_parquet_export_explains(tmp_path: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: _mock_refused_v4(mock, "/dataset/v5/etlExport/1", big_data=False) with ( @@ -150,7 +153,7 @@ def test_non_lake_parquet_export_explains(tmp_path) -> None: easy.export_to_file(client, 1, 2, tmp_path / "out.zip", file_format="parquet") -def test_timeout_is_not_retried_with_v3(tmp_path) -> None: +def test_timeout_is_not_retried_with_v3(tmp_path: Path) -> None: with respx.mock(base_url=BASE_URL, assert_all_called=False) as mock: mock.get("/dataset/v4/etlExport/1").mock( return_value=httpx.Response(200, json={"pollingUrl": V4_POLL + "?datasetId=1"}) diff --git a/tests/test_file_export.py b/tests/test_file_export.py index 3c6ca4b..003d6f4 100644 --- a/tests/test_file_export.py +++ b/tests/test_file_export.py @@ -2,6 +2,7 @@ import asyncio import json +from pathlib import Path import httpx import pytest @@ -31,7 +32,7 @@ def test_start_table_export_sends_params_and_filter_body() -> None: assert json.loads(request.content) == {"levelError": ["ERROR"], "idRules": "", "fieldValue": ""} -def test_big_data_variants_use_dl_routes_and_qc_codes(tmp_path) -> None: +def test_big_data_variants_use_dl_routes_and_qc_codes(tmp_path: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: table = mock.post("/dataset/exportFileDL").mock(return_value=httpx.Response(200)) dataset = mock.get("/dataset/1/exportDatasetFileDL").mock(return_value=httpx.Response(200)) @@ -49,7 +50,7 @@ def test_big_data_variants_use_dl_routes_and_qc_codes(tmp_path) -> None: assert dl.called -def test_fetch_returns_false_for_empty_body_and_leaves_no_file(tmp_path) -> None: +def test_fetch_returns_false_for_empty_body_and_leaves_no_file(tmp_path: Path) -> None: dest = tmp_path / "t.csv" with respx.mock(base_url=BASE_URL) as mock: mock.get(DL).mock(return_value=httpx.Response(200, content=b"")) @@ -58,7 +59,7 @@ def test_fetch_returns_false_for_empty_body_and_leaves_no_file(tmp_path) -> None assert list(tmp_path.iterdir()) == [] -def test_fetch_saves_and_returns_true(tmp_path) -> None: +def test_fetch_saves_and_returns_true(tmp_path: Path) -> None: dest = tmp_path / "t.csv" with respx.mock(base_url=BASE_URL) as mock: route = mock.get(DL).mock(return_value=httpx.Response(200, content=b"a,b\n1,2\n")) @@ -69,7 +70,9 @@ def test_fetch_saves_and_returns_true(tmp_path) -> None: assert [p.name for p in tmp_path.iterdir()] == ["t.csv"] # no .part left -def test_export_table_to_file_polls_until_ready(tmp_path, monkeypatch) -> None: +def test_export_table_to_file_polls_until_ready( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr("reportnet_api.resources.datasets_ext.time.sleep", lambda s: None) dest = tmp_path / "t.csv" with respx.mock(base_url=BASE_URL) as mock: @@ -91,7 +94,7 @@ def test_export_table_to_file_polls_until_ready(tmp_path, monkeypatch) -> None: assert dest.read_bytes() == b"ok" -def test_export_times_out_with_job_error(tmp_path, monkeypatch) -> None: +def test_export_times_out_with_job_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("reportnet_api.resources.datasets_ext.time.sleep", lambda s: None) with respx.mock(base_url=BASE_URL) as mock: mock.post("/dataset/exportFileDL").mock(return_value=httpx.Response(200)) @@ -110,7 +113,7 @@ def test_dataset_file_names() -> None: assert _dataset_file_name("Test", "xlsx") == "Test.xlsx" -def test_async_export_table_and_dataset(tmp_path, monkeypatch) -> None: +def test_async_export_table_and_dataset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: async def no_sleep(seconds: float) -> None: return None @@ -135,7 +138,9 @@ async def run() -> None: assert (tmp_path / "d.csv").read_bytes() == b"ok" -def test_easy_export_table_resolves_name_and_rejects_unknown(tmp_path, monkeypatch) -> None: +def test_easy_export_table_resolves_name_and_rejects_unknown( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: pytest.importorskip("pandas") monkeypatch.setattr("reportnet_api.resources.datasets_ext.time.sleep", lambda s: None) schema = { @@ -153,7 +158,9 @@ def test_easy_export_table_resolves_name_and_rejects_unknown(tmp_path, monkeypat assert export.calls.last.request.url.params["tableSchemaId"] == "abc" -def test_existing_server_file_triggers_stale_warning(tmp_path, monkeypatch) -> None: +def test_existing_server_file_triggers_stale_warning( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr("reportnet_api.resources.datasets_ext.time.sleep", lambda s: None) with respx.mock(base_url=BASE_URL) as mock: mock.post("/dataset/exportFileDL").mock(return_value=httpx.Response(200)) @@ -166,7 +173,7 @@ def test_existing_server_file_triggers_stale_warning(tmp_path, monkeypatch) -> N assert (tmp_path / "t.csv").read_bytes() == b"old" -def test_404_while_polling_means_not_ready_and_leaves_no_file(tmp_path) -> None: +def test_404_while_polling_means_not_ready_and_leaves_no_file(tmp_path: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.get(DL).mock(return_value=httpx.Response(404)) with ReportnetClient(base_url=BASE_URL, token="t") as client: @@ -174,7 +181,7 @@ def test_404_while_polling_means_not_ready_and_leaves_no_file(tmp_path) -> None: assert list(tmp_path.iterdir()) == [] -def test_other_errors_while_polling_propagate(tmp_path) -> None: +def test_other_errors_while_polling_propagate(tmp_path: Path) -> None: from reportnet_api import ServerError with respx.mock(base_url=BASE_URL) as mock: @@ -185,7 +192,9 @@ def test_other_errors_while_polling_propagate(tmp_path) -> None: assert list(tmp_path.iterdir()) == [] -def test_sleep_is_clamped_to_remaining_time(tmp_path, monkeypatch) -> None: +def test_sleep_is_clamped_to_remaining_time( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: sleeps: list[float] = [] monkeypatch.setattr("reportnet_api.resources.datasets_ext.time.sleep", sleeps.append) with respx.mock(base_url=BASE_URL) as mock: @@ -205,7 +214,7 @@ def test_sleep_is_clamped_to_remaining_time(tmp_path, monkeypatch) -> None: @pytest.mark.parametrize("bad", ["", ".csv", "zip csv", " csv"]) -def test_table_export_rejects_non_extension_mime(tmp_path, bad: str) -> None: +def test_table_export_rejects_non_extension_mime(tmp_path: Path, bad: str) -> None: with ReportnetClient(base_url=BASE_URL, token="t") as client: with pytest.raises(ValueError, match="bare file extension"): client.datasets.export_table_to_file( @@ -213,7 +222,7 @@ def test_table_export_rejects_non_extension_mime(tmp_path, bad: str) -> None: ) -def test_table_schema_id_is_required(tmp_path) -> None: +def test_table_schema_id_is_required(tmp_path: Path) -> None: with ReportnetClient(base_url=BASE_URL, token="t") as client: with pytest.raises(TypeError): client.datasets.export_table_to_file( # type: ignore[call-arg] @@ -221,7 +230,7 @@ def test_table_schema_id_is_required(tmp_path) -> None: ) -def test_easy_export_table_validates_format_and_duplicate_names(tmp_path) -> None: +def test_easy_export_table_validates_format_and_duplicate_names(tmp_path: Path) -> None: pytest.importorskip("pandas") schema = { "tableSchemas": [ diff --git a/tests/test_generate_resources.py b/tests/test_generate_resources.py index 86daa29..026fea2 100644 --- a/tests/test_generate_resources.py +++ b/tests/test_generate_resources.py @@ -3,6 +3,7 @@ import importlib.util import sys from pathlib import Path +from typing import Any import pytest @@ -29,7 +30,7 @@ def test_accept_for(produces: list[str], expected: str | None) -> None: assert gen.accept_for(produces) == expected -def _op(produces: list[str]) -> dict: +def _op(produces: list[str]) -> dict[str, Any]: return { "operationId": "fooUsingGET", "tags": ["Datasets : Dataset Manager"], diff --git a/tests/test_handles.py b/tests/test_handles.py index 175470d..61eba2d 100644 --- a/tests/test_handles.py +++ b/tests/test_handles.py @@ -1,15 +1,20 @@ """Tests for the Dataflow/Dataset handles (mocked).""" +from collections.abc import Sequence +from pathlib import Path +from typing import NoReturn + import pandas as pd import pytest -from reportnet_api import ForbiddenError, easy -from reportnet_api.handles import Dataflow, find_dataflow +from reportnet_api import ForbiddenError, ReportnetClient, easy +from reportnet_api.handles import Dataflow, Dataset, find_dataflow +from tests.helpers import api_error TOKEN = "SECRET-TOKEN-123" -def _datasets(extra=()): +def _datasets(extra: Sequence[tuple[int, str, str, int | None]] = ()) -> pd.DataFrame: rows = [ (1, "Design", "design", None), (2, "Austria reporting", "reporting", 10), @@ -20,7 +25,7 @@ def _datasets(extra=()): return pd.DataFrame(rows, columns=["dataset_id", "name", "kind", "provider_id"]) -def _providers(): +def _providers() -> pd.DataFrame: return pd.DataFrame( [(10, "AT", "Austria", 1), (11, "EL", "Greece", 1)], columns=["provider_id", "code", "label", "group_id"], @@ -28,47 +33,49 @@ def _providers(): @pytest.fixture -def flow(monkeypatch): +def flow(monkeypatch: pytest.MonkeyPatch) -> Dataflow: client = easy.connect(TOKEN) monkeypatch.setattr(easy, "list_datasets", lambda *a, **k: _datasets()) monkeypatch.setattr(easy, "list_providers", lambda *a, **k: _providers()) return Dataflow(id=5, name="Air", status="DRAFT", big_data=False, client=client) -def test_dataset_by_id(flow): +def test_dataset_by_id(flow: Dataflow) -> None: assert flow.dataset(4).name == "Greece reporting" with pytest.raises(LookupError): flow.dataset(99) -def test_dataset_by_country_code(flow): +def test_dataset_by_country_code(flow: Dataflow) -> None: ds = flow.dataset("el") assert ds.id == 4 and ds.country_code == "EL" -def test_code_matching_two_datasets_lists_candidates(flow): +def test_code_matching_two_datasets_lists_candidates(flow: Dataflow) -> None: with pytest.raises(LookupError) as err: flow.dataset("AT") assert "2" in str(err.value) and "3" in str(err.value) assert flow.dataset("AT", kind="test").id == 3 -def test_exact_name_beats_substring(flow, monkeypatch): +def test_exact_name_beats_substring(flow: Dataflow, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( easy, "list_datasets", lambda *a, **k: _datasets([(5, "Austria", "reporting", None)]) ) assert flow.dataset("austria").id == 5 -def test_unknown_key_lists_available(flow): +def test_unknown_key_lists_available(flow: Dataflow) -> None: with pytest.raises(LookupError) as err: flow.dataset("nothing") assert "Greece reporting" in str(err.value) and "AT" in str(err.value) -def test_forbidden_providers_fall_back_to_names(flow, monkeypatch): - def deny(*a, **k): - raise ForbiddenError("no", status_code=403, response_body="", request=None, response=None) +def test_forbidden_providers_fall_back_to_names( + flow: Dataflow, monkeypatch: pytest.MonkeyPatch +) -> None: + def deny(*a: object, **k: object) -> NoReturn: + raise api_error(ForbiddenError, 403, "", "no") monkeypatch.setattr(easy, "list_providers", deny) assert flow.dataset("Greece reporting").id == 4 @@ -76,54 +83,56 @@ def deny(*a, **k): flow.dataset("EL") -def test_missing_provider_gives_none_country(flow): +def test_missing_provider_gives_none_country(flow: Dataflow) -> None: assert flow.dataset(1).country_code is None -def test_repr_hides_client(flow): +def test_repr_hides_client(flow: Dataflow) -> None: assert "client" not in repr(flow) and TOKEN not in repr(flow) assert TOKEN not in repr(flow.dataset(4)) -def _flows(names=("Air quality 2025", "Air quality archive", "Water")): +def _flows( + names: Sequence[str] = ("Air quality 2025", "Air quality archive", "Water"), +) -> pd.DataFrame: return pd.DataFrame( [(5 + i, n, "DRAFT") for i, n in enumerate(names)], columns=["id", "name", "status"] ) @pytest.fixture -def client(monkeypatch): +def client(monkeypatch: pytest.MonkeyPatch) -> ReportnetClient: monkeypatch.setattr(easy, "list_dataflows", lambda *a, **k: _flows()) monkeypatch.setattr(easy, "is_data_lake", lambda *a, **k: True) return easy.connect(TOKEN) -def test_find_by_name_unique(client): +def test_find_by_name_unique(client: ReportnetClient) -> None: flow = find_dataflow(client, "water") assert flow.id == 7 and flow.big_data is True and flow.status == "DRAFT" -def test_find_ambiguous_lists_candidates(client): +def test_find_ambiguous_lists_candidates(client: ReportnetClient) -> None: with pytest.raises(LookupError) as err: find_dataflow(client, "air quality") assert "5" in str(err.value) and "6" in str(err.value) -def test_find_exact_name_wins(client, monkeypatch): +def test_find_exact_name_wins(client: ReportnetClient, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( easy, "list_dataflows", lambda *a, **k: _flows(("Air quality", "Air quality 2025")) ) assert find_dataflow(client, "AIR QUALITY").id == 5 -def test_find_by_int_id(client, monkeypatch): +def test_find_by_int_id(client: ReportnetClient, monkeypatch: pytest.MonkeyPatch) -> None: assert find_dataflow(client, 7).name == "Water" # Not in the listing but readable: handle named after the id. monkeypatch.setattr(easy, "list_datasets", lambda *a, **k: _datasets()) assert find_dataflow(client, 99).name == "99" - def refuse(*a, **k): + def refuse(*a: object, **k: object) -> NoReturn: raise LookupError("empty") monkeypatch.setattr(easy, "list_datasets", refuse) @@ -131,23 +140,28 @@ def refuse(*a, **k): find_dataflow(client, 98) -def test_find_no_match(client): +def test_find_no_match(client: ReportnetClient) -> None: with pytest.raises(LookupError, match="Water"): find_dataflow(client, "zzz") -def test_easy_find_dataflow_delegates(client): +def test_easy_find_dataflow_delegates(client: ReportnetClient) -> None: assert easy.find_dataflow(client, "water").id == 7 -def test_tables_delegates(flow, monkeypatch): - seen = [] +def test_tables_delegates(flow: Dataflow, monkeypatch: pytest.MonkeyPatch) -> None: + seen: list[int] = [] frame = pd.DataFrame({"name": ["T"]}) - monkeypatch.setattr(easy, "list_tables", lambda c, i: seen.append(i) or frame) + + def list_tables(c: object, i: int) -> pd.DataFrame: + seen.append(i) + return frame + + monkeypatch.setattr(easy, "list_tables", list_tables) assert flow.dataset(4).tables is frame and seen == [4] -def test_refresh_returns_new_handle(flow, monkeypatch): +def test_refresh_returns_new_handle(flow: Dataflow, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(easy, "list_dataflows", lambda *a, **k: _flows(("Renamed",))) monkeypatch.setattr(easy, "is_data_lake", lambda *a, **k: False) new = flow.refresh() @@ -155,9 +169,11 @@ def test_refresh_returns_new_handle(flow, monkeypatch): assert flow.name == "Air" -def test_find_by_int_id_survives_unreadable_listing(client, monkeypatch): - def deny(*a, **k): - raise ForbiddenError("no", status_code=403, response_body="", request=None, response=None) +def test_find_by_int_id_survives_unreadable_listing( + client: ReportnetClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def deny(*a: object, **k: object) -> NoReturn: + raise api_error(ForbiddenError, 403, "", "no") monkeypatch.setattr(easy, "list_dataflows", deny) monkeypatch.setattr(easy, "list_datasets", lambda *a, **k: _datasets()) @@ -166,7 +182,7 @@ def deny(*a, **k): find_dataflow(client, "water") -def test_country_code_without_dataset_never_falls_back_to_a_name(flow): +def test_country_code_without_dataset_never_falls_back_to_a_name(flow: Dataflow) -> None: # "de" is inside "Design", but DE is a real provider with no dataset here. monkey = _providers() monkey.loc[len(monkey)] = (12, "DE", "Germany", 1) @@ -175,16 +191,18 @@ def test_country_code_without_dataset_never_falls_back_to_a_name(flow): flow.dataset("DE") -def test_short_key_without_readable_providers_is_not_a_substring_guess(flow, monkeypatch): - def deny(*a, **k): - raise ForbiddenError("no", status_code=403, response_body="", request=None, response=None) +def test_short_key_without_readable_providers_is_not_a_substring_guess( + flow: Dataflow, monkeypatch: pytest.MonkeyPatch +) -> None: + def deny(*a: object, **k: object) -> NoReturn: + raise api_error(ForbiddenError, 403, "", "no") monkeypatch.setattr(easy, "list_providers", deny) with pytest.raises(LookupError, match="providers"): flow.dataset("DE") -def test_numpy_integer_keys(flow, monkeypatch): +def test_numpy_integer_keys(flow: Dataflow, monkeypatch: pytest.MonkeyPatch) -> None: import numpy as np assert flow.dataset(np.int64(4)).id == 4 @@ -193,25 +211,33 @@ def test_numpy_integer_keys(flow, monkeypatch): assert find_dataflow(flow.client, np.int64(7)).id == 7 -def test_id_lookup_survives_any_provider_failure(flow, monkeypatch): +def test_id_lookup_survives_any_provider_failure( + flow: Dataflow, monkeypatch: pytest.MonkeyPatch +) -> None: from reportnet_api import ServerError - def boom(*a, **k): - raise ServerError("x", status_code=500, response_body="", request=None, response=None) + def boom(*a: object, **k: object) -> NoReturn: + raise api_error(ServerError, 500, "", "x") monkeypatch.setattr(easy, "list_providers", boom) ds = flow.dataset(4) assert ds.id == 4 and ds.country_code is None -def _ds(flow): +def _ds(flow: Dataflow) -> Dataset: return flow.dataset(4) -def test_read_passes_ids_and_provider(flow, monkeypatch): - seen = {} +def test_read_passes_ids_and_provider(flow: Dataflow, monkeypatch: pytest.MonkeyPatch) -> None: + seen: dict[str, object] = {} - def fake(client, dataset_id, dataflow_id, table=None, **kw): + def fake( + client: ReportnetClient, + dataset_id: int, + dataflow_id: int, + table: str | None = None, + **kw: object, + ) -> str: seen.update(ds=dataset_id, flow=dataflow_id, table=table, **kw) return "frame" @@ -222,10 +248,12 @@ def fake(client, dataset_id, dataflow_id, table=None, **kw): assert seen["provider_id"] == 99 -def test_read_all_passes_ids_and_options(flow, monkeypatch): - seen = {} +def test_read_all_passes_ids_and_options(flow: Dataflow, monkeypatch: pytest.MonkeyPatch) -> None: + seen: dict[str, object] = {} - def fake(client, dataset_id, dataflow_id, **kw): + def fake( + client: ReportnetClient, dataset_id: int, dataflow_id: int, **kw: object + ) -> dict[str, object]: seen.update(ds=dataset_id, flow=dataflow_id, **kw) return {} @@ -234,7 +262,9 @@ def fake(client, dataset_id, dataflow_id, **kw): assert seen == {"ds": 4, "flow": 5, "provider_id": 11, "page_size": 50} -def test_validation_methods_delegate(flow, monkeypatch, tmp_path): +def test_validation_methods_delegate( + flow: Dataflow, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: calls = [] monkeypatch.setattr(easy, "validation_errors", lambda c, d, f, **k: calls.append(("e", d, f))) monkeypatch.setattr(easy, "validation_summary", lambda c, d, f: calls.append(("s", d, f))) @@ -248,23 +278,31 @@ def test_validation_methods_delegate(flow, monkeypatch, tmp_path): assert calls == [("e", 4, 5), ("s", 4, 5), ("x", 4, str(tmp_path / "v.csv"))] -def test_find_digit_string_is_an_id_and_empty_is_rejected(client): +def test_find_digit_string_is_an_id_and_empty_is_rejected(client: ReportnetClient) -> None: assert find_dataflow(client, " 7 ").name == "Water" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="query is empty"): find_dataflow(client, " ") -def test_export_upload_validate_delegate(flow, monkeypatch, tmp_path): - calls = [] - monkeypatch.setattr( - easy, "export_to_file", lambda c, d, f, dest, **k: calls.append(("x", d, f, k)) or dest - ) - monkeypatch.setattr( - easy, "import_file", lambda c, d, f, path, **k: calls.append(("u", d, f, k)) or {} - ) - monkeypatch.setattr( - easy, "validate_and_wait", lambda c, d, f, **k: calls.append(("v", d, f, k)) or None - ) +def test_export_upload_validate_delegate( + flow: Dataflow, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + calls: list[tuple[str, int, int, dict[str, object]]] = [] + + def export_to_file(c: object, d: int, f: int, dest: object, **k: object) -> object: + calls.append(("x", d, f, k)) + return dest + + def import_file(c: object, d: int, f: int, path: object, **k: object) -> dict[str, object]: + calls.append(("u", d, f, k)) + return {} + + def validate_and_wait(c: object, d: int, f: int, **k: object) -> None: + calls.append(("v", d, f, k)) + + monkeypatch.setattr(easy, "export_to_file", export_to_file) + monkeypatch.setattr(easy, "import_file", import_file) + monkeypatch.setattr(easy, "validate_and_wait", validate_and_wait) ds = _ds(flow) ds.export(tmp_path / "e.zip", file_format="parquet") ds.upload(tmp_path / "T.csv", replace=True) @@ -276,7 +314,7 @@ def test_export_upload_validate_delegate(flow, monkeypatch, tmp_path): ] -def test_read_many_stacks_and_skips(flow, monkeypatch): +def test_read_many_stacks_and_skips(flow: Dataflow, monkeypatch: pytest.MonkeyPatch) -> None: from reportnet_api.handles import read_many data = { @@ -294,28 +332,42 @@ def test_read_many_stacks_and_skips(flow, monkeypatch): assert read_many(sets[1:], "T").empty -def test_upload_passes_delimiter(flow, monkeypatch, tmp_path): +def test_upload_passes_delimiter( + flow: Dataflow, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: f = tmp_path / "T.csv" f.write_text("a|b\n1|2\n") - seen = {} + seen: dict[str, object] = {} class Datasets: - def import_file_and_wait(self, dataset_id, path, **kw): + def import_file_and_wait( + self, dataset_id: int, path: object, **kw: object + ) -> dict[str, object]: seen.update(kw) return {} - flow.client.datasets = Datasets() + checked: dict[str, object] = {} + + def check( + client: ReportnetClient, dataset_id: int, path: object, *, delimiter: str | None = None + ) -> list[str]: + checked["delimiter"] = delimiter + return [] + + monkeypatch.setattr(easy, "check_import_file", check) # else it fetches the real schema + monkeypatch.setattr(flow.client, "datasets", Datasets()) _ds(flow).upload(f, delimiter="|") assert seen["delimiter"] == "|" and seen["provider_id"] == 11 + assert checked["delimiter"] == "|" -def test_find_by_id_lets_an_authentication_error_through(client, monkeypatch): +def test_find_by_id_lets_an_authentication_error_through( + client: ReportnetClient, monkeypatch: pytest.MonkeyPatch +) -> None: from reportnet_api import AuthenticationError - def deny(*a, **k): - raise AuthenticationError( - "401", status_code=401, response_body="", request=None, response=None - ) + def deny(*a: object, **k: object) -> NoReturn: + raise api_error(AuthenticationError, 401, "", "401") monkeypatch.setattr(easy, "list_dataflows", deny) monkeypatch.setattr(easy, "list_datasets", deny) @@ -323,7 +375,7 @@ def deny(*a, **k): find_dataflow(client, 99) -def test_reads_refuse_a_data_lake_dataflow(monkeypatch): +def test_reads_refuse_a_data_lake_dataflow(monkeypatch: pytest.MonkeyPatch) -> None: from reportnet_api.handles import DataLakeError, read_many client = easy.connect(TOKEN) @@ -335,7 +387,12 @@ def test_reads_refuse_a_data_lake_dataflow(monkeypatch): for call in (ds.read, ds.read_all, lambda: read_many([ds], "T")): with pytest.raises(DataLakeError, match="export"): call() - seen = {} - monkeypatch.setattr(easy, "read_tables", lambda c, d, f, **k: seen.update(k) or {}) + seen: dict[str, object] = {} + + def read_tables(c: object, d: int, f: int, **k: object) -> dict[str, object]: + seen.update(k) + return {} + + monkeypatch.setattr(easy, "read_tables", read_tables) ds.read_all(allow_data_lake=True) assert seen["provider_id"] == 11 diff --git a/tests/test_import_and_wait.py b/tests/test_import_and_wait.py index d6890e4..e196f9e 100644 --- a/tests/test_import_and_wait.py +++ b/tests/test_import_and_wait.py @@ -1,6 +1,9 @@ """Tests for import_file_and_wait (sync + async), the statistics check and easy.import_file.""" import asyncio +from collections.abc import Callable +from pathlib import Path +from typing import Any import httpx import pytest @@ -16,24 +19,24 @@ SMALL = "/dataset/v1/1/importFileData" POLL = "/orchestrator/jobs/pollForJobStatus/9" T = "tbl" -EMPTY: dict = {} +EMPTY: dict[str, Any] = {} ONE = {T: {"lastImportDate": 100, "numberOfRecordsImported": 1, "fileExtension": "csv"}} TWO = {T: {"lastImportDate": 200, "numberOfRecordsImported": 2, "fileExtension": "csv"}} SAME_COUNT_NEW_DATE = {T: {"lastImportDate": 300, "numberOfRecordsImported": 1}} @pytest.fixture -def csv(tmp_path): +def csv(tmp_path: Path) -> Path: f = tmp_path / "a.csv" f.write_text("x\n1\n") return f -def _job(mock): +def _job(mock: respx.MockRouter) -> None: mock.post(BIG).mock(return_value=httpx.Response(200, json={"jobId": 9, "pollingUrl": POLL})) -def _stats(mock, *answers): +def _stats(mock: respx.MockRouter, *answers: dict[str, Any]) -> respx.Route: return mock.get(STATS).mock(side_effect=[httpx.Response(200, json=a) for a in answers]) @@ -47,7 +50,7 @@ def test_statistics_changed_logic() -> None: assert import_statistics_changed(ONE, TWO, T) -def test_big_file_waits_for_job_then_statistics(csv) -> None: +def test_big_file_waits_for_job_then_statistics(csv: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: _job(mock) mock.get(POLL).mock( @@ -64,7 +67,7 @@ def test_big_file_waits_for_job_then_statistics(csv) -> None: assert result == TWO -def test_small_file_polls_statistics_only(csv) -> None: +def test_small_file_polls_statistics_only(csv: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.post(SMALL).mock(return_value=httpx.Response(200)) _stats(mock, EMPTY, EMPTY, ONE) @@ -73,7 +76,7 @@ def test_small_file_polls_statistics_only(csv) -> None: assert result == ONE -def test_job_failure(csv) -> None: +def test_job_failure(csv: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: _job(mock) mock.get(POLL).mock(return_value=httpx.Response(200, json={"status": "FAILED"})) @@ -83,7 +86,7 @@ def test_job_failure(csv) -> None: client.datasets.import_file_and_wait(1, csv, poll_interval=0) -def test_timeout_when_statistics_never_change(csv) -> None: +def test_timeout_when_statistics_never_change(csv: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.post(SMALL).mock(return_value=httpx.Response(200)) mock.get(STATS).mock(return_value=httpx.Response(200, json=ONE)) @@ -94,7 +97,7 @@ def test_timeout_when_statistics_never_change(csv) -> None: ) -def test_job_timeout(csv) -> None: +def test_job_timeout(csv: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: _job(mock) mock.get(POLL).mock(return_value=httpx.Response(200, json={"status": "QUEUED"})) @@ -104,7 +107,7 @@ def test_job_timeout(csv) -> None: client.datasets.import_file_and_wait(1, csv, poll_interval=0, timeout=0) -def test_trust_job_without_statistics_confirmation(csv) -> None: +def test_trust_job_without_statistics_confirmation(csv: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: _job(mock) mock.get(POLL).mock(return_value=httpx.Response(200, json={"status": "FINISHED"})) @@ -116,7 +119,9 @@ def test_trust_job_without_statistics_confirmation(csv) -> None: assert result == ONE -def test_finished_job_with_unchanged_statistics_is_success(csv, monkeypatch) -> None: +def test_finished_job_with_unchanged_statistics_is_success( + csv: Path, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr(datasets_ext, "_STATS_GRACE", 0.0) with respx.mock(base_url=BASE_URL) as mock: _job(mock) @@ -128,7 +133,9 @@ def test_finished_job_with_unchanged_statistics_is_success(csv, monkeypatch) -> @pytest.mark.parametrize("body", [b'{"jobId": 9}', b"null"]) -def test_no_polling_url_with_trusting_job_raises_and_says_upload_was_sent(csv, body) -> None: +def test_no_polling_url_with_trusting_job_raises_and_says_upload_was_sent( + csv: Path, body: bytes +) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.post(BIG).mock(return_value=httpx.Response(200, content=body)) _stats(mock, EMPTY) @@ -137,7 +144,7 @@ def test_no_polling_url_with_trusting_job_raises_and_says_upload_was_sent(csv, b client.datasets.import_file_and_wait(1, csv, confirm_statistics=False) -def test_non_dict_job_answer_falls_back_to_statistics(csv) -> None: +def test_non_dict_job_answer_falls_back_to_statistics(csv: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.post(BIG).mock(return_value=httpx.Response(200, content=b"null")) _stats(mock, EMPTY, ONE) @@ -146,7 +153,7 @@ def test_non_dict_job_answer_falls_back_to_statistics(csv) -> None: assert result == ONE -def test_async_success_failure_and_timeout(csv) -> None: +def test_async_success_failure_and_timeout(csv: Path) -> None: import asyncio async def run() -> None: @@ -174,7 +181,7 @@ async def run() -> None: asyncio.run(run()) -def test_easy_import_file_checks_and_runs(csv) -> None: +def test_easy_import_file_checks_and_runs(csv: Path) -> None: client = easy.connect("k", base_url=BASE_URL) with pytest.raises(ValueError, match="dataset_id"): easy.import_file(client, 0, 2, csv) @@ -187,7 +194,7 @@ def test_easy_import_file_checks_and_runs(csv) -> None: assert easy.import_file(client, 1, 2, csv, timeout=5, check=False) == ONE -def test_easy_import_file_403_hint(csv) -> None: +def test_easy_import_file_403_hint(csv: Path) -> None: client = easy.connect("k", base_url=BASE_URL) with respx.mock(base_url=BASE_URL) as mock: mock.get(STATS).mock(return_value=httpx.Response(403)) @@ -212,7 +219,7 @@ async def async_sleep(seconds: float) -> None: return now -def _slow_upload(clock: list[float]): +def _slow_upload(clock: list[float]) -> Callable[[httpx.Request], httpx.Response]: def upload(request: httpx.Request) -> httpx.Response: clock[0] += 1000 # longer than the whole timeout below return httpx.Response(200) @@ -220,7 +227,7 @@ def upload(request: httpx.Request) -> httpx.Response: return upload -def test_timeout_starts_after_the_upload(csv, clock) -> None: +def test_timeout_starts_after_the_upload(csv: Path, clock: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.post(SMALL).mock(side_effect=_slow_upload(clock)) _stats(mock, EMPTY, EMPTY, ONE) # baseline, not merged yet, merged @@ -231,8 +238,8 @@ def test_timeout_starts_after_the_upload(csv, clock) -> None: assert result == ONE -def test_async_timeout_starts_after_the_upload(csv, clock) -> None: - async def run() -> dict: +def test_async_timeout_starts_after_the_upload(csv: Path, clock: list[float]) -> None: + async def run() -> dict[str, Any]: with respx.mock(base_url=BASE_URL) as mock: mock.post(SMALL).mock(side_effect=_slow_upload(clock)) _stats(mock, EMPTY, EMPTY, ONE) diff --git a/tests/test_import_safety.py b/tests/test_import_safety.py index 51c70fc..ef6a54c 100644 --- a/tests/test_import_safety.py +++ b/tests/test_import_safety.py @@ -1,13 +1,20 @@ +from pathlib import Path from types import SimpleNamespace as NS +from typing import Any, NoReturn, cast import pandas as pd import pytest -from reportnet_api import easy +from reportnet_api import ReportnetAPIError, ReportnetClient, easy from reportnet_api.jobs import JobError +from tests.helpers import api_error +Calls = list[tuple[int, dict[str, Any], str]] -def _client(monkeypatch, calls=None): + +def _client( + monkeypatch: pytest.MonkeyPatch, calls: Calls | None = None +) -> tuple[ReportnetClient, Calls]: schema = { "TestTable": [ ("Id", "NUMBER_INTEGER"), @@ -19,16 +26,17 @@ def _client(monkeypatch, calls=None): "Other": [("Id", "NUMBER_INTEGER")], } monkeypatch.setattr(easy, "_schema_fields", lambda c, d: schema) - done = calls if calls is not None else [] + done: Calls = calls if calls is not None else [] - def fake(dataset_id, path, **kw): - done.append((dataset_id, kw, open(path).read())) + def fake(dataset_id: int, path: str | Path, **kw: Any) -> dict[str, Any]: + done.append((dataset_id, kw, Path(path).read_text())) return {"tid": {"numberOfRecordsImported": 1}} - return NS(datasets=NS(import_file_and_wait=fake)), done + # A duck-typed fake: easy only uses client.datasets.import_file_and_wait here. + return cast(ReportnetClient, NS(datasets=NS(import_file_and_wait=fake))), done -def _csv(tmp_path, text, name="TestTable.csv"): +def _csv(tmp_path: Path, text: str, name: str = "TestTable.csv") -> Path: f = tmp_path / name f.write_text(text, encoding="utf-8") return f @@ -37,7 +45,7 @@ def _csv(tmp_path, text, name="TestTable.csv"): H = "Id,Amount,Day,At,Note" -def test_check_ok_any_delimiter_and_case(monkeypatch, tmp_path): +def test_check_ok_any_delimiter_and_case(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: c, _ = _client(monkeypatch) assert easy.check_import_file(c, 1, _csv(tmp_path, H + "\n1,2,3,4,5\n")) == [] assert ( @@ -47,7 +55,9 @@ def test_check_ok_any_delimiter_and_case(monkeypatch, tmp_path): assert easy.check_import_file(c, 1, _csv(tmp_path, "" + H + "\r\n")) == [] -def test_check_reports_partial_header_wrong_delimiter_and_name(monkeypatch, tmp_path): +def test_check_reports_partial_header_wrong_delimiter_and_name( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: c, _ = _client(monkeypatch) (msg,) = easy.check_import_file(c, 1, _csv(tmp_path, "Id,Note\n1,x\n")) assert "missing" in msg and "Amount" in msg @@ -59,20 +69,22 @@ def test_check_reports_partial_header_wrong_delimiter_and_name(monkeypatch, tmp_ assert "unknown" in msg and "Zed" in msg -def test_check_skips_non_csv_and_unreadable_schema(monkeypatch, tmp_path): +def test_check_skips_non_csv_and_unreadable_schema( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: c, _ = _client(monkeypatch) assert easy.check_import_file(c, 1, tmp_path / "anything.zip") == [] - def boom(*a): - raise easy.ReportnetAPIError( - "x", status_code=500, response_body="", request=None, response=None - ) + def boom(*a: object) -> NoReturn: + raise api_error(ReportnetAPIError, 500, "", "x") monkeypatch.setattr(easy, "_schema_fields", boom) assert easy.check_import_file(c, 1, _csv(tmp_path, "zzz\n")) == [] -def test_import_file_blocks_bad_file_and_unconfirmed_replace(monkeypatch, tmp_path): +def test_import_file_blocks_bad_file_and_unconfirmed_replace( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: c, calls = _client(monkeypatch) with pytest.raises(ValueError, match="would be rejected"): easy.import_file(c, 1, 1, _csv(tmp_path, "Id\n1\n")) @@ -87,18 +99,22 @@ def test_import_file_blocks_bad_file_and_unconfirmed_replace(monkeypatch, tmp_pa assert len(calls) == 2 -def test_import_file_adds_headers_hint_to_cancelled_job(monkeypatch, tmp_path): +def test_import_file_adds_headers_hint_to_cancelled_job( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: c, _ = _client(monkeypatch) - def cancel(*a, **k): + def cancel(*a: object, **k: object) -> NoReturn: raise JobError("Job CANCELED: The uploaded files do not contain any records.") - c.datasets.import_file_and_wait = cancel + monkeypatch.setattr(c.datasets, "import_file_and_wait", cancel) with pytest.raises(JobError, match="wrong headers"): easy.import_file(c, 1, 1, _csv(tmp_path, H + "\n")) -def test_import_dataframe_writes_full_header_in_schema_order(monkeypatch): +def test_import_dataframe_writes_full_header_in_schema_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: c, calls = _client(monkeypatch) df = pd.DataFrame( { @@ -109,12 +125,12 @@ def test_import_dataframe_writes_full_header_in_schema_order(monkeypatch): } ) easy.import_dataframe(c, 1, 1, "TestTable", df) - dataset_id, kw, text = calls[0] + _dataset_id, kw, text = calls[0] assert kw["delimiter"] == "," assert text == ("Id,Amount,Day,At,Note\n1,,2026-11-07,2026-11-07T10:30:00Z,a\n,,,,\n") -def test_import_dataframe_rejects_bad_input(monkeypatch): +def test_import_dataframe_rejects_bad_input(monkeypatch: pytest.MonkeyPatch) -> None: c, calls = _client(monkeypatch) with pytest.raises(KeyError, match="Nope"): easy.import_dataframe(c, 1, 1, "Nope", pd.DataFrame()) @@ -127,7 +143,9 @@ def test_import_dataframe_rejects_bad_input(monkeypatch): assert calls == [] -def test_check_reads_header_of_a_non_utf8_file(monkeypatch, tmp_path): +def test_check_reads_header_of_a_non_utf8_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: # Excel often saves cp1252/latin-1; only the (ASCII) header line is needed. c, _ = _client(monkeypatch) f = tmp_path / "TestTable.csv" @@ -135,7 +153,7 @@ def test_check_reads_header_of_a_non_utf8_file(monkeypatch, tmp_path): assert easy.check_import_file(c, 1, f) == [] -def test_decimals_are_written_as_plain_digits(): +def test_decimals_are_written_as_plain_digits() -> None: fmt = easy._format_cell assert fmt(1e-07, "NUMBER_DECIMAL") == "0.0000001" assert fmt(1e16, "NUMBER_DECIMAL") == "10000000000000000" @@ -144,7 +162,7 @@ def test_decimals_are_written_as_plain_digits(): assert fmt(4.0, "NUMBER_INTEGER") == "4" -def test_import_dataframe_rejects_duplicate_columns(monkeypatch): +def test_import_dataframe_rejects_duplicate_columns(monkeypatch: pytest.MonkeyPatch) -> None: c, calls = _client(monkeypatch) df = pd.DataFrame([[1, 2]], columns=["Id", "Id"]) with pytest.raises(ValueError, match="Duplicate column names"): @@ -152,7 +170,7 @@ def test_import_dataframe_rejects_duplicate_columns(monkeypatch): assert calls == [] -def test_decimals_from_a_typed_read_round_trip(monkeypatch): +def test_decimals_from_a_typed_read_round_trip(monkeypatch: pytest.MonkeyPatch) -> None: # apply_schema_dtypes makes Float64 columns; iterating them yields numpy floats. import numpy as np diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 1077874..0b9ffcf 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -1,6 +1,8 @@ """Tests for the shared Orchestrator job poller and the *_to_file / wait_for_job helpers.""" import asyncio +from pathlib import Path +from typing import Any import httpx import pytest @@ -29,7 +31,7 @@ def _start(mock: respx.MockRouter, route: str) -> None: @pytest.mark.parametrize(("name", "route"), EXPORTS) -def test_export_to_file_polls_then_downloads(tmp_path, name, route) -> None: +def test_export_to_file_polls_then_downloads(tmp_path: Path, name: str, route: str) -> None: dest = tmp_path / "out.zip" with respx.mock(base_url=BASE_URL) as mock: _start(mock, route) @@ -47,7 +49,7 @@ def test_export_to_file_polls_then_downloads(tmp_path, name, route) -> None: @pytest.mark.parametrize(("name", "route"), EXPORTS) -def test_async_export_to_file(tmp_path, name, route) -> None: +def test_async_export_to_file(tmp_path: Path, name: str, route: str) -> None: dest = tmp_path / "out.zip" async def run() -> None: @@ -65,7 +67,7 @@ async def run() -> None: @pytest.mark.parametrize( "status", ["FAILED", "REFUSED", "CANCELED", "CANCELED_BY_ADMIN", "ABORTED", "CANCELLED"] ) -def test_wait_for_job_failure_statuses(status) -> None: +def test_wait_for_job_failure_statuses(status: str) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.get(POLL).mock(return_value=httpx.Response(200, json={"status": status})) with ( @@ -112,7 +114,7 @@ def test_wait_for_import_job_returns_finished_answer() -> None: assert answer == {"status": "FINISHED"} -def test_export_finished_without_download_url(tmp_path) -> None: +def test_export_finished_without_download_url(tmp_path: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: _start(mock, "/dataset/v4/etlExport/1") mock.get(POLL).mock(return_value=httpx.Response(200, json={"status": "FINISHED"})) @@ -141,7 +143,7 @@ async def run() -> None: asyncio.run(run()) -def test_easy_export_to_file(tmp_path) -> None: +def test_easy_export_to_file(tmp_path: Path) -> None: from reportnet_api.easy import export_to_file dest = tmp_path / "out.zip" @@ -161,7 +163,7 @@ def test_failed_job_message_includes_info() -> None: with pytest.raises(JobError, match="CANCELED: File name does not match"): _check({"status": "CANCELED", "info": "File name does not match any table name."}) - with pytest.raises(JobError, match="^Job FAILED$"): + with pytest.raises(JobError, match=r"^Job FAILED$"): _check({"status": "FAILED"}) @@ -195,7 +197,7 @@ def test_every_backend_status_is_handled_deliberately(status: str) -> None: @pytest.mark.parametrize("answer", [{"status": "SOMETHING_NEW"}, {}, {"status": None}]) -def test_unknown_status_keeps_waiting_but_warns(answer: dict) -> None: +def test_unknown_status_keeps_waiting_but_warns(answer: dict[str, Any]) -> None: from reportnet_api.jobs import _check with pytest.warns(RuntimeWarning, match="Unrecognised job status"): diff --git a/tests/test_no_network_guard.py b/tests/test_no_network_guard.py new file mode 100644 index 0000000..38aeedb --- /dev/null +++ b/tests/test_no_network_guard.py @@ -0,0 +1,35 @@ +"""The autouse guard in conftest.py: real network access fails, loopback does not.""" + +import socket + +import pytest + + +@pytest.mark.parametrize("call", ["connect", "connect_ex"]) +def test_remote_connections_are_refused(call: str) -> None: + with socket.socket() as s, pytest.raises(RuntimeError, match="network connection"): + getattr(s, call)(("192.0.2.1", 443)) # TEST-NET-1, never routed + + +def test_udp_sendto_is_refused() -> None: + with ( + socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s, + pytest.raises(RuntimeError, match="network connection"), + ): + s.sendto(b"x", ("192.0.2.1", 53)) + + +def test_resolving_a_public_name_is_refused() -> None: + with pytest.raises(RuntimeError, match="resolve the host name"): + socket.getaddrinfo("api.reportnet.europa.eu", 443) + + +@pytest.mark.parametrize("host", ["127.0.0.1", "127.0.0.2", "localhost"]) +def test_loopback_is_allowed(host: str) -> None: + with socket.socket() as s: + s.settimeout(1) + assert s.connect_ex((host, 9)) != 0 # refused by the OS, not by the guard + + +def test_loopback_names_resolve() -> None: + assert socket.getaddrinfo("localhost", 80) diff --git a/tests/test_notebooks.py b/tests/test_notebooks.py index d93845a..88cf772 100644 --- a/tests/test_notebooks.py +++ b/tests/test_notebooks.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Any import pytest @@ -7,13 +8,13 @@ NOTEBOOKS = sorted(EXAMPLES.glob("*.ipynb")) -def _code_cells(path): +def _code_cells(path: Path) -> tuple[dict[str, Any], list[str]]: nb = json.loads(path.read_text(encoding="utf-8")) return nb, ["".join(c["source"]) for c in nb["cells"] if c["cell_type"] == "code"] @pytest.mark.parametrize("path", NOTEBOOKS, ids=lambda p: p.name) -def test_notebooks_are_clean(path): +def test_notebooks_are_clean(path: Path) -> None: nb, _ = _code_cells(path) assert nb["nbformat"] == 4 for cell in nb["cells"]: @@ -24,9 +25,9 @@ def test_notebooks_are_clean(path): assert "eeadata/reportnet-api-python" in text # the install and badge links -def test_demo_notebook_runs_offline_from_start_to_finish(): +def test_demo_notebook_runs_offline_from_start_to_finish() -> None: _, cells = _code_cells(EXAMPLES / "try_it_demo.ipynb") - namespace: dict = {} + namespace: dict[str, Any] = {} for source in cells: code = "\n".join(line for line in source.splitlines() if not line.startswith("%pip")) exec(compile(code, "try_it_demo", "exec"), namespace) # noqa: S102 diff --git a/tests/test_progress.py b/tests/test_progress.py index 4c04273..e5e83e6 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -1,26 +1,34 @@ """Tests for `progress=`, the `Job` object and Ctrl-C reporting (P10-15). Mocked only.""" import io +from pathlib import Path +from typing import Any import httpx import pytest import respx from reportnet_api import ReportnetClient, easy -from reportnet_api.jobs import Job, JobError, ProgressPrinter, progress_event, wait_for_job +from reportnet_api.jobs import ( + Job, + JobError, + ProgressEvent, + ProgressPrinter, + progress_event, + wait_for_job, +) BASE_URL = "https://api.reportnet.europa.eu" POLL = "/orchestrator/jobs/pollForJobStatus/9" URL = POLL + "?datasetId=1&dataflowId=2" -def _answers(*statuses: str): +def _answers(*statuses: str) -> list[httpx.Response]: return [httpx.Response(200, json={"status": s}) for s in statuses] -def _event(**kw): - base = {"kind": "job", "status": "QUEUED", "elapsed": 3.0, "job_id": "9"} - return progress_event(**{**base, **kw}) +def _event(*, status: str = "QUEUED") -> ProgressEvent: + return progress_event("job", status=status, elapsed=3.0, job_id="9") def test_printer_non_tty_prints_only_on_change() -> None: @@ -59,13 +67,13 @@ def test_printer_page_and_other_kinds() -> None: ] -def test_printer_default_stream_is_stdout_not_a_tty(capsys) -> None: +def test_printer_default_stream_is_stdout_not_a_tty(capsys: pytest.CaptureFixture[str]) -> None: ProgressPrinter()(_event()) # pytest capture is not a tty and ipykernel isn't loaded assert capsys.readouterr().out == "Job 9: QUEUED (3s)\n" -def test_wait_for_job_progress_true_and_callable(capsys) -> None: - events: list[dict] = [] +def test_wait_for_job_progress_true_and_callable(capsys: pytest.CaptureFixture[str]) -> None: + events: list[dict[str, Any]] = [] with respx.mock(base_url=BASE_URL) as mock: mock.get(POLL).mock(side_effect=_answers("QUEUED", "IN_PROGRESS", "FINISHED") * 2) with ReportnetClient(base_url=BASE_URL, token="t") as client: @@ -82,7 +90,7 @@ def test_wait_for_job_progress_true_and_callable(capsys) -> None: assert events[0]["page"] is None -def test_wait_for_job_default_is_silent(capsys) -> None: +def test_wait_for_job_default_is_silent(capsys: pytest.CaptureFixture[str]) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.get(POLL).mock(side_effect=_answers("QUEUED", "FINISHED")) with ReportnetClient(base_url=BASE_URL, token="t") as client: @@ -90,7 +98,7 @@ def test_wait_for_job_default_is_silent(capsys) -> None: assert capsys.readouterr() == ("", "") -def test_progress_shows_failure_status_then_raises(capsys) -> None: +def test_progress_shows_failure_status_then_raises(capsys: pytest.CaptureFixture[str]) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.get(POLL).mock(return_value=httpx.Response(200, json={"status": "CANCELED"})) with ReportnetClient(base_url=BASE_URL, token="t") as client: @@ -99,14 +107,16 @@ def test_progress_shows_failure_status_then_raises(capsys) -> None: assert capsys.readouterr().out == "Job 9: CANCELED (0s)\n" -def _ctrl_c_on_sleep(monkeypatch) -> None: +def _ctrl_c_on_sleep(monkeypatch: pytest.MonkeyPatch) -> None: def interrupt(_seconds: float) -> None: raise KeyboardInterrupt monkeypatch.setattr("reportnet_api.jobs.time.sleep", interrupt) -def test_ctrl_c_with_progress_reports_job_id_and_reraises(capsys, monkeypatch) -> None: +def test_ctrl_c_with_progress_reports_job_id_and_reraises( + capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: _ctrl_c_on_sleep(monkeypatch) with respx.mock(base_url=BASE_URL) as mock: mock.get(POLL).mock(return_value=_answers("QUEUED")[0]) @@ -118,7 +128,9 @@ def test_ctrl_c_with_progress_reports_job_id_and_reraises(capsys, monkeypatch) - assert "keeps running on the server" in err -def test_ctrl_c_without_progress_stays_silent(capsys, monkeypatch) -> None: +def test_ctrl_c_without_progress_stays_silent( + capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: _ctrl_c_on_sleep(monkeypatch) with respx.mock(base_url=BASE_URL) as mock: mock.get(POLL).mock(return_value=_answers("QUEUED")[0]) @@ -128,7 +140,7 @@ def test_ctrl_c_without_progress_stays_silent(capsys, monkeypatch) -> None: assert capsys.readouterr().err == "" -def test_job_status_wait_and_repr(capsys) -> None: +def test_job_status_wait_and_repr(capsys: pytest.CaptureFixture[str]) -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.get(POLL).mock(side_effect=_answers("QUEUED", "IN_PROGRESS", "FINISHED")) with ReportnetClient(base_url=BASE_URL, token="t") as client: @@ -144,7 +156,9 @@ def test_job_status_wait_and_repr(capsys) -> None: assert capsys.readouterr().out.splitlines()[-1] == "Job 9: FINISHED (0s)" -def test_job_wait_ctrl_c_reports_even_without_progress(capsys, monkeypatch) -> None: +def test_job_wait_ctrl_c_reports_even_without_progress( + capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: _ctrl_c_on_sleep(monkeypatch) with respx.mock(base_url=BASE_URL) as mock: mock.get(POLL).mock(return_value=_answers("QUEUED")[0]) @@ -161,8 +175,8 @@ def test_job_without_start_answer_url_raises() -> None: Job.from_polling_url(client, {"jobId": 1}) -def test_read_tables_page_progress(no_schema_check) -> None: - events: list[dict] = [] +def test_read_tables_page_progress(no_schema_check: None) -> None: + events: list[dict[str, Any]] = [] def page(request: httpx.Request) -> httpx.Response: n = int(request.url.params["offset"]) # a 0-based row offset @@ -179,7 +193,9 @@ def page(request: httpx.Request) -> httpx.Response: assert [(e["kind"], e["page"], e["rows"]) for e in events] == [("page", 1, 1), ("page", 2, 2)] -def test_easy_import_file_progress(tmp_path, capsys, monkeypatch) -> None: +def test_easy_import_file_progress( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.setattr("reportnet_api.jobs.time.sleep", lambda _s: None) csv = tmp_path / "a.csv" csv.write_text("x\n1\n") @@ -201,5 +217,5 @@ def test_easy_import_file_progress(tmp_path, capsys, monkeypatch) -> None: @pytest.fixture -def no_schema_check(monkeypatch): +def no_schema_check(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(easy, "_schema_table_names", lambda *a, **k: None) diff --git a/tests/test_reference_dataset.py b/tests/test_reference_dataset.py index bcf01b6..33d702b 100644 --- a/tests/test_reference_dataset.py +++ b/tests/test_reference_dataset.py @@ -15,7 +15,7 @@ def test_set_updatable_sends_lowercase_bool() -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.put("/referenceDataset/7").mock(return_value=httpx.Response(200)) with ReportnetClient(base_url=BASE_URL, token="t") as c: - assert c.reference_dataset.set_updatable(7, False) is None + c.reference_dataset.set_updatable(7, False) assert route.calls.last.request.url.params["updatable"] == "false" c.reference_dataset.set_updatable(7, True) assert route.calls.last.request.url.params["updatable"] == "true" @@ -53,7 +53,7 @@ async def run() -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.put("/referenceDataset/9").mock(return_value=httpx.Response(200)) async with AsyncReportnetClient(base_url=BASE_URL, token="t") as c: - assert await c.reference_dataset.set_updatable(9, True) is None + await c.reference_dataset.set_updatable(9, True) assert request_param(route) == "true" await c.reference_dataset.set_updatable(9, False) assert request_param(route) == "false" diff --git a/tests/test_representative.py b/tests/test_representative.py index ed7dfca..fd52acf 100644 --- a/tests/test_representative.py +++ b/tests/test_representative.py @@ -2,6 +2,7 @@ import asyncio import json +from pathlib import Path import httpx import pytest @@ -134,7 +135,7 @@ def _uploaded(request: httpx.Request) -> bytes: return body[start : body.index(b"\r\n--", start)] -def test_sync_lead_reporter_routes(tmp_path) -> None: +def test_sync_lead_reporter_routes(tmp_path: Path) -> None: upload = tmp_path / "Lead-Reporters.csv" upload.write_bytes(EXPORT_CSV) with respx.mock(base_url=BASE_URL) as mock: @@ -164,7 +165,7 @@ def test_sync_lead_reporter_routes(tmp_path) -> None: representative_id=77, dataflow_id=5, email="a@example.org" ) assert new_id == 901 - assert rep.validate_lead_reporters(5) is None + rep.validate_lead_reporters(5) assert export.calls.last.request.headers["Accept"] == "*/*" request = imported.calls.last.request @@ -186,7 +187,7 @@ def test_sync_lead_reporter_routes(tmp_path) -> None: assert validate.calls.last.request.content == b"" -def test_async_lead_reporter_routes(tmp_path) -> None: +def test_async_lead_reporter_routes(tmp_path: Path) -> None: upload = tmp_path / "list.CSV" upload.write_bytes(EXPORT_CSV) @@ -283,7 +284,7 @@ async def run() -> None: asyncio.run(run()) -def test_import_normalises_line_ends_and_blank_lines(tmp_path) -> None: +def test_import_normalises_line_ends_and_blank_lines(tmp_path: Path) -> None: upload = tmp_path / "crlf.csv" upload.write_bytes(b'\xef\xbb\xbfRepresenting,Email\r\n"AT","a@example.org"\r\n\r\n"BE",""\r\n') with respx.mock(base_url=BASE_URL) as mock: @@ -298,7 +299,7 @@ def test_import_normalises_line_ends_and_blank_lines(tmp_path) -> None: @pytest.mark.parametrize( - "name, content, message", + ("name", "content", "message"), [ ("list.txt", EXPORT_CSV, r"\.csv"), ("list.csv", b'"AT","a@example.org"\n', "header"), @@ -308,7 +309,9 @@ def test_import_normalises_line_ends_and_blank_lines(tmp_path) -> None: ("list.csv", b'Representing,Email\n"AT"\n', "exactly two"), ], ) -def test_import_refuses_files_reportnet_would_misread(tmp_path, name, content, message) -> None: +def test_import_refuses_files_reportnet_would_misread( + tmp_path: Path, name: str, content: bytes, message: str +) -> None: upload = tmp_path / name upload.write_bytes(content) with respx.mock(base_url=BASE_URL, assert_all_called=False) as mock: diff --git a/tests/test_resources_dataflow_ext.py b/tests/test_resources_dataflow_ext.py index d025bd5..5d6b2fb 100644 --- a/tests/test_resources_dataflow_ext.py +++ b/tests/test_resources_dataflow_ext.py @@ -2,6 +2,7 @@ import asyncio import json +from typing import Any import httpx import pytest @@ -44,7 +45,7 @@ def test_dataflows_kinds(kind: str, path: str) -> None: def test_dataflows_bad_kind() -> None: with ReportnetClient(base_url=BASE_URL, token="t") as client: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="kind must be one of"): client.dataflow.dataflows("nope") # type: ignore[arg-type] @@ -55,12 +56,13 @@ def test_dataflow_datasets() -> None: ) with ReportnetClient(base_url=BASE_URL, token="t") as client: result = client.dataflow.dataflow_datasets(4242, provider_id=5) + assert result is not None assert result["id"] == 4242 assert route.calls.last.request.url.params["providerId"] == "5" def test_async_dataflows_and_datasets() -> None: - async def run() -> tuple[dict, dict]: + async def run() -> tuple[dict[str, Any], dict[str, Any] | None]: with respx.mock(base_url=BASE_URL) as mock: mock.post("/dataflow/getDataflows").mock(return_value=httpx.Response(200, json=PAGE)) mock.get("/dataflow/v1/7").mock(return_value=httpx.Response(200, json={"id": 7})) @@ -77,9 +79,9 @@ async def run() -> tuple[dict, dict]: def test_dataflows_paging_needs_both() -> None: with ReportnetClient(base_url=BASE_URL, token="t") as client: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="given together"): client.dataflow.dataflows(page_size=10) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="given together"): client.dataflow.dataflows(page_num=0) @@ -142,7 +144,7 @@ def test_delete_dataflow() -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.delete("/dataflow/13555").mock(return_value=httpx.Response(200)) with ReportnetClient(base_url=BASE_URL, token="t") as client: - assert client.dataflow.delete_dataflow(13555) is None + client.dataflow.delete_dataflow(13555) assert route.called diff --git a/tests/test_resources_dataset_schema.py b/tests/test_resources_dataset_schema.py index a8914d2..4882a88 100644 --- a/tests/test_resources_dataset_schema.py +++ b/tests/test_resources_dataset_schema.py @@ -98,7 +98,7 @@ def test_import_field_schemas(tmp_path: Path) -> None: return_value=httpx.Response(200) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.dataset_schema.import_field_schemas( + client.dataset_schema.import_field_schemas( dataset_schema_id="s1", file=src, dataset_id=1, replace=True, table_schema_id="t1" ) @@ -109,7 +109,6 @@ def test_import_field_schemas(tmp_path: Path) -> None: assert request.url.params["tableSchemaId"] == "t1" assert b"" in request.content assert b'filename="schema.xml"' in request.content - assert result is None def test_update_manually_editable_by_dataset_id() -> None: @@ -118,7 +117,7 @@ def test_update_manually_editable_by_dataset_id() -> None: return_value=httpx.Response(200) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.dataset_schema.update_manually_editable_by_dataset_id( + client.dataset_schema.update_manually_editable_by_dataset_id( dataset_id=1, manually_editable=True ) @@ -126,4 +125,3 @@ def test_update_manually_editable_by_dataset_id() -> None: assert request.method == "PUT" assert request.url.path == "/dataschema/updateManuallyEditable/1" assert request.url.params["manuallyEditable"] == "true" - assert result is None diff --git a/tests/test_resources_dataset_schema_ext.py b/tests/test_resources_dataset_schema_ext.py index 1989c1e..b54b845 100644 --- a/tests/test_resources_dataset_schema_ext.py +++ b/tests/test_resources_dataset_schema_ext.py @@ -5,6 +5,7 @@ import json import zipfile from pathlib import Path +from typing import Any import httpx import pytest @@ -172,7 +173,7 @@ async def async_sleep(seconds: float) -> None: return now -def _wait(client: ReportnetClient, upload: Path, **kwargs: float) -> list[dict]: +def _wait(client: ReportnetClient, upload: Path, **kwargs: float) -> list[dict[str, Any]]: return client.dataset_schema.import_dataflow_schemas_zip_and_wait( 5, upload, **{"poll_interval": 5, **kwargs} ) @@ -210,7 +211,7 @@ def test_import_and_wait_waits_for_datasets_then_stable_export( def test_import_and_wait_needs_a_schema_in_the_zip(tmp_path: Path) -> None: upload = _schema_zip(tmp_path / "s.zip", []) with ReportnetClient(base_url=BASE_URL, token="t") as client: - with pytest.raises(ValueError, match="no .schema"): + with pytest.raises(ValueError, match=r"no .schema"): _wait(client, upload) @@ -325,7 +326,7 @@ def test_import_and_wait_propagates_auth_errors(tmp_path: Path, clock: list[floa def test_async_import_and_wait(tmp_path: Path, clock: list[float]) -> None: upload = _schema_zip(tmp_path / "s.zip", ["A"]) - async def run() -> tuple[list[dict], respx.Route]: + async def run() -> tuple[list[dict[str, Any]], respx.Route]: with respx.mock(base_url=BASE_URL) as mock: mock.get("/dataflow/v1/5").mock( side_effect=[_designs(1), _designs(1), _designs(1, 9), _designs(1, 9)] @@ -413,7 +414,7 @@ def test_async_import_and_wait_keeps_polling_through_transient_errors( ) -> None: upload = _schema_zip(tmp_path / "s.zip", ["A"]) - async def run() -> list[dict]: + async def run() -> list[dict[str, Any]]: with respx.mock(base_url=BASE_URL) as mock: mock.get("/dataflow/v1/5").mock( side_effect=[ diff --git a/tests/test_resources_dataset_snapshot.py b/tests/test_resources_dataset_snapshot.py index 81fe1fc..4aa31f6 100644 --- a/tests/test_resources_dataset_snapshot.py +++ b/tests/test_resources_dataset_snapshot.py @@ -18,12 +18,11 @@ def test_delete_snapshot() -> None: return_value=httpx.Response(200) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.dataset_snapshot.delete_snapshot(id_dataset=1, id_snapshot=7) + client.dataset_snapshot.delete_snapshot(id_dataset=1, id_snapshot=7) request = route.calls.last.request assert request.method == "DELETE" assert request.url.path == "/snapshot/v1/7/dataset/1/delete" - assert result is None def test_export_historic_releases_csv() -> None: diff --git a/tests/test_resources_datasets.py b/tests/test_resources_datasets.py index 0f7850d..94ccd1a 100644 --- a/tests/test_resources_datasets.py +++ b/tests/test_resources_datasets.py @@ -37,7 +37,7 @@ def test_convert_iceberg_to_parquet_tables() -> None: return_value=httpx.Response(200) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.convert_iceberg_to_parquet_tables( + client.datasets.convert_iceberg_to_parquet_tables( dataflow_id=2, dataset_id=1, provider_id=3, table_schema_ids=["a", "b"] ) @@ -46,7 +46,6 @@ def test_convert_iceberg_to_parquet_tables() -> None: assert request.url.params["dataflowId"] == "2" assert request.url.params["providerId"] == "3" assert request.url.params.get_list("tableSchemaIds") == ["a", "b"] - assert result is None def test_convert_parquet_to_iceberg_tables() -> None: @@ -55,37 +54,34 @@ def test_convert_parquet_to_iceberg_tables() -> None: return_value=httpx.Response(200) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.convert_parquet_to_iceberg_tables(dataflow_id=2, dataset_id=1) + client.datasets.convert_parquet_to_iceberg_tables(dataflow_id=2, dataset_id=1) request = route.calls.last.request assert request.url.path == "/dataset/convertParquetToIcebergTables/1" assert request.url.params["dataflowId"] == "2" - assert result is None def test_create_empty_tables_v3() -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.post("/dataset/1/createEmptyTablesV3").mock(return_value=httpx.Response(200)) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.create_empty_tables_v3(dataset_id=1, table_schema_id="t1") + client.datasets.create_empty_tables_v3(dataset_id=1, table_schema_id="t1") request = route.calls.last.request assert request.url.path == "/dataset/1/createEmptyTablesV3" assert request.url.params["tableSchemaId"] == "t1" - assert result is None def test_create_public_files() -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.post("/dataset/createPublicFiles").mock(return_value=httpx.Response(200)) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.create_public_files(dataflow_id=2, provider_id=3) + client.datasets.create_public_files(dataflow_id=2, provider_id=3) request = route.calls.last.request assert request.url.path == "/dataset/createPublicFiles" assert request.url.params["dataflowId"] == "2" assert request.url.params["providerId"] == "3" - assert result is None def test_delete_attachment() -> None: @@ -94,13 +90,12 @@ def test_delete_attachment() -> None: return_value=httpx.Response(200) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.delete_attachment(dataset_id=1, field_id="f1", record_id="r1") + client.datasets.delete_attachment(dataset_id=1, field_id="f1", record_id="r1") request = route.calls.last.request assert request.method == "DELETE" assert request.url.path == "/dataset/v1/1/field/f1/attachment" assert request.url.params["recordId"] == "r1" - assert result is None def test_delete_dataset_data() -> None: @@ -139,7 +134,7 @@ def test_download_imported_file(tmp_path: Path) -> None: return_value=httpx.Response(200, content=content) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.download_imported_file( + client.datasets.download_imported_file( dataflow_id=2, dataset_id=1, file_name="f.csv", dest=dest ) @@ -148,7 +143,6 @@ def test_download_imported_file(tmp_path: Path) -> None: assert request.url.params["datasetId"] == "1" assert request.url.params["fileName"] == "f.csv" assert dest.read_bytes() == content - assert result is None def test_duplicate_field_value_exists() -> None: @@ -176,13 +170,12 @@ def test_etl_export_dataset(tmp_path: Path) -> None: return_value=httpx.Response(200, content=content) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.etl_export_dataset(dataset_id=1, dest=dest, dataflow_id=2) + client.datasets.etl_export_dataset(dataset_id=1, dest=dest, dataflow_id=2) request = route.calls.last.request assert request.url.path == "/dataset/v1/1/etlExport" assert request.url.params["dataflowId"] == "2" assert dest.read_bytes() == content - assert result is None def test_etl_export_dataset_v2(tmp_path: Path) -> None: @@ -194,12 +187,11 @@ def test_etl_export_dataset_v2(tmp_path: Path) -> None: return_value=httpx.Response(200, content=content) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.etl_export_dataset_v2(dataset_id=1, dest=dest, dataflow_id=2) + client.datasets.etl_export_dataset_v2(dataset_id=1, dest=dest, dataflow_id=2) request = route.calls.last.request assert request.url.path == "/dataset/v2/etlExport/1" assert dest.read_bytes() == content - assert result is None def test_etl_export_dataset_with_job() -> None: @@ -305,7 +297,7 @@ def test_get_attachment(tmp_path: Path) -> None: return_value=httpx.Response(200, content=content) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.get_attachment( + client.datasets.get_attachment( dataset_id=1, field_id="f1", dest=dest, dataflow_id=2, record_id="r1" ) @@ -313,7 +305,6 @@ def test_get_attachment(tmp_path: Path) -> None: assert request.url.path == "/dataset/v1/1/field/f1/attachment" assert request.url.params["recordId"] == "r1" assert dest.read_bytes() == content - assert result is None def test_get_available_for_manual_editing_tables() -> None: @@ -439,12 +430,11 @@ def test_restore_prefilled_tables() -> None: return_value=httpx.Response(200) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.restore_prefilled_tables(dataset_id=1, table_schema_id="t1") + client.datasets.restore_prefilled_tables(dataset_id=1, table_schema_id="t1") request = route.calls.last.request assert request.url.path == "/dataset/restorePrefilledTables/1" assert request.url.params["tableSchemaId"] == "t1" - assert result is None def test_update_attachment(tmp_path: Path) -> None: @@ -454,38 +444,33 @@ def test_update_attachment(tmp_path: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.put("/dataset/v1/1/field/f1/attachment").mock(return_value=httpx.Response(200)) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.update_attachment( - dataset_id=1, field_id="f1", file=src, record_id="r1" - ) + client.datasets.update_attachment(dataset_id=1, field_id="f1", file=src, record_id="r1") request = route.calls.last.request assert request.method == "PUT" assert request.url.path == "/dataset/v1/1/field/f1/attachment" assert request.url.params["recordId"] == "r1" assert b'filename="attach.pdf"' in request.content - assert result is None def test_update_geometry() -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.put("/dataset/1/updateGeometry").mock(return_value=httpx.Response(200)) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.update_geometry(dataset_id=1) + client.datasets.update_geometry(dataset_id=1) request = route.calls.last.request assert request.url.path == "/dataset/1/updateGeometry" - assert result is None def test_update_geometry_v2() -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.put("/dataset/v2/1/updateGeometry").mock(return_value=httpx.Response(200)) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.update_geometry_v2(dataset_id=1) + client.datasets.update_geometry_v2(dataset_id=1) request = route.calls.last.request assert request.url.path == "/dataset/v2/1/updateGeometry" - assert result is None def test_dataflow_id_is_required_keyword_only(tmp_path: Path) -> None: diff --git a/tests/test_resources_datasets_ext.py b/tests/test_resources_datasets_ext.py index c28b553..8587be1 100644 --- a/tests/test_resources_datasets_ext.py +++ b/tests/test_resources_datasets_ext.py @@ -16,9 +16,8 @@ def test_delete_import_table() -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.delete(PATH).mock(return_value=httpx.Response(200)) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.datasets.delete_import_table(7, "abc123", dataflow_id=1) + client.datasets.delete_import_table(7, "abc123", dataflow_id=1) - assert result is None request = route.calls.last.request assert request.url.params["dataflowId"] == "1" assert "providerId" not in request.url.params @@ -29,7 +28,7 @@ async def run() -> respx.Route: with respx.mock(base_url=BASE_URL) as mock: route = mock.delete(PATH).mock(return_value=httpx.Response(200)) async with AsyncReportnetClient(base_url=BASE_URL, token="t") as client: - assert await client.datasets.delete_import_table(7, "abc123") is None + await client.datasets.delete_import_table(7, "abc123") return route assert asyncio.run(run()).called @@ -41,7 +40,7 @@ def test_update_dataset_name() -> None: return_value=httpx.Response(200) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - assert client.datasets.update_dataset_name(44266, "Codelist (2)") is None + client.datasets.update_dataset_name(44266, "Codelist (2)") params = route.calls.last.request.url.params assert params["datasetId"] == "44266" @@ -61,7 +60,7 @@ async def run() -> respx.Route: assert asyncio.run(run()).calls.last.request.url.params["datasetName"] == "Reporting data" -def test_export_zip_csv_to_file_polls_then_downloads(tmp_path) -> None: +def test_export_zip_csv_to_file_polls_then_downloads(tmp_path: Path) -> None: import pytest from reportnet_api.resources.datasets_ext import ExportJobError @@ -108,7 +107,7 @@ def test_export_zip_csv_to_file_polls_then_downloads(tmp_path) -> None: client.datasets.export_zip_csv_to_file(1, dataflow_id=2, dest=dest, poll_interval=0) -def test_export_zip_csv_to_file_times_out(tmp_path) -> None: +def test_export_zip_csv_to_file_times_out(tmp_path: Path) -> None: import pytest from reportnet_api.resources.datasets_ext import ExportJobError @@ -128,7 +127,7 @@ def test_export_zip_csv_to_file_times_out(tmp_path) -> None: ) -def test_async_export_zip_csv_to_file(tmp_path) -> None: +def test_async_export_zip_csv_to_file(tmp_path: Path) -> None: poll = "/orchestrator/jobs/pollForJobStatus/9" dest = tmp_path / "out.zip" diff --git a/tests/test_resources_eu_datasets.py b/tests/test_resources_eu_datasets.py index 1fb7999..01fd0c1 100644 --- a/tests/test_resources_eu_datasets.py +++ b/tests/test_resources_eu_datasets.py @@ -14,13 +14,12 @@ def test_populate_data_from_data_collection() -> None: return_value=httpx.Response(200) ) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.eu_datasets.populate_data_from_data_collection(dataflow_id=5, job_id=9) + client.eu_datasets.populate_data_from_data_collection(dataflow_id=5, job_id=9) request = route.calls.last.request assert request.method == "POST" assert request.url.path == "/euDataset/v1/populateData/dataflow/5" assert request.url.params["jobId"] == "9" - assert result is None def test_populate_data_from_data_collection_without_job_id() -> None: diff --git a/tests/test_resources_release_receipts.py b/tests/test_resources_release_receipts.py index 801ac5c..2202daf 100644 --- a/tests/test_resources_release_receipts.py +++ b/tests/test_resources_release_receipts.py @@ -30,12 +30,11 @@ def test_delete_release_receipt_by_dataflow_id() -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.delete("/release-receipts/dataflow/3").mock(return_value=httpx.Response(200)) with ReportnetClient(base_url=BASE_URL, token="t") as client: - result = client.release_receipts.delete_release_receipt_by_dataflow_id(dataflow_id=3) + client.release_receipts.delete_release_receipt_by_dataflow_id(dataflow_id=3) request = route.calls.last.request assert request.method == "DELETE" assert request.url.path == "/release-receipts/dataflow/3" - assert result is None def test_get_release_receipt_by_dataflow_id() -> None: diff --git a/tests/test_resources_user_ext.py b/tests/test_resources_user_ext.py index 4a86b82..fa2de71 100644 --- a/tests/test_resources_user_ext.py +++ b/tests/test_resources_user_ext.py @@ -1,6 +1,7 @@ """Tests for the hand-written User resource.""" import asyncio +from typing import Any import httpx import pytest @@ -40,7 +41,7 @@ def test_refresh_token_error() -> None: def test_async_refresh_token() -> None: - async def run() -> dict: + async def run() -> dict[str, Any]: with respx.mock(base_url=BASE_URL) as mock: mock.post("/user/refreshToken").mock(return_value=httpx.Response(200, json=TOKEN_VO)) async with AsyncReportnetClient(base_url=BASE_URL, token="t") as client: diff --git a/tests/test_retry.py b/tests/test_retry.py index 2defdd7..e04229f 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -1,7 +1,10 @@ """Tests for the opt-in RetryPolicy (sync + async clients).""" import asyncio +from collections.abc import Iterator from datetime import UTC +from pathlib import Path +from typing import Any import httpx import pytest @@ -22,7 +25,7 @@ @pytest.fixture -def sleeps(monkeypatch): +def sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]: calls: list[float] = [] monkeypatch.setattr("reportnet_api.client.time.sleep", calls.append) @@ -37,7 +40,7 @@ def _client(policy: RetryPolicy | None = POLICY) -> ReportnetClient: return ReportnetClient(base_url=BASE_URL, token="t", retry=policy) -def test_no_policy_means_no_retry(sleeps) -> None: +def test_no_policy_means_no_retry(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.get("/x").mock(return_value=httpx.Response(503)) with _client(None) as client, pytest.raises(ServerError): @@ -46,7 +49,7 @@ def test_no_policy_means_no_retry(sleeps) -> None: assert sleeps == [] -def test_get_503_is_retried_with_backoff_then_succeeds(sleeps) -> None: +def test_get_503_is_retried_with_backoff_then_succeeds(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.get("/x").mock( side_effect=[httpx.Response(503), httpx.Response(502), httpx.Response(200, json={})] @@ -57,7 +60,7 @@ def test_get_503_is_retried_with_backoff_then_succeeds(sleeps) -> None: assert sleeps == [0.5, 1.0] -def test_retries_exhausted_raises_last_error(sleeps) -> None: +def test_retries_exhausted_raises_last_error(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.get("/x").mock(return_value=httpx.Response(503)) with _client() as client, pytest.raises(ServerError): @@ -66,7 +69,7 @@ def test_retries_exhausted_raises_last_error(sleeps) -> None: assert sleeps == [0.5, 1.0, 2.0] -def test_backoff_is_capped(sleeps) -> None: +def test_backoff_is_capped(sleeps: list[float]) -> None: policy = RetryPolicy(retries=4, backoff=1.0, max_delay=3.0) with respx.mock(base_url=BASE_URL) as mock: mock.get("/x").mock(return_value=httpx.Response(503)) @@ -75,7 +78,7 @@ def test_backoff_is_capped(sleeps) -> None: assert sleeps == [1.0, 2.0, 3.0, 3.0] -def test_retry_after_replaces_backoff_and_is_capped(sleeps) -> None: +def test_retry_after_replaces_backoff_and_is_capped(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.get("/x").mock( side_effect=[ @@ -89,7 +92,7 @@ def test_retry_after_replaces_backoff_and_is_capped(sleeps) -> None: assert sleeps == [2.0, 4.0] -def test_429_still_raises_typed_error_when_exhausted(sleeps) -> None: +def test_429_still_raises_typed_error_when_exhausted(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.get("/x").mock(return_value=httpx.Response(429, headers={"Retry-After": "1"})) with _client(RetryPolicy(retries=1)) as client: @@ -99,7 +102,7 @@ def test_429_still_raises_typed_error_when_exhausted(sleeps) -> None: @pytest.mark.parametrize("status", [500, 423, 404, 403]) -def test_other_statuses_are_never_retried(sleeps, status: int) -> None: +def test_other_statuses_are_never_retried(sleeps: list[float], status: int) -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.get("/x").mock(return_value=httpx.Response(status)) with _client() as client, pytest.raises(ReportnetAPIError): @@ -108,7 +111,7 @@ def test_other_statuses_are_never_retried(sleeps, status: int) -> None: assert sleeps == [] -def test_status_retry_is_get_only(sleeps) -> None: +def test_status_retry_is_get_only(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.post("/x").mock(return_value=httpx.Response(503)) with _client() as client, pytest.raises(ServerError): @@ -116,7 +119,7 @@ def test_status_retry_is_get_only(sleeps) -> None: assert route.call_count == 1 -def test_connect_error_is_retried_for_any_method(sleeps) -> None: +def test_connect_error_is_retried_for_any_method(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.post("/x").mock(side_effect=[httpx.ConnectError("down"), httpx.Response(200)]) with _client() as client: @@ -124,7 +127,7 @@ def test_connect_error_is_retried_for_any_method(sleeps) -> None: assert route.call_count == 2 -def test_read_timeout_is_retried_for_get_but_not_post(sleeps) -> None: +def test_read_timeout_is_retried_for_get_but_not_post(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: get = mock.get("/x").mock(side_effect=[httpx.ReadTimeout("slow"), httpx.Response(200)]) post = mock.post("/x").mock(side_effect=httpx.ReadTimeout("slow")) @@ -136,7 +139,7 @@ def test_read_timeout_is_retried_for_get_but_not_post(sleeps) -> None: assert post.call_count == 1 -def test_file_uploads_are_not_retried(sleeps, tmp_path) -> None: +def test_file_uploads_are_not_retried(sleeps: list[float], tmp_path: Path) -> None: f = tmp_path / "a.csv" f.write_text("x\n1\n") with respx.mock(base_url=BASE_URL) as mock: @@ -146,7 +149,7 @@ def test_file_uploads_are_not_retried(sleeps, tmp_path) -> None: assert route.call_count == 1 -def test_lock_423_is_not_retried(sleeps) -> None: +def test_lock_423_is_not_retried(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: route = mock.get("/x").mock(return_value=httpx.Response(423)) with _client() as client, pytest.raises(LockedError): @@ -154,7 +157,7 @@ def test_lock_423_is_not_retried(sleeps) -> None: assert route.call_count == 1 -def test_async_retry_and_no_retry_for_post(sleeps) -> None: +def test_async_retry_and_no_retry_for_post(sleeps: list[float]) -> None: async def run() -> None: with respx.mock(base_url=BASE_URL) as mock: get = mock.get("/x").mock(side_effect=[httpx.Response(504), httpx.Response(200)]) @@ -170,7 +173,7 @@ async def run() -> None: assert sleeps == [0.5] -def test_retry_after_nan_and_garbage_fall_back_to_backoff(sleeps) -> None: +def test_retry_after_nan_and_garbage_fall_back_to_backoff(sleeps: list[float]) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.get("/x").mock( side_effect=[ @@ -184,7 +187,7 @@ def test_retry_after_nan_and_garbage_fall_back_to_backoff(sleeps) -> None: assert sleeps == [0.5, 1.0] -def test_retry_after_http_date_is_honoured_and_capped(sleeps) -> None: +def test_retry_after_http_date_is_honoured_and_capped(sleeps: list[float]) -> None: from datetime import datetime, timedelta from email.utils import format_datetime @@ -230,8 +233,8 @@ def test_pool_timeout_is_retried_for_any_method() -> None: assert POLICY.delay(0, "POST", error=httpx.PoolTimeout("busy")) == 0.5 -def test_streamed_bodies_are_not_retried(sleeps) -> None: - def chunks(): +def test_streamed_bodies_are_not_retried(sleeps: list[float]) -> None: + def chunks() -> Iterator[bytes]: yield b"abc" with respx.mock(base_url=BASE_URL) as mock: @@ -244,6 +247,6 @@ def chunks(): @pytest.mark.parametrize( "kwargs", [{"retries": -1}, {"backoff": -1.0}, {"max_delay": -0.1}, {"backoff": float("nan")}] ) -def test_policy_validates_fields(kwargs: dict) -> None: - with pytest.raises(ValueError): +def test_policy_validates_fields(kwargs: dict[str, Any]) -> None: + with pytest.raises(ValueError, match="must be"): RetryPolicy(**kwargs) diff --git a/tests/test_same_origin.py b/tests/test_same_origin.py index 4c1dd3b..9b14afc 100644 --- a/tests/test_same_origin.py +++ b/tests/test_same_origin.py @@ -1,4 +1,5 @@ import asyncio +from pathlib import Path import httpx import pytest @@ -16,7 +17,7 @@ @pytest.mark.parametrize("url", OTHER) @respx.mock -def test_sync_request_and_download_refuse_other_origin(url, tmp_path): +def test_sync_request_and_download_refuse_other_origin(url: str, tmp_path: Path) -> None: route = respx.route().respond(200, json={}) with ReportnetClient(BASE, "SECRET") as c: with pytest.raises(ValueError, match="Refusing to send the credential"): @@ -28,10 +29,10 @@ def test_sync_request_and_download_refuse_other_origin(url, tmp_path): @pytest.mark.parametrize("url", OTHER) @respx.mock -def test_async_request_and_download_refuse_other_origin(url, tmp_path): +def test_async_request_and_download_refuse_other_origin(url: str, tmp_path: Path) -> None: route = respx.route().respond(200, json={}) - async def run(): + async def run() -> None: async with AsyncReportnetClient(BASE, "SECRET") as c: with pytest.raises(ValueError, match="Refusing to send the credential"): await c.request("GET", url) @@ -43,7 +44,7 @@ async def run(): @respx.mock -def test_protocol_relative_url_stays_on_the_configured_host(): +def test_protocol_relative_url_stays_on_the_configured_host() -> None: # httpx reads `//host/x` as a path on the base host, so nothing leaves it. route = respx.route(host="api.example.invalid").respond(200, json={}) with ReportnetClient(BASE, "SECRET") as c: @@ -52,7 +53,7 @@ def test_protocol_relative_url_stays_on_the_configured_host(): @respx.mock -def test_same_origin_paths_and_absolute_urls_still_work(): +def test_same_origin_paths_and_absolute_urls_still_work() -> None: route = respx.get(f"{BASE}/orchestrator/jobs/pollForJobStatus/1").respond(200, json={"ok": 1}) with ReportnetClient(BASE, "SECRET") as c: assert c.request("GET", "/orchestrator/jobs/pollForJobStatus/1").json() == {"ok": 1} @@ -62,12 +63,12 @@ def test_same_origin_paths_and_absolute_urls_still_work(): assert all(call.request.headers["authorization"].endswith("SECRET") for call in route.calls) -def test_polling_url_on_other_host_is_not_followed(): +def test_polling_url_on_other_host_is_not_followed() -> None: from reportnet_api.jobs import wait_for_job - seen = [] + seen: list[str] = [] - def handler(request): + def handler(request: httpx.Request) -> httpx.Response: seen.append(str(request.url)) return httpx.Response(200, json={"status": "FINISHED"}) @@ -78,7 +79,7 @@ def handler(request): assert seen == [] -def test_message_names_scheme_and_port_when_only_they_differ(): +def test_message_names_scheme_and_port_when_only_they_differ() -> None: with ReportnetClient(BASE, "SECRET") as c: with pytest.raises( ValueError, match=r"to http://api\.example\.invalid:.*\(https://api\.example\.invalid\)" @@ -86,14 +87,14 @@ def test_message_names_scheme_and_port_when_only_they_differ(): c.request("GET", "http://api.example.invalid/x") -def test_plain_http_base_url_warns_but_localhost_does_not(recwarn): +def test_plain_http_base_url_warns_but_localhost_does_not(recwarn: pytest.WarningsRecorder) -> None: with pytest.warns(UserWarning, match="plain http"): ReportnetClient("http://api.example.invalid", "SECRET").close() ReportnetClient("http://127.0.0.1:8080", "SECRET").close() assert not [w for w in recwarn if "plain http" in str(w.message)] -def test_insecure_warning_points_at_the_callers_line_through_easy_connect(): +def test_insecure_warning_points_at_the_callers_line_through_easy_connect() -> None: from reportnet_api import easy with pytest.warns(UserWarning, match="plain http") as caught: @@ -102,12 +103,12 @@ def test_insecure_warning_points_at_the_callers_line_through_easy_connect(): @pytest.mark.parametrize("url", ["http://127.0.0.2:8000", "http://app.localhost", "http://[::1]:9"]) -def test_any_loopback_address_counts_as_local(url, recwarn): +def test_any_loopback_address_counts_as_local(url: str, recwarn: pytest.WarningsRecorder) -> None: ReportnetClient(url, "SECRET").close() assert not [w for w in recwarn if "plain http" in str(w.message)] -def test_ipv6_origin_is_bracketed(): +def test_ipv6_origin_is_bracketed() -> None: with ReportnetClient("https://[::1]:9443", "SECRET") as c: with pytest.raises(ValueError, match=r"\(https://\[::1\]:9443\)"): c.request("GET", "https://evil.example/x") diff --git a/tests/test_validation.py b/tests/test_validation.py index 697fba0..b926524 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,6 +1,7 @@ """Tests for the Validation resource (sync + async), mocked.""" import asyncio +from pathlib import Path import httpx import pytest @@ -19,7 +20,7 @@ def test_start_validation_sends_released_only_when_true() -> None: return_value=httpx.Response(200) ) with ReportnetClient(BASE_URL, "t") as c: - assert c.validation.start_validation(7) is None + c.validation.start_validation(7) assert "released" not in route.calls[-1].request.url.params c.validation.start_validation(7, released=True) assert route.calls[-1].request.url.params["released"] == "true" @@ -75,7 +76,7 @@ def test_list_validations_empty_body_is_none() -> None: assert c.validation.list_validations(7) is None -def test_export_waits_for_file(tmp_path) -> None: +def test_export_waits_for_file(tmp_path: Path) -> None: dest = tmp_path / "v.csv" with respx.mock(base_url=BASE_URL) as mock: post = mock.post("/validation/export/7").mock(return_value=httpx.Response(200)) @@ -90,7 +91,7 @@ def test_export_waits_for_file(tmp_path) -> None: assert list(tmp_path.iterdir()) == [dest] # no .part left behind -def test_export_times_out(tmp_path) -> None: +def test_export_times_out(tmp_path: Path) -> None: with respx.mock(base_url=BASE_URL) as mock: mock.post("/validation/export/7").mock(return_value=httpx.Response(200)) mock.get("/validation/downloadFile/7").mock(return_value=httpx.Response(404)) @@ -101,7 +102,7 @@ def test_export_times_out(tmp_path) -> None: ) -def test_async_routes(tmp_path) -> None: +def test_async_routes(tmp_path: Path) -> None: async def run() -> None: with respx.mock(base_url=BASE_URL) as mock: mock.put("/orchestrator/jobs/addValidationJob/7").mock(return_value=httpx.Response(200)) @@ -123,7 +124,7 @@ async def run() -> None: asyncio.run(run()) -def test_download_false_on_404_and_empty_body_and_errors_propagate(tmp_path) -> None: +def test_download_false_on_404_and_empty_body_and_errors_propagate(tmp_path: Path) -> None: from reportnet_api import ForbiddenError dest = tmp_path / "v.csv" @@ -157,7 +158,7 @@ def test_list_validations_query_and_paging_checks() -> None: c.validation.list_validations(7, page_size=0) -def test_async_timeout_and_empty_body(tmp_path) -> None: +def test_async_timeout_and_empty_body(tmp_path: Path) -> None: async def run() -> None: with respx.mock(base_url=BASE_URL) as mock: mock.post("/validation/export/7").mock(return_value=httpx.Response(200)) diff --git a/tools/streamlit_app.py b/tools/streamlit_app.py index bb5d3a3..71f93d0 100644 --- a/tools/streamlit_app.py +++ b/tools/streamlit_app.py @@ -25,6 +25,7 @@ import streamlit as st from pydantic import BaseModel +from streamlit.runtime.uploaded_file_manager import UploadedFile from reportnet_api import ReportnetAPIError, ReportnetClient @@ -98,15 +99,17 @@ def convert_param(name: str, raw: object, inner: object, required: bool) -> tupl if required: raise ValueError("file is required") return False, None + if not isinstance(raw, UploadedFile): + raise TypeError(f"file: expected an uploaded file, got {type(raw).__name__}") tmp_path = Path(tempfile.gettempdir()) / raw.name tmp_path.write_bytes(raw.getvalue()) return True, tmp_path if name == "dest": - Path(raw).parent.mkdir(parents=True, exist_ok=True) + Path(str(raw)).parent.mkdir(parents=True, exist_ok=True) return True, raw - text = (raw or "").strip() + text = str(raw or "").strip() if not text: if required: raise ValueError(f"{name} is required") @@ -164,10 +167,10 @@ def render_result(result: object, dest_path: str | None) -> None: # ReportnetClient prepends the scheme itself, so the prefix is stripped # here to avoid ending up with "Authorization: Bearer Bearer ". if raw_token.lower().startswith("bearer "): - token_scheme = "Bearer" + token_scheme = "Bearer" # noqa: S105 - an auth scheme name, not a secret token = raw_token[len("bearer ") :] else: - token_scheme = "ApiKey" + token_scheme = "ApiKey" # noqa: S105 - an auth scheme name, not a secret token = raw_token st.caption(f"Detected scheme: **{token_scheme}**") @@ -201,7 +204,7 @@ def render_result(result: object, dest_path: str | None) -> None: ) if requires_line: msg = requires_line - if token_scheme == "ApiKey": + if token_scheme == "ApiKey": # noqa: S105 - a scheme name, not a secret msg += " Your detected scheme is ApiKey, which is Reporter-level and will not satisfy this." st.warning(msg) @@ -246,14 +249,14 @@ def render_result(result: object, dest_path: str | None) -> None: kwargs[name] = value if name == "dest": dest_value = str(value) - except (ValueError, Exception) as exc: # noqa: BLE001 - surface any param error inline + except (ValueError, Exception) as exc: # surface any parameter error inline st.error(f"Invalid parameters: {exc}") else: try: result = method(**kwargs) except ReportnetAPIError as exc: st.error(f"{exc.status_code} {type(exc).__name__} — {exc.response_body}") - except Exception as exc: # noqa: BLE001 - surface any call error inline + except Exception as exc: # surface any call error inline st.exception(exc) else: render_result(result, dest_value)