diff --git a/CHANGELOG.md b/CHANGELOG.md index a5815b4..99f63ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes are listed here. The project follows [Semantic Versioning](h while the major version is `0`, a **minor** release (0.x.0) may change the API, and a patch release (0.x.y) only fixes bugs. Breaking changes are called out below. +## 0.1.3 (unreleased) + +- Downloads (`client.download()` and everything built on it, such as `export_to_file`) + write to a temporary file next to `dest` and replace `dest` only when complete: a + failed download no longer leaves a partial file in place of the previous one. +- `export_to_file`'s fallback on non-data-lake dataflows now stays within the one + `timeout` and no longer overwrites a file named `.part`. +- `TooManyRequestsError.retry_after` returns `None` (not `inf`/`nan`) for such + `Retry-After` values. +- Development: strict mypy over the whole project, a wider ruff rule set, warnings as + errors in tests (which also no longer reach the network), pre-commit hooks. + ## 0.1.2 (2026-11-19) - `easy.export_to_file` (and `Dataset.export`) work again on dataflows that are not a diff --git a/docs/reference/easy.md b/docs/reference/easy.md index 88d4576..3a8a5b7 100644 --- a/docs/reference/easy.md +++ b/docs/reference/easy.md @@ -235,8 +235,8 @@ that is not a data lake (live on the sandbox, 2026-11-18). For CSV this then falls back to the older JSON export and writes the ZIP itself: one ``.csv`` per table, comma-separated UTF-8, fields in schema order (then `country_code` when the records carry one), no record id column. That export is -read into memory whole. Parquet has no such -fallback; the `JobError` says so. +read into memory whole. `timeout` covers both jobs together. +Parquet has no such fallback; the `JobError` says so. **Live-tested notes:** diff --git a/pyproject.toml b/pyproject.toml index c72de97..1c50c2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "reportnet-api" -version = "0.1.2" +version = "0.1.3" description = "Python library for the EEA Reportnet 3 REST API" readme = "README.md" license = "MIT" diff --git a/reportnet_api/_core.py b/reportnet_api/_core.py index 88a06c9..20024cd 100644 --- a/reportnet_api/_core.py +++ b/reportnet_api/_core.py @@ -4,6 +4,7 @@ import math import os import sys +import uuid from collections.abc import Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass @@ -214,6 +215,22 @@ def handle_response(self, response: httpx.Response) -> httpx.Response: return response +@contextmanager +def atomic_destination(dest: str | os.PathLike[str]) -> Iterator[Path]: + """Yield a unique temporary path next to `dest`; move it onto `dest` only on success. + + A failed or interrupted write leaves an existing `dest` untouched and no + leftover file. The random suffix keeps concurrent writers apart. + """ + final = Path(dest) + part = final.with_name(f"{final.name}.{uuid.uuid4().hex[:8]}.part") + try: + yield part + part.replace(final) + finally: + part.unlink(missing_ok=True) + + @contextmanager def stream_upload_file( field_name: str, file: str | os.PathLike[str] diff --git a/reportnet_api/async_client.py b/reportnet_api/async_client.py index 4a2fb88..37b11b7 100644 --- a/reportnet_api/async_client.py +++ b/reportnet_api/async_client.py @@ -2,7 +2,6 @@ import asyncio import os -from pathlib import Path from typing import Any import httpx @@ -13,6 +12,7 @@ DEFAULT_TIMEOUT, RequestCore, RetryPolicy, + atomic_destination, ensure_same_origin, request_is_replayable, warn_if_insecure, @@ -100,7 +100,7 @@ async def download( if not response.is_success: await response.aread() self._core.handle_response(response) - with Path(dest).open("wb") as fh: + with atomic_destination(dest) as part, part.open("wb") as fh: async for chunk in response.aiter_bytes(): fh.write(chunk) diff --git a/reportnet_api/client.py b/reportnet_api/client.py index fb48309..7e263db 100644 --- a/reportnet_api/client.py +++ b/reportnet_api/client.py @@ -2,7 +2,6 @@ import os import time -from pathlib import Path from typing import Any import httpx @@ -13,6 +12,7 @@ DEFAULT_TIMEOUT, RequestCore, RetryPolicy, + atomic_destination, ensure_same_origin, request_is_replayable, warn_if_insecure, @@ -101,7 +101,7 @@ def download( if not response.is_success: response.read() self._core.handle_response(response) - with Path(dest).open("wb") as fh: + with atomic_destination(dest) as part, part.open("wb") as fh: for chunk in response.iter_bytes(): fh.write(chunk) diff --git a/reportnet_api/easy.py b/reportnet_api/easy.py index a375670..b782f03 100644 --- a/reportnet_api/easy.py +++ b/reportnet_api/easy.py @@ -29,7 +29,7 @@ import httpx -from ._core import RetryPolicy, caller_stacklevel +from ._core import RetryPolicy, atomic_destination, caller_stacklevel from ._exceptions import ForbiddenError, ReportnetAPIError, _is_rejected_credential_500 from .client import ReportnetClient from .demo import demo_client as _demo_client @@ -907,8 +907,8 @@ def export_to_file( falls back to the older JSON export and writes the ZIP itself: one ``
.csv`` per table, comma-separated UTF-8, fields in schema order (then `country_code` when the records carry one), no record id column. That export is - read into memory whole. Parquet has no such - fallback; the `JobError` says so. + read into memory whole. `timeout` covers both jobs together. + Parquet has no such fallback; the `JobError` says so. """ _require_id(dataset_id, "dataset_id", _DATASET_HINT) _require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT) @@ -919,6 +919,7 @@ def export_to_file( if file_format == "csv" else client.datasets.export_zip_parquet_to_file ) + started = time.monotonic() try: try: export( @@ -939,8 +940,9 @@ def export_to_file( status=err.status, info=err.info, ) from err + left = max(0.0, timeout - (time.monotonic() - started)) # one budget for both jobs _export_csv_zip_from_json( - client, dataset_id, dataflow_id, Path(dest), table_schema_id, timeout, progress + client, dataset_id, dataflow_id, Path(dest), table_schema_id, left, progress ) except ForbiddenError as err: raise _hinted(err, _FORBIDDEN_HINT) from err @@ -976,24 +978,19 @@ def _export_csv_zip_from_json( field_order = _schema_field_types(client, dataset_id) except ReportnetAPIError: field_order = {} # columns then follow the records - part = dest.with_name(dest.name + ".part") - try: - with zipfile.ZipFile(part, "w", zipfile.ZIP_DEFLATED) as out: - for table in payload.get("tables") or []: - name = str(table.get("tableName") or "table") - rows = _records_to_rows(table.get("records") or []) - columns = list(field_order.get(name, {})) - for row in rows: - columns += [k for k in row if k not in columns] - buf = io.StringIO() - writer = csv.writer(buf, lineterminator="\n") - writer.writerow(columns) - for row in rows: - writer.writerow(["" if row.get(c) is None else row[c] for c in columns]) - out.writestr(f"{name}.csv", buf.getvalue().encode("utf-8")) - part.replace(dest) - finally: - part.unlink(missing_ok=True) + with atomic_destination(dest) as part, zipfile.ZipFile(part, "w", zipfile.ZIP_DEFLATED) as out: + for table in payload.get("tables") or []: + name = str(table.get("tableName") or "table") + rows = _records_to_rows(table.get("records") or []) + columns = list(field_order.get(name, {})) + for row in rows: + columns += [k for k in row if k not in columns] + buf = io.StringIO() + writer = csv.writer(buf, lineterminator="\n") + writer.writerow(columns) + for row in rows: + writer.writerow(["" if row.get(c) is None else row[c] for c in columns]) + out.writestr(f"{name}.csv", buf.getvalue().encode("utf-8")) def export_table( diff --git a/tests/test_export_fallback.py b/tests/test_export_fallback.py index 25f7bea..15f2290 100644 --- a/tests/test_export_fallback.py +++ b/tests/test_export_fallback.py @@ -9,7 +9,8 @@ import json import zipfile from pathlib import Path -from typing import Any +from types import SimpleNamespace +from typing import Any, NoReturn import httpx import pytest @@ -166,3 +167,48 @@ def test_timeout_is_not_retried_with_v3(tmp_path: Path) -> None: ): easy.export_to_file(client, 1, 2, tmp_path / "out.zip", timeout=0) assert not lake.called + + +def test_fallback_keeps_a_foreign_part_file(tmp_path: Path) -> None: + dest = tmp_path / "out.zip" + foreign = tmp_path / "out.zip.part" + foreign.write_bytes(b"not ours") + with respx.mock(base_url=BASE_URL) as mock: + _mock_refused_v4(mock, "/dataset/v4/etlExport/1", big_data=False) + mock.get("/dataset/v3/etlExport/1").mock( + return_value=httpx.Response(200, json={"pollingUrl": V3_POLL + "?datasetId=1"}) + ) + mock.get(V3_POLL).mock( + return_value=httpx.Response( + 200, json={"status": "FINISHED", "downloadUrl": V3_DOWNLOAD + "?datasetId=1"} + ) + ) + mock.get(V3_DOWNLOAD).mock( + return_value=httpx.Response(200, content=_zip_of("43.json", ETL_JSON)) + ) + mock.get("/dataschema/v1/datasetId/1").mock(return_value=httpx.Response(200, json=SCHEMA)) + with ReportnetClient(base_url=BASE_URL, token="t") as client: + easy.export_to_file(client, 1, 2, dest) + assert foreign.read_bytes() == b"not ours" + assert sorted(p.name for p in tmp_path.iterdir()) == ["out.zip", "out.zip.part"] + + +def test_fallback_gets_only_the_time_left(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + clock = iter([100.0, 340.0]) # export starts at 100 s; the v4 job fails at 340 s + monkeypatch.setattr("reportnet_api.easy.time", SimpleNamespace(monotonic=lambda: next(clock))) + monkeypatch.setattr(easy, "is_data_lake", lambda client, dataflow_id: False) + seen: dict[str, float] = {} + + def refused(*a: object, **k: object) -> NoReturn: + raise JobError("Job FAILED", status="FAILED", info=REFUSED_INFO) + + def fallback(*args: object) -> None: + timeout = args[5] + assert isinstance(timeout, float) + seen["timeout"] = timeout + + monkeypatch.setattr(easy, "_export_csv_zip_from_json", fallback) + with ReportnetClient(base_url=BASE_URL, token="t") as client: + monkeypatch.setattr(client.datasets, "export_zip_csv_to_file", refused) + easy.export_to_file(client, 1, 2, tmp_path / "out.zip", timeout=300.0) + assert seen["timeout"] == 60.0 diff --git a/tests/test_safe_downloads.py b/tests/test_safe_downloads.py new file mode 100644 index 0000000..d19a08d --- /dev/null +++ b/tests/test_safe_downloads.py @@ -0,0 +1,72 @@ +"""download() only replaces `dest` once the whole body has arrived.""" + +import asyncio +from collections.abc import AsyncIterator, Iterator +from pathlib import Path + +import httpx +import pytest +import respx + +from reportnet_api import AsyncReportnetClient, ReportnetClient + +BASE_URL = "https://api.reportnet.europa.eu" + + +class _BrokenStream(httpx.SyncByteStream): + def __iter__(self) -> Iterator[bytes]: + yield b"partial" + raise httpx.ReadError("connection dropped") + + +class _AsyncBrokenStream(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b"partial" + raise httpx.ReadError("connection dropped") + + +def _leftovers(folder: Path) -> list[str]: + return sorted(p.name for p in folder.iterdir() if p.name.endswith(".part")) + + +def test_failed_download_keeps_the_old_file(tmp_path: Path) -> None: + dest = tmp_path / "export.zip" + dest.write_bytes(b"previous good export") + with respx.mock(base_url=BASE_URL) as mock: + mock.get("/f").mock(return_value=httpx.Response(200, stream=_BrokenStream())) + with ( + ReportnetClient(base_url=BASE_URL, token="t") as client, + pytest.raises(httpx.ReadError), + ): + client.download("GET", "/f", dest=dest) + assert dest.read_bytes() == b"previous good export" + assert _leftovers(tmp_path) == [] + + +def test_async_failed_download_keeps_the_old_file(tmp_path: Path) -> None: + dest = tmp_path / "export.zip" + dest.write_bytes(b"previous good export") + + async def run() -> None: + with respx.mock(base_url=BASE_URL) as mock: + mock.get("/f").mock(return_value=httpx.Response(200, stream=_AsyncBrokenStream())) + async with AsyncReportnetClient(base_url=BASE_URL, token="t") as client: + await client.download("GET", "/f", dest=dest) + + with pytest.raises(httpx.ReadError): + asyncio.run(run()) + assert dest.read_bytes() == b"previous good export" + assert _leftovers(tmp_path) == [] + + +def test_successful_download_replaces_the_file(tmp_path: Path) -> None: + dest = tmp_path / "export.zip" + dest.write_bytes(b"old") + (tmp_path / "export.zip.part").write_bytes(b"someone else's file") + with respx.mock(base_url=BASE_URL) as mock: + mock.get("/f").mock(return_value=httpx.Response(200, content=b"new")) + with ReportnetClient(base_url=BASE_URL, token="t") as client: + client.download("GET", "/f", dest=dest) + assert dest.read_bytes() == b"new" + assert (tmp_path / "export.zip.part").read_bytes() == b"someone else's file" + assert _leftovers(tmp_path) == ["export.zip.part"] diff --git a/uv.lock b/uv.lock index 0a0e25a..e660733 100644 --- a/uv.lock +++ b/uv.lock @@ -1505,7 +1505,7 @@ wheels = [ [[package]] name = "reportnet-api" -version = "0.1.2" +version = "0.1.3" source = { editable = "." } dependencies = [ { name = "httpx" },