Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@ All notable changes are listed here. The project follows [Semantic Versioning](h
while the major version is `0`, a **minor** release (0.x.0) may change the API, and a patch
release (0.x.y) only fixes bugs. Breaking changes are called out below.

## 0.1.3 (unreleased)

- Downloads (`client.download()` and everything built on it, such as `export_to_file`)
write to a temporary file next to `dest` and replace `dest` only when complete: a
failed download no longer leaves a partial file in place of the previous one.
- `export_to_file`'s fallback on non-data-lake dataflows now stays within the one
`timeout` and no longer overwrites a file named `<dest>.part`.
- `TooManyRequestsError.retry_after` returns `None` (not `inf`/`nan`) for such
`Retry-After` values.
- Development: strict mypy over the whole project, a wider ruff rule set, warnings as
errors in tests (which also no longer reach the network), pre-commit hooks.

## 0.1.2 (2026-11-19)

- `easy.export_to_file` (and `Dataset.export`) work again on dataflows that are not a
Expand Down
4 changes: 2 additions & 2 deletions docs/reference/easy.md
Original file line number Diff line number Diff line change
Expand Up @@ -235,8 +235,8 @@ that is not a data lake (live on the sandbox, 2026-11-18). For CSV this then
falls back to the older JSON export and writes the ZIP itself: one
``<table>.csv`` per table, comma-separated UTF-8, fields in schema order (then
`country_code` when the records carry one), no record id column. That export is
read into memory whole. Parquet has no such
fallback; the `JobError` says so.
read into memory whole. `timeout` covers both jobs together.
Parquet has no such fallback; the `JobError` says so.

**Live-tested notes:**

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "reportnet-api"
version = "0.1.2"
version = "0.1.3"
description = "Python library for the EEA Reportnet 3 REST API"
readme = "README.md"
license = "MIT"
Expand Down
17 changes: 17 additions & 0 deletions reportnet_api/_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import math
import os
import sys
import uuid
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass
Expand Down Expand Up @@ -214,6 +215,22 @@ def handle_response(self, response: httpx.Response) -> httpx.Response:
return response


@contextmanager
def atomic_destination(dest: str | os.PathLike[str]) -> Iterator[Path]:
"""Yield a unique temporary path next to `dest`; move it onto `dest` only on success.

A failed or interrupted write leaves an existing `dest` untouched and no
leftover file. The random suffix keeps concurrent writers apart.
"""
final = Path(dest)
part = final.with_name(f"{final.name}.{uuid.uuid4().hex[:8]}.part")
try:
yield part
part.replace(final)
finally:
part.unlink(missing_ok=True)


@contextmanager
def stream_upload_file(
field_name: str, file: str | os.PathLike[str]
Expand Down
4 changes: 2 additions & 2 deletions reportnet_api/async_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import asyncio
import os
from pathlib import Path
from typing import Any

import httpx
Expand All @@ -13,6 +12,7 @@
DEFAULT_TIMEOUT,
RequestCore,
RetryPolicy,
atomic_destination,
ensure_same_origin,
request_is_replayable,
warn_if_insecure,
Expand Down Expand Up @@ -100,7 +100,7 @@ async def download(
if not response.is_success:
await response.aread()
self._core.handle_response(response)
with Path(dest).open("wb") as fh:
with atomic_destination(dest) as part, part.open("wb") as fh:
async for chunk in response.aiter_bytes():
fh.write(chunk)

Expand Down
4 changes: 2 additions & 2 deletions reportnet_api/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import os
import time
from pathlib import Path
from typing import Any

import httpx
Expand All @@ -13,6 +12,7 @@
DEFAULT_TIMEOUT,
RequestCore,
RetryPolicy,
atomic_destination,
ensure_same_origin,
request_is_replayable,
warn_if_insecure,
Expand Down Expand Up @@ -101,7 +101,7 @@ def download(
if not response.is_success:
response.read()
self._core.handle_response(response)
with Path(dest).open("wb") as fh:
with atomic_destination(dest) as part, part.open("wb") as fh:
for chunk in response.iter_bytes():
fh.write(chunk)

Expand Down
41 changes: 19 additions & 22 deletions reportnet_api/easy.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

import httpx

from ._core import RetryPolicy, caller_stacklevel
from ._core import RetryPolicy, atomic_destination, caller_stacklevel
from ._exceptions import ForbiddenError, ReportnetAPIError, _is_rejected_credential_500
from .client import ReportnetClient
from .demo import demo_client as _demo_client
Expand Down Expand Up @@ -907,8 +907,8 @@ def export_to_file(
falls back to the older JSON export and writes the ZIP itself: one
``<table>.csv`` per table, comma-separated UTF-8, fields in schema order (then
`country_code` when the records carry one), no record id column. That export is
read into memory whole. Parquet has no such
fallback; the `JobError` says so.
read into memory whole. `timeout` covers both jobs together.
Parquet has no such fallback; the `JobError` says so.
"""
_require_id(dataset_id, "dataset_id", _DATASET_HINT)
_require_id(dataflow_id, "dataflow_id", _DATAFLOW_HINT)
Expand All @@ -919,6 +919,7 @@ def export_to_file(
if file_format == "csv"
else client.datasets.export_zip_parquet_to_file
)
started = time.monotonic()
try:
try:
export(
Expand All @@ -939,8 +940,9 @@ def export_to_file(
status=err.status,
info=err.info,
) from err
left = max(0.0, timeout - (time.monotonic() - started)) # one budget for both jobs
_export_csv_zip_from_json(
client, dataset_id, dataflow_id, Path(dest), table_schema_id, timeout, progress
client, dataset_id, dataflow_id, Path(dest), table_schema_id, left, progress
)
except ForbiddenError as err:
raise _hinted(err, _FORBIDDEN_HINT) from err
Expand Down Expand Up @@ -976,24 +978,19 @@ def _export_csv_zip_from_json(
field_order = _schema_field_types(client, dataset_id)
except ReportnetAPIError:
field_order = {} # columns then follow the records
part = dest.with_name(dest.name + ".part")
try:
with zipfile.ZipFile(part, "w", zipfile.ZIP_DEFLATED) as out:
for table in payload.get("tables") or []:
name = str(table.get("tableName") or "table")
rows = _records_to_rows(table.get("records") or [])
columns = list(field_order.get(name, {}))
for row in rows:
columns += [k for k in row if k not in columns]
buf = io.StringIO()
writer = csv.writer(buf, lineterminator="\n")
writer.writerow(columns)
for row in rows:
writer.writerow(["" if row.get(c) is None else row[c] for c in columns])
out.writestr(f"{name}.csv", buf.getvalue().encode("utf-8"))
part.replace(dest)
finally:
part.unlink(missing_ok=True)
with atomic_destination(dest) as part, zipfile.ZipFile(part, "w", zipfile.ZIP_DEFLATED) as out:
for table in payload.get("tables") or []:
name = str(table.get("tableName") or "table")
rows = _records_to_rows(table.get("records") or [])
columns = list(field_order.get(name, {}))
for row in rows:
columns += [k for k in row if k not in columns]
buf = io.StringIO()
writer = csv.writer(buf, lineterminator="\n")
writer.writerow(columns)
for row in rows:
writer.writerow(["" if row.get(c) is None else row[c] for c in columns])
out.writestr(f"{name}.csv", buf.getvalue().encode("utf-8"))


def export_table(
Expand Down
48 changes: 47 additions & 1 deletion tests/test_export_fallback.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
import json
import zipfile
from pathlib import Path
from typing import Any
from types import SimpleNamespace
from typing import Any, NoReturn

import httpx
import pytest
Expand Down Expand Up @@ -166,3 +167,48 @@ def test_timeout_is_not_retried_with_v3(tmp_path: Path) -> None:
):
easy.export_to_file(client, 1, 2, tmp_path / "out.zip", timeout=0)
assert not lake.called


def test_fallback_keeps_a_foreign_part_file(tmp_path: Path) -> None:
dest = tmp_path / "out.zip"
foreign = tmp_path / "out.zip.part"
foreign.write_bytes(b"not ours")
with respx.mock(base_url=BASE_URL) as mock:
_mock_refused_v4(mock, "/dataset/v4/etlExport/1", big_data=False)
mock.get("/dataset/v3/etlExport/1").mock(
return_value=httpx.Response(200, json={"pollingUrl": V3_POLL + "?datasetId=1"})
)
mock.get(V3_POLL).mock(
return_value=httpx.Response(
200, json={"status": "FINISHED", "downloadUrl": V3_DOWNLOAD + "?datasetId=1"}
)
)
mock.get(V3_DOWNLOAD).mock(
return_value=httpx.Response(200, content=_zip_of("43.json", ETL_JSON))
)
mock.get("/dataschema/v1/datasetId/1").mock(return_value=httpx.Response(200, json=SCHEMA))
with ReportnetClient(base_url=BASE_URL, token="t") as client:
easy.export_to_file(client, 1, 2, dest)
assert foreign.read_bytes() == b"not ours"
assert sorted(p.name for p in tmp_path.iterdir()) == ["out.zip", "out.zip.part"]


def test_fallback_gets_only_the_time_left(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
clock = iter([100.0, 340.0]) # export starts at 100 s; the v4 job fails at 340 s
monkeypatch.setattr("reportnet_api.easy.time", SimpleNamespace(monotonic=lambda: next(clock)))
monkeypatch.setattr(easy, "is_data_lake", lambda client, dataflow_id: False)
seen: dict[str, float] = {}

def refused(*a: object, **k: object) -> NoReturn:
raise JobError("Job FAILED", status="FAILED", info=REFUSED_INFO)

def fallback(*args: object) -> None:
timeout = args[5]
assert isinstance(timeout, float)
seen["timeout"] = timeout

monkeypatch.setattr(easy, "_export_csv_zip_from_json", fallback)
with ReportnetClient(base_url=BASE_URL, token="t") as client:
monkeypatch.setattr(client.datasets, "export_zip_csv_to_file", refused)
easy.export_to_file(client, 1, 2, tmp_path / "out.zip", timeout=300.0)
assert seen["timeout"] == 60.0
72 changes: 72 additions & 0 deletions tests/test_safe_downloads.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""download() only replaces `dest` once the whole body has arrived."""

import asyncio
from collections.abc import AsyncIterator, Iterator
from pathlib import Path

import httpx
import pytest
import respx

from reportnet_api import AsyncReportnetClient, ReportnetClient

BASE_URL = "https://api.reportnet.europa.eu"


class _BrokenStream(httpx.SyncByteStream):
def __iter__(self) -> Iterator[bytes]:
yield b"partial"
raise httpx.ReadError("connection dropped")


class _AsyncBrokenStream(httpx.AsyncByteStream):
async def __aiter__(self) -> AsyncIterator[bytes]:
yield b"partial"
raise httpx.ReadError("connection dropped")


def _leftovers(folder: Path) -> list[str]:
return sorted(p.name for p in folder.iterdir() if p.name.endswith(".part"))


def test_failed_download_keeps_the_old_file(tmp_path: Path) -> None:
dest = tmp_path / "export.zip"
dest.write_bytes(b"previous good export")
with respx.mock(base_url=BASE_URL) as mock:
mock.get("/f").mock(return_value=httpx.Response(200, stream=_BrokenStream()))
with (
ReportnetClient(base_url=BASE_URL, token="t") as client,
pytest.raises(httpx.ReadError),
):
client.download("GET", "/f", dest=dest)
assert dest.read_bytes() == b"previous good export"
assert _leftovers(tmp_path) == []


def test_async_failed_download_keeps_the_old_file(tmp_path: Path) -> None:
dest = tmp_path / "export.zip"
dest.write_bytes(b"previous good export")

async def run() -> None:
with respx.mock(base_url=BASE_URL) as mock:
mock.get("/f").mock(return_value=httpx.Response(200, stream=_AsyncBrokenStream()))
async with AsyncReportnetClient(base_url=BASE_URL, token="t") as client:
await client.download("GET", "/f", dest=dest)

with pytest.raises(httpx.ReadError):
asyncio.run(run())
assert dest.read_bytes() == b"previous good export"
assert _leftovers(tmp_path) == []


def test_successful_download_replaces_the_file(tmp_path: Path) -> None:
dest = tmp_path / "export.zip"
dest.write_bytes(b"old")
(tmp_path / "export.zip.part").write_bytes(b"someone else's file")
with respx.mock(base_url=BASE_URL) as mock:
mock.get("/f").mock(return_value=httpx.Response(200, content=b"new"))
with ReportnetClient(base_url=BASE_URL, token="t") as client:
client.download("GET", "/f", dest=dest)
assert dest.read_bytes() == b"new"
assert (tmp_path / "export.zip.part").read_bytes() == b"someone else's file"
assert _leftovers(tmp_path) == ["export.zip.part"]
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.