Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,14 @@ All notable changes are listed here. The project follows [Semantic Versioning](h
while the major version is `0`, a **minor** release (0.x.0) may change the API, and a patch
release (0.x.y) only fixes bugs. Breaking changes are called out below.

## Unreleased

- `read_tables` (and `read_table`, `read_records`, `read_all_records`): when the export
has no rows at all, the dataflow's `bigData` flag is looked up. A data-lake dataflow
gets a definite warning pointing to `export_to_file()`; a dataflow that answers
`bigData: false` gets no warning, since its tables really are empty. If the lookup
fails or its answer does not say, the old "may be a data lake" warning stays.

## 0.1.3 (2026-11-19)

- Downloads (`client.download()` and everything built on it, such as `export_to_file`)
Expand Down
10 changes: 6 additions & 4 deletions docs/reference/easy.md
Original file line number Diff line number Diff line change
Expand Up @@ -573,9 +573,11 @@ row count that differs from `totalRecords`, or tables of the schema missing
from the answer.

Data-lake caveat: for a dataflow with `bigData: true` this route answers `200`
with 0 rows even when data exists (live-tested), so a result
with no rows at all raises a `RuntimeWarning`; use `export_to_file()` (job export)
for such a dataset.
with 0 rows even when data exists (live-tested). When a result has no rows at
all, the dataflow's `bigData` flag is looked up (one extra call): a lake gets a
`RuntimeWarning` pointing to `export_to_file()` (job export), a dataflow that
answers `bigData: false` gets none (the tables are empty), and if the lookup
fails or does not say, the warning says the dataset may be in a lake.

Values arrive from the backend as text, empty cells as `''`. With
`dtypes="schema"` columns are converted using the field types in the dataset
Expand All @@ -590,7 +592,7 @@ a callable receives an event dict per page (`reportnet_api.jobs.progress_event`)
**Live-tested notes:**

- Uses the synchronous ETL export, whose access is credential-dependent.
- On a data-lake dataflow (`bigData: true`) the export answers 200 with 0 rows although rows exist, so an all-empty result warns; use `export_to_file()`.
- On a data-lake dataflow (`bigData: true`) the export answers 200 with 0 rows although rows exist; an all-empty result looks up `bigData` and warns only for a lake (or when the lookup fails); use `export_to_file()` there.

### validate_and_wait

Expand Down
51 changes: 41 additions & 10 deletions reportnet_api/easy.py
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,23 @@ def is_data_lake(client: ReportnetClient, dataflow_id: int) -> bool:
return bool((flow or {}).get("bigData"))


def _big_data_flag(
client: ReportnetClient, dataflow_id: int, provider_id: int | None
) -> bool | None:
"""The dataflow's `bigData` flag, or None when the answer does not say.

Advisory only (it picks a warning), so it never raises: a refused or failed
call, an empty 200 (the route swallows its own errors) or a body without a
boolean `bigData` all give None.
"""
try:
flow = client.dataflow.dataflow_datasets(dataflow_id, provider_id=provider_id)
except (ReportnetAPIError, httpx.HTTPError, ValueError):
return None
flag = flow.get("bigData") if isinstance(flow, dict) else None
return flag if isinstance(flag, bool) else None


_ERROR_COLUMNS = [
"short_code",
"level",
Expand Down Expand Up @@ -707,13 +724,25 @@ def _collect_tables(
stacklevel=caller_stacklevel(),
)
if not any(collected.values()):
warnings.warn(
"The ETL export answered with no rows. If the dataset is not empty, it may be "
"in a data-lake dataflow (`bigData: true`), where this route answers 200 with "
"0 rows (live-tested); read it with `easy.export_to_file()` instead",
RuntimeWarning,
stacklevel=caller_stacklevel(),
)
# Only looked up here: a lake answers 200 with 0 rows (live-tested), so an
# empty result is the one case where `bigData` changes what it means.
lake = _big_data_flag(client, dataflow_id, provider_id)
if lake:
warnings.warn(
f"The ETL export answered with no rows: dataflow {dataflow_id} is a "
"data-lake dataflow (`bigData: true`), where this route never returns "
"rows (live-tested); read it with `easy.export_to_file()` instead",
RuntimeWarning,
stacklevel=caller_stacklevel(),
)
elif lake is None:
warnings.warn(
"The ETL export answered with no rows. If the dataset is not empty, it may be "
"in a data-lake dataflow (`bigData: true`), where this route answers 200 with "
"0 rows (live-tested); read it with `easy.export_to_file()` instead",
RuntimeWarning,
stacklevel=caller_stacklevel(),
)
if table_schema_id is None:
expected = _schema_table_names(client, dataset_id)
missing = sorted(expected - collected.keys()) if expected else []
Expand Down Expand Up @@ -752,9 +781,11 @@ def read_tables(
from the answer.

Data-lake caveat: for a dataflow with `bigData: true` this route answers `200`
with 0 rows even when data exists (live-tested), so a result
with no rows at all raises a `RuntimeWarning`; use `export_to_file()` (job export)
for such a dataset.
with 0 rows even when data exists (live-tested). When a result has no rows at
all, the dataflow's `bigData` flag is looked up (one extra call): a lake gets a
`RuntimeWarning` pointing to `export_to_file()` (job export), a dataflow that
answers `bigData: false` gets none (the tables are empty), and if the lookup
fails or does not say, the warning says the dataset may be in a lake.

Values arrive from the backend as text, empty cells as `''`. With
`dtypes="schema"` columns are converted using the field types in the dataset
Expand Down
3 changes: 2 additions & 1 deletion scripts/docs_caveats.py
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,8 @@
),
(
"On a data-lake dataflow (`bigData: true`) the export answers 200 with 0 rows "
"although rows exist, so an all-empty result warns; use `export_to_file()`.",
"although rows exist; an all-empty result looks up `bigData` and warns only "
"for a lake (or when the lookup fails); use `export_to_file()` there.",
"",
),
],
Expand Down
66 changes: 64 additions & 2 deletions tests/test_easy.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,14 +169,74 @@ def test_read_tables_warns_on_table_missing_from_export(monkeypatch: pytest.Monk


@respx.mock
def test_read_tables_warns_when_export_has_no_rows(no_schema_check: None) -> None:
def test_read_tables_warns_when_lake_export_has_no_rows(no_schema_check: None) -> None:
# Live (sandbox, 2026-11-06): a data-lake dataset answers 200 with empty tables.
respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(
json={"tables": [{"tableName": "T", "records": []}]}
)
with pytest.warns(RuntimeWarning, match="data-lake"):
flow = respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True})
with pytest.warns(RuntimeWarning, match=r"is a data-lake dataflow.*export_to_file"):
out = easy.read_tables(easy.connect("k"), 7, 1)
assert out["T"].empty
assert flow.call_count == 1


@respx.mock
def test_read_tables_no_warning_when_relational_export_is_empty(
no_schema_check: None, recwarn: pytest.WarningsRecorder
) -> None:
# Not a lake: zero rows means the tables really are empty.
respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(
json={"tables": [{"tableName": "T", "records": []}]}
)
respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": False})
out = easy.read_tables(easy.connect("k"), 7, 1)
assert out["T"].empty
assert not [w for w in recwarn if issubclass(w.category, RuntimeWarning)]


@pytest.mark.parametrize(
"lookup",
[
httpx.Response(403),
httpx.Response(200, content=b""), # the route swallows its own errors
httpx.Response(200, json={}),
httpx.Response(200, json={"bigData": None}),
httpx.Response(200, text="<html>gateway</html>"),
httpx.Response(200, json=[1, 2]),
],
)
@respx.mock
def test_read_tables_vague_warning_when_lake_lookup_does_not_say(
no_schema_check: None, lookup: httpx.Response
) -> None:
respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(
json={"tables": [{"tableName": "T", "records": []}]}
)
respx.get(f"{BASE}/dataflow/v1/1").mock(return_value=lookup)
with pytest.warns(RuntimeWarning, match=r"may be\s+in a data-lake dataflow"):
easy.read_tables(easy.connect("k"), 7, 1)


@respx.mock
def test_read_tables_lake_lookup_passes_provider_id(no_schema_check: None) -> None:
respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(
json={"tables": [{"tableName": "T", "records": []}]}
)
flow = respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True})
with pytest.warns(RuntimeWarning, match="is a data-lake dataflow"):
easy.read_tables(easy.connect("k"), 7, 1, provider_id=60)
assert flow.calls[-1].request.url.params["providerId"] == "60"


@respx.mock
def test_read_tables_skips_lake_lookup_when_rows_arrive(no_schema_check: None) -> None:
respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(
json={"tables": [{"tableName": "T", "records": [_rec("r1", "x")]}]}
)
flow = respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True})
easy.read_tables(easy.connect("k"), 7, 1)
assert flow.call_count == 0


def _vrow(code: str, rule: str) -> dict[str, Any]:
Expand Down Expand Up @@ -464,6 +524,7 @@ def test_warnings_point_at_the_callers_line(no_schema_check: None) -> None:
import warnings

respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(json={"tables": []})
respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True})
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
easy.read_all_records(easy.connect("k"), 7, 1)
Expand All @@ -489,6 +550,7 @@ def test_page_size_accepts_numpy_ints_and_rejects_bools(no_schema_check: None) -
easy.read_records(client, 7, 1, page_size=bad)
with respx.mock:
respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(json={"tables": []})
respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True})
with pytest.warns(RuntimeWarning):
easy.read_all_records(client, 7, 1, page_size=np.int64(500))

Expand Down