From cfdc766259a4a0530c0020144cf639d2d08b05a1 Mon Sep 17 00:00:00 2001 From: Nikolaos Paximadakis Date: Fri, 20 Nov 2026 11:39:36 +0100 Subject: [PATCH 1/2] read_tables: look up bigData when the export has no rows An empty ETL export was ambiguous: a data-lake dataflow answers 200 with 0 rows even when data exists, so every empty result warned "may be a data lake", also for tables that really are empty. Now an all-empty result looks up the dataflow's bigData flag (one extra call, only in that case): a lake gets a definite warning pointing to export_to_file(), a non-lake dataflow gets none, and a failed lookup keeps the old warning without failing the read. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++++++ docs/reference/easy.md | 10 ++++++---- reportnet_api/easy.py | 37 ++++++++++++++++++++++++++---------- scripts/docs_caveats.py | 3 ++- tests/test_easy.py | 42 +++++++++++++++++++++++++++++++++++++++-- 5 files changed, 83 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3796cc5..c64cb6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes are listed here. The project follows [Semantic Versioning](h while the major version is `0`, a **minor** release (0.x.0) may change the API, and a patch release (0.x.y) only fixes bugs. Breaking changes are called out below. +## Unreleased + +- `read_tables` (and `read_table`, `read_records`, `read_all_records`): when the export + has no rows at all, the dataflow's `bigData` flag is looked up. A data-lake dataflow + gets a definite warning pointing to `export_to_file()`; a non-lake dataflow gets no + warning, since its tables really are empty. If the lookup fails, the old "may be a + data lake" warning stays. + ## 0.1.3 (2026-11-19) - Downloads (`client.download()` and everything built on it, such as `export_to_file`) diff --git a/docs/reference/easy.md b/docs/reference/easy.md index 3a8a5b7..0f6a416 100644 --- a/docs/reference/easy.md +++ b/docs/reference/easy.md @@ -573,9 +573,11 @@ row count that differs from `totalRecords`, or tables of the schema missing from the answer. Data-lake caveat: for a dataflow with `bigData: true` this route answers `200` -with 0 rows even when data exists (live-tested), so a result -with no rows at all raises a `RuntimeWarning`; use `export_to_file()` (job export) -for such a dataset. +with 0 rows even when data exists (live-tested). When a result has no rows at +all, the dataflow's `bigData` flag is looked up (one extra call): a lake gets a +`RuntimeWarning` pointing to `export_to_file()` (job export), a non-lake dataflow +gets none (the tables are empty), and if the lookup fails the warning says the +dataset may be in a lake. Values arrive from the backend as text, empty cells as `''`. With `dtypes="schema"` columns are converted using the field types in the dataset @@ -590,7 +592,7 @@ a callable receives an event dict per page (`reportnet_api.jobs.progress_event`) **Live-tested notes:** - Uses the synchronous ETL export, whose access is credential-dependent. -- On a data-lake dataflow (`bigData: true`) the export answers 200 with 0 rows although rows exist, so an all-empty result warns; use `export_to_file()`. +- On a data-lake dataflow (`bigData: true`) the export answers 200 with 0 rows although rows exist; an all-empty result looks up `bigData` and warns only for a lake (or when the lookup fails); use `export_to_file()` there. ### validate_and_wait diff --git a/reportnet_api/easy.py b/reportnet_api/easy.py index b782f03..622639b 100644 --- a/reportnet_api/easy.py +++ b/reportnet_api/easy.py @@ -707,13 +707,28 @@ def _collect_tables( stacklevel=caller_stacklevel(), ) if not any(collected.values()): - warnings.warn( - "The ETL export answered with no rows. If the dataset is not empty, it may be " - "in a data-lake dataflow (`bigData: true`), where this route answers 200 with " - "0 rows (live-tested); read it with `easy.export_to_file()` instead", - RuntimeWarning, - stacklevel=caller_stacklevel(), - ) + # Only looked up here: a lake answers 200 with 0 rows (live-tested), so an + # empty result is the one case where `bigData` changes what it means. + try: + lake: bool | None = is_data_lake(client, dataflow_id) + except (ReportnetAPIError, httpx.HTTPError): + lake = None # the rows are already read: never fail over a warning + if lake: + warnings.warn( + f"The ETL export answered with no rows: dataflow {dataflow_id} is a " + "data-lake dataflow (`bigData: true`), where this route never returns " + "rows (live-tested); read it with `easy.export_to_file()` instead", + RuntimeWarning, + stacklevel=caller_stacklevel(), + ) + elif lake is None: + warnings.warn( + "The ETL export answered with no rows. If the dataset is not empty, it may be " + "in a data-lake dataflow (`bigData: true`), where this route answers 200 with " + "0 rows (live-tested); read it with `easy.export_to_file()` instead", + RuntimeWarning, + stacklevel=caller_stacklevel(), + ) if table_schema_id is None: expected = _schema_table_names(client, dataset_id) missing = sorted(expected - collected.keys()) if expected else [] @@ -752,9 +767,11 @@ def read_tables( from the answer. Data-lake caveat: for a dataflow with `bigData: true` this route answers `200` - with 0 rows even when data exists (live-tested), so a result - with no rows at all raises a `RuntimeWarning`; use `export_to_file()` (job export) - for such a dataset. + with 0 rows even when data exists (live-tested). When a result has no rows at + all, the dataflow's `bigData` flag is looked up (one extra call): a lake gets a + `RuntimeWarning` pointing to `export_to_file()` (job export), a non-lake dataflow + gets none (the tables are empty), and if the lookup fails the warning says the + dataset may be in a lake. Values arrive from the backend as text, empty cells as `''`. With `dtypes="schema"` columns are converted using the field types in the dataset diff --git a/scripts/docs_caveats.py b/scripts/docs_caveats.py index c5d63b3..c08d718 100644 --- a/scripts/docs_caveats.py +++ b/scripts/docs_caveats.py @@ -368,7 +368,8 @@ ), ( "On a data-lake dataflow (`bigData: true`) the export answers 200 with 0 rows " - "although rows exist, so an all-empty result warns; use `export_to_file()`.", + "although rows exist; an all-empty result looks up `bigData` and warns only " + "for a lake (or when the lookup fails); use `export_to_file()` there.", "", ), ], diff --git a/tests/test_easy.py b/tests/test_easy.py index a7b3118..be5c992 100644 --- a/tests/test_easy.py +++ b/tests/test_easy.py @@ -169,14 +169,50 @@ def test_read_tables_warns_on_table_missing_from_export(monkeypatch: pytest.Monk @respx.mock -def test_read_tables_warns_when_export_has_no_rows(no_schema_check: None) -> None: +def test_read_tables_warns_when_lake_export_has_no_rows(no_schema_check: None) -> None: # Live (sandbox, 2026-11-06): a data-lake dataset answers 200 with empty tables. respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond( json={"tables": [{"tableName": "T", "records": []}]} ) - with pytest.warns(RuntimeWarning, match="data-lake"): + flow = respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True}) + with pytest.warns(RuntimeWarning, match=r"is a data-lake dataflow.*export_to_file"): out = easy.read_tables(easy.connect("k"), 7, 1) assert out["T"].empty + assert flow.call_count == 1 + + +@respx.mock +def test_read_tables_no_warning_when_relational_export_is_empty( + no_schema_check: None, recwarn: pytest.WarningsRecorder +) -> None: + # Not a lake: zero rows means the tables really are empty. + respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond( + json={"tables": [{"tableName": "T", "records": []}]} + ) + respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": False}) + out = easy.read_tables(easy.connect("k"), 7, 1) + assert out["T"].empty + assert not [w for w in recwarn if issubclass(w.category, RuntimeWarning)] + + +@respx.mock +def test_read_tables_vague_warning_when_lake_lookup_fails(no_schema_check: None) -> None: + respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond( + json={"tables": [{"tableName": "T", "records": []}]} + ) + respx.get(f"{BASE}/dataflow/v1/1").respond(403) + with pytest.warns(RuntimeWarning, match=r"may be\s+in a data-lake dataflow"): + easy.read_tables(easy.connect("k"), 7, 1) + + +@respx.mock +def test_read_tables_skips_lake_lookup_when_rows_arrive(no_schema_check: None) -> None: + respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond( + json={"tables": [{"tableName": "T", "records": [_rec("r1", "x")]}]} + ) + flow = respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True}) + easy.read_tables(easy.connect("k"), 7, 1) + assert flow.call_count == 0 def _vrow(code: str, rule: str) -> dict[str, Any]: @@ -464,6 +500,7 @@ def test_warnings_point_at_the_callers_line(no_schema_check: None) -> None: import warnings respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(json={"tables": []}) + respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True}) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") easy.read_all_records(easy.connect("k"), 7, 1) @@ -489,6 +526,7 @@ def test_page_size_accepts_numpy_ints_and_rejects_bools(no_schema_check: None) - easy.read_records(client, 7, 1, page_size=bad) with respx.mock: respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(json={"tables": []}) + respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True}) with pytest.warns(RuntimeWarning): easy.read_all_records(client, 7, 1, page_size=np.int64(500)) From 4ae15a60456e30549e3a54b348b8c3c320c21058 Mon Sep 17 00:00:00 2001 From: Nikolaos Paximadakis Date: Fri, 20 Nov 2026 11:53:06 +0100 Subject: [PATCH 2/2] read_tables lake lookup: review fixes Only an explicit boolean bigData decides; an empty 200 (the route swallows its own errors), a body without bigData or a non-JSON body keeps the vague warning instead of silencing it. The lookup passes the caller's provider_id (reporters need it) and never raises (ValueError from a non-JSON body is caught too). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +++--- docs/reference/easy.md | 6 +++--- reportnet_api/easy.py | 28 +++++++++++++++++++++------- tests/test_easy.py | 28 ++++++++++++++++++++++++++-- 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c64cb6c..d22e00c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,9 @@ release (0.x.y) only fixes bugs. Breaking changes are called out below. - `read_tables` (and `read_table`, `read_records`, `read_all_records`): when the export has no rows at all, the dataflow's `bigData` flag is looked up. A data-lake dataflow - gets a definite warning pointing to `export_to_file()`; a non-lake dataflow gets no - warning, since its tables really are empty. If the lookup fails, the old "may be a - data lake" warning stays. + gets a definite warning pointing to `export_to_file()`; a dataflow that answers + `bigData: false` gets no warning, since its tables really are empty. If the lookup + fails or its answer does not say, the old "may be a data lake" warning stays. ## 0.1.3 (2026-11-19) diff --git a/docs/reference/easy.md b/docs/reference/easy.md index 0f6a416..8ff9515 100644 --- a/docs/reference/easy.md +++ b/docs/reference/easy.md @@ -575,9 +575,9 @@ from the answer. Data-lake caveat: for a dataflow with `bigData: true` this route answers `200` with 0 rows even when data exists (live-tested). When a result has no rows at all, the dataflow's `bigData` flag is looked up (one extra call): a lake gets a -`RuntimeWarning` pointing to `export_to_file()` (job export), a non-lake dataflow -gets none (the tables are empty), and if the lookup fails the warning says the -dataset may be in a lake. +`RuntimeWarning` pointing to `export_to_file()` (job export), a dataflow that +answers `bigData: false` gets none (the tables are empty), and if the lookup +fails or does not say, the warning says the dataset may be in a lake. Values arrive from the backend as text, empty cells as `''`. With `dtypes="schema"` columns are converted using the field types in the dataset diff --git a/reportnet_api/easy.py b/reportnet_api/easy.py index 622639b..cafb35a 100644 --- a/reportnet_api/easy.py +++ b/reportnet_api/easy.py @@ -273,6 +273,23 @@ def is_data_lake(client: ReportnetClient, dataflow_id: int) -> bool: return bool((flow or {}).get("bigData")) +def _big_data_flag( + client: ReportnetClient, dataflow_id: int, provider_id: int | None +) -> bool | None: + """The dataflow's `bigData` flag, or None when the answer does not say. + + Advisory only (it picks a warning), so it never raises: a refused or failed + call, an empty 200 (the route swallows its own errors) or a body without a + boolean `bigData` all give None. + """ + try: + flow = client.dataflow.dataflow_datasets(dataflow_id, provider_id=provider_id) + except (ReportnetAPIError, httpx.HTTPError, ValueError): + return None + flag = flow.get("bigData") if isinstance(flow, dict) else None + return flag if isinstance(flag, bool) else None + + _ERROR_COLUMNS = [ "short_code", "level", @@ -709,10 +726,7 @@ def _collect_tables( if not any(collected.values()): # Only looked up here: a lake answers 200 with 0 rows (live-tested), so an # empty result is the one case where `bigData` changes what it means. - try: - lake: bool | None = is_data_lake(client, dataflow_id) - except (ReportnetAPIError, httpx.HTTPError): - lake = None # the rows are already read: never fail over a warning + lake = _big_data_flag(client, dataflow_id, provider_id) if lake: warnings.warn( f"The ETL export answered with no rows: dataflow {dataflow_id} is a " @@ -769,9 +783,9 @@ def read_tables( Data-lake caveat: for a dataflow with `bigData: true` this route answers `200` with 0 rows even when data exists (live-tested). When a result has no rows at all, the dataflow's `bigData` flag is looked up (one extra call): a lake gets a - `RuntimeWarning` pointing to `export_to_file()` (job export), a non-lake dataflow - gets none (the tables are empty), and if the lookup fails the warning says the - dataset may be in a lake. + `RuntimeWarning` pointing to `export_to_file()` (job export), a dataflow that + answers `bigData: false` gets none (the tables are empty), and if the lookup + fails or does not say, the warning says the dataset may be in a lake. Values arrive from the backend as text, empty cells as `''`. With `dtypes="schema"` columns are converted using the field types in the dataset diff --git a/tests/test_easy.py b/tests/test_easy.py index be5c992..b9499e6 100644 --- a/tests/test_easy.py +++ b/tests/test_easy.py @@ -195,16 +195,40 @@ def test_read_tables_no_warning_when_relational_export_is_empty( assert not [w for w in recwarn if issubclass(w.category, RuntimeWarning)] +@pytest.mark.parametrize( + "lookup", + [ + httpx.Response(403), + httpx.Response(200, content=b""), # the route swallows its own errors + httpx.Response(200, json={}), + httpx.Response(200, json={"bigData": None}), + httpx.Response(200, text="gateway"), + httpx.Response(200, json=[1, 2]), + ], +) @respx.mock -def test_read_tables_vague_warning_when_lake_lookup_fails(no_schema_check: None) -> None: +def test_read_tables_vague_warning_when_lake_lookup_does_not_say( + no_schema_check: None, lookup: httpx.Response +) -> None: respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond( json={"tables": [{"tableName": "T", "records": []}]} ) - respx.get(f"{BASE}/dataflow/v1/1").respond(403) + respx.get(f"{BASE}/dataflow/v1/1").mock(return_value=lookup) with pytest.warns(RuntimeWarning, match=r"may be\s+in a data-lake dataflow"): easy.read_tables(easy.connect("k"), 7, 1) +@respx.mock +def test_read_tables_lake_lookup_passes_provider_id(no_schema_check: None) -> None: + respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond( + json={"tables": [{"tableName": "T", "records": []}]} + ) + flow = respx.get(f"{BASE}/dataflow/v1/1").respond(json={"bigData": True}) + with pytest.warns(RuntimeWarning, match="is a data-lake dataflow"): + easy.read_tables(easy.connect("k"), 7, 1, provider_id=60) + assert flow.calls[-1].request.url.params["providerId"] == "60" + + @respx.mock def test_read_tables_skips_lake_lookup_when_rows_arrive(no_schema_check: None) -> None: respx.get(url__regex=rf"{BASE}/dataset/v1/.*").respond(