From ac32091110c68c60c564fdda35e22314d6eb394e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 12:49:06 +0000 Subject: [PATCH 1/7] Fix bulk() lookups given a set of resources bulk() accepts a set of resources in its validation, but the set was handed to the JSON encoder as-is, which cannot serialise it, so every call with a set failed with "Object of type set is not JSON serializable" wrapped in an IPDataException. The resources are now sent as a list. Co-Authored-By: Claude Fable 5.1 --- src/ipdata/ipdata.py | 6 +++--- src/ipdata/test_client.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/ipdata/ipdata.py b/src/ipdata/ipdata.py index f21ddfe..cec8575 100644 --- a/src/ipdata/ipdata.py +++ b/src/ipdata/ipdata.py @@ -349,7 +349,7 @@ def bulk(self, resources, fields=[]): """ Lookup up to 100 resources in one request. Makes a POST request wth the resources as a JSON array and the specified fields. - :param resources: A list of resources. This can be a list of IP addresses or ASNs or a mix of both which is not recommended unless you handle it during parsing. Mixing resources might lead to weird behavior when writing results to CSV. + :param resources: A list or set of resources. This can be a list of IP addresses or ASNs or a mix of both which is not recommended unless you handle it during parsing. Mixing resources might lead to weird behavior when writing results to CSV. :param fields: A collection of API fields to be returned. :returns: A DotDict object with the data contained under the 'responses' key. @@ -368,9 +368,9 @@ def bulk(self, resources, fields=[]): self._validate_fields(fields) query_params = {"fields": ",".join(fields)} - # Make the requests + # Make the requests (as a list: a set passes the validation above but is not JSON serialisable) try: - response = self._request("POST", "/bulk", params=query_params, json=resources) + response = self._request("POST", "/bulk", params=query_params, json=list(resources)) except Exception as e: raise IPDataException(f"Error when looking up {resources} - {e}") diff --git a/src/ipdata/test_client.py b/src/ipdata/test_client.py index fb8d93e..0b44fb5 100644 --- a/src/ipdata/test_client.py +++ b/src/ipdata/test_client.py @@ -166,6 +166,18 @@ def test_lookup_uses_the_default_timeout(): assert recorder.requests[0].extensions["timeout"] == {"connect": 60, "read": 60, "write": 60, "pool": 60} +def test_bulk_accepts_a_set_of_resources(): + responses = [dict(LOOKUP_RESPONSE), dict(LOOKUP_RESPONSE, ip="1.1.1.1")] + client, recorder = make_client(httpx.Response(200, json=responses)) + + data = client.bulk({"8.8.8.8", "1.1.1.1"}, fields=["ip"]) + + # a set passes the validation but is not JSON serialisable, so it is sent as a list + assert sorted(json.loads(recorder.requests[0].content)) == ["1.1.1.1", "8.8.8.8"] + assert data.status == 200 + assert [r.ip for r in data.responses] == ["8.8.8.8", "1.1.1.1"] + + def test_bulk_non_200_returns_payload_with_status(): client, _ = make_client(httpx.Response(403, json={"message": "Forbidden"})) From 7ce7d149ebe9193c632f836fa0f69e3d08ae5ba9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 12:49:41 +0000 Subject: [PATCH 2/7] Keep non-dict list items as they are in DotDict attribute access DotDict.__getattr__ wrapped every element of a list value in a DotDict, so reading an attribute whose value is a list of scalars, for example [1, 2, 3], raised TypeError. Only dict elements are wrapped now and the other elements are returned unchanged. Lists of objects such as threat.blocklists behave as before. Co-Authored-By: Claude Fable 5.1 --- src/ipdata/ipdata.py | 3 ++- src/ipdata/test_client.py | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/ipdata/ipdata.py b/src/ipdata/ipdata.py index cec8575..068c317 100644 --- a/src/ipdata/ipdata.py +++ b/src/ipdata/ipdata.py @@ -60,7 +60,8 @@ def __getattr__(*args): if type(val) is dict: return DotDict(val) elif type(val) is list: - return [DotDict(item) for item in val] + # wrap the dicts in a list of objects; a list of scalars is returned as it is + return [DotDict(item) if type(item) is dict else item for item in val] return val __setattr__ = dict.__setitem__ diff --git a/src/ipdata/test_client.py b/src/ipdata/test_client.py index 0b44fb5..94c81f3 100644 --- a/src/ipdata/test_client.py +++ b/src/ipdata/test_client.py @@ -12,7 +12,7 @@ import ipdata from ipdata import ipdata as ipdata_module -from ipdata.ipdata import IPData, IPDataException +from ipdata.ipdata import DotDict, IPData, IPDataException LOOKUP_RESPONSE = { "ip": "8.8.8.8", @@ -139,6 +139,23 @@ def test_lookup_returns_error_payload_with_status(): assert data.message.startswith("You have not provided") +def test_dotdict_wraps_dict_list_items_and_leaves_the_others_alone(): + data = DotDict( + { + "numbers": [1, 2, 3], + "mixed": [{"name": "a"}, "b", None], + "nested": {"entries": [{"name": "c"}]}, + } + ) + + # dot access to a list of scalars used to raise TypeError + assert data.numbers == [1, 2, 3] + assert data.mixed[0].name == "a" + assert data.mixed[1:] == ["b", None] + assert data.nested.entries[0].name == "c" + assert data.missing is None + + def test_bulk_posts_json_with_the_client_timeout(): responses = [dict(LOOKUP_RESPONSE), dict(LOOKUP_RESPONSE, ip="1.1.1.1")] client, recorder = make_client(httpx.Response(200, json=responses), timeout=7) From 017c61e7a466cb7cc50be599c253b43736a45d9a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 12:51:34 +0000 Subject: [PATCH 3/7] Fix the CLI batch --exclude option "ipdata batch -e " raised UnboundLocalError because the excluded fields were subtracted from ipdata.valid_fields before the ipdata client was assigned in the same function. The class attribute is used instead, and the remaining fields are sorted so that CSV columns always come out in the same order. The check that CSV output needs a field list now runs after --exclude has been turned into one, so "--format CSV --exclude " works as well. Co-Authored-By: Claude Fable 5.1 --- src/ipdata/cli.py | 22 +++---- src/ipdata/test_cli.py | 127 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 src/ipdata/test_cli.py diff --git a/src/ipdata/cli.py b/src/ipdata/cli.py index d71b67e..2e71b94 100644 --- a/src/ipdata/cli.py +++ b/src/ipdata/cli.py @@ -380,26 +380,28 @@ def batch(ctx, input, fields, output, format, exclude): :param input: A list of resources to lookup either IP addresses or ASNs. You could mix them however this could break output processing when writing to CSV :param fields: A list of valid fields to extract from the individual responses + :param exclude: A list of fields to leave out, every other field is extracted. Mutually exclusive with fields :param output: The path to write the results to - :param format: The format to write the results to. When using the CSV format it is required to provide fields. + :param format: The format to write the results to. When using the CSV format it is required to provide fields or exclude. """ - if format == "CSV" and not fields: - print( - f'You need to specify a "--fields" argument with a list of fields to extract to get results in CSV. To get entire responses use JSON.', - file=stderr, - ) - return - # enforce mutual exclusivity of fields and exclude if exclude and fields: raise click.ClickException( "'--fields / -f' and '--exclude / -e' are mutually exclusive." ) - # if the user wants to exclude some fields, get all the fields in fields that are not in exclude + # if the user wants to exclude some fields, request all the other ones. They are sorted so that the + # CSV columns always come out in the same order. if exclude: - fields = set(ipdata.valid_fields).difference(set(exclude)) + fields = sorted(IPData.valid_fields.difference(exclude)) + + if format == "CSV" and not fields: + print( + 'You need to specify a "--fields" argument with a list of fields to extract (or "--exclude" with the fields to leave out) to get results in CSV. To get entire responses use JSON.', + file=stderr, + ) + return # Prepare requests ipdata = IPData(ctx.obj["api-key"]) diff --git a/src/ipdata/test_cli.py b/src/ipdata/test_cli.py new file mode 100644 index 0000000..1a9861f --- /dev/null +++ b/src/ipdata/test_cli.py @@ -0,0 +1,127 @@ +""" +Offline tests for the command line interface. The IPData client that the commands create is replaced by one +wired to an httpx.MockTransport, so nothing here talks to the live API. +""" +import csv +import json + +import httpx +import pytest +from click.testing import CliRunner + +from ipdata import cli as cli_module +from ipdata.ipdata import IPData + +ASN = {"asn": "AS15169", "name": "Google LLC", "domain": "about.google", "route": "8.8.8.0/24", "type": "hosting"} + +# the members of the threat object, in the order the API returns them +THREAT = { + "is_tor": False, + "is_vpn": True, + "is_icloud_relay": False, + "is_proxy": False, + "is_datacenter": False, + "is_anonymous": True, + "is_known_attacker": False, + "is_known_abuser": False, + "is_threat": False, + "is_bogon": False, + "blocklists": [ + { + "name": "ipdata vpn", + "site": "https://ipdata.co", + "type": "vpn", + "first_seen": "2026-10-31", + "last_seen": "2026-11-02", + "days_seen": 2, + } + ], + "scores": {"vpn_score": 100, "proxy_score": 0, "threat_score": 0, "trust_score": 0}, +} + + +def bulk_response(request): + """Answers POST /bulk with one response per requested resource, in request order.""" + responses = [ + {"ip": ip, "country_code": "US", "asn": ASN, "threat": THREAT} for ip in json.loads(request.content) + ] + return httpx.Response(200, json=responses) + + +@pytest.fixture +def cli_requests(monkeypatch, tmp_path): + """ + Points the CLI at a stored API key and makes it create clients that record their requests and answer them + with bulk_response instead of sending them. Returns the list of recorded requests. + """ + key_file = tmp_path / "api-key" + key_file.write_text("test") + monkeypatch.setattr(cli_module, "API_KEY_FILE", str(key_file)) + + requests = [] + + def handler(request): + requests.append(request) + return bulk_response(request) + + class RecordingIPData(IPData): + def __init__(self, api_key=None, **kwargs): + super().__init__(api_key, transport=httpx.MockTransport(handler), **kwargs) + + monkeypatch.setattr(cli_module, "IPData", RecordingIPData) + return requests + + +def run_batch(tmp_path, *options, resources=("8.8.8.8", "1.1.1.1")): + """Runs 'ipdata batch' on a file listing 'resources' and returns the click result and the output path.""" + source = tmp_path / "ips.txt" + source.write_text("".join(f"{resource}\n" for resource in resources)) + output = tmp_path / "results" + result = CliRunner().invoke(cli_module.cli, ["batch", str(source), "--output", str(output), *options]) + return result, output + + +def test_batch_exclude_requests_every_other_field(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "--exclude", "threat", "-e", "asn") + + assert result.exit_code == 0, result.output + assert result.exception is None + assert len(cli_requests) == 1 + requested = cli_requests[0].url.params["fields"].split(",") + assert set(requested) == IPData.valid_fields - {"threat", "asn"} + assert requested == sorted(requested) + assert [json.loads(line)["ip"] for line in output.read_text().splitlines()] == ["8.8.8.8", "1.1.1.1"] + + +def test_batch_exclude_with_csv_output(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "--format", "CSV", "--exclude", "asn", "-e", "company") + + assert result.exit_code == 0, result.output + reader = csv.DictReader(output.read_text().splitlines()) + rows = list(reader) + assert len(rows) == 2 + assert not any(column.startswith(("asn_", "company_")) for column in reader.fieldnames) + # the remaining fields are requested, and therefore written, in sorted order + plain_columns = [column for column in reader.fieldnames if not column.startswith("threat_")] + assert plain_columns == sorted(plain_columns) + assert "ip" in plain_columns and "country_code" in plain_columns + assert rows[0]["ip"] == "8.8.8.8" + assert rows[0]["country_code"] == "US" + assert rows[0]["threat_is_tor"] == "False" + assert rows[1]["ip"] == "1.1.1.1" + + +def test_batch_rejects_fields_and_exclude_together(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "-f", "ip", "-e", "asn") + + assert result.exit_code != 0 + assert "mutually exclusive" in result.output + assert cli_requests == [] + + +def test_batch_csv_needs_fields_or_exclude(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "--format", "CSV") + + assert result.exit_code == 0 + assert cli_requests == [] + assert not output.exists() From 350428914ae60128fd242a8fce6c819547cf4054 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 12:51:35 +0000 Subject: [PATCH 4/7] Add is_vpn and scores to the CLI batch CSV threat columns The list of threat members that the batch command expands into CSV columns did not include is_vpn or scores, so threat_is_vpn and threat_scores were silently dropped from CSV rows while the JSON output had them. Both columns are written now, in the position the API returns them. The blocklist dates need no column of their own: they are part of each entry in the threat_blocklists column. Co-Authored-By: Claude Fable 5.1 --- src/ipdata/cli.py | 2 ++ src/ipdata/test_cli.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/ipdata/cli.py b/src/ipdata/cli.py index 2e71b94..01bbbb5 100644 --- a/src/ipdata/cli.py +++ b/src/ipdata/cli.py @@ -429,6 +429,7 @@ def batch(ctx, input, fields, output, format, exclude): f"threat_{sub_field}" for sub_field in [ "is_tor", + "is_vpn", "is_icloud_relay", "is_proxy", "is_datacenter", @@ -438,6 +439,7 @@ def batch(ctx, input, fields, output, format, exclude): "is_threat", "is_bogon", "blocklists", + "scores", ] ] continue diff --git a/src/ipdata/test_cli.py b/src/ipdata/test_cli.py index 1a9861f..9e0a2e5 100644 --- a/src/ipdata/test_cli.py +++ b/src/ipdata/test_cli.py @@ -125,3 +125,19 @@ def test_batch_csv_needs_fields_or_exclude(cli_requests, tmp_path): assert result.exit_code == 0 assert cli_requests == [] assert not output.exists() + + +def test_batch_csv_threat_columns_include_is_vpn_and_scores(cli_requests, tmp_path): + result, output = run_batch(tmp_path, "--format", "CSV", "-f", "ip", "-f", "threat", resources=("8.8.8.8",)) + + assert result.exit_code == 0, result.output + reader = csv.DictReader(output.read_text().splitlines()) + rows = list(reader) + assert reader.fieldnames == ["ip"] + [f"threat_{member}" for member in THREAT] + assert rows[0]["ip"] == "8.8.8.8" + assert rows[0]["threat_is_vpn"] == "True" + assert rows[0]["threat_is_tor"] == "False" + # nested values are written as their Python representation, the blocklist dates included + assert rows[0]["threat_blocklists"] == str(THREAT["blocklists"]) + assert "'first_seen': '2026-10-31'" in rows[0]["threat_blocklists"] + assert rows[0]["threat_scores"] == str(THREAT["scores"]) From 924985c5f4a272267cb43a18978726ce6bc6d73c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 12:52:36 +0000 Subject: [PATCH 5/7] Fix the --format flag in the README batch CSV example The example used --output-format, which the batch command does not have; the option is --format. The sentence after it now also names --exclude as the alternative to --fields for CSV output. Co-Authored-By: Claude Fable 5.1 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ab6dab..77976cc 100644 --- a/README.md +++ b/README.md @@ -544,10 +544,10 @@ ipdata batch my_ip_backlog.csv --output geolocation_results.json --format JSON ### Batch lookup with output to CSV file ``` -ipdata batch my_ip_backlog.csv --output results.csv --output-format CSV --fields ip --fields country_code +ipdata batch my_ip_backlog.csv --output results.csv --format CSV --fields ip --fields country_code ``` -The `--fields` option is required to use CSV output. +Either `--fields` or `--exclude` is required to use CSV output. #### Example Results From 9d42f29d30ab97c75771332d47d542da91484d59 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 12:52:37 +0000 Subject: [PATCH 6/7] Compare the requested and returned IPs in the live bulk tests test_bulk_v4 and test_bulk_v6 compared each requested IP with itself, so they never checked the addresses the API returned. They now check that the returned addresses are exactly the requested ones, compared case-insensitively for IPv6. Co-Authored-By: Claude Fable 5.1 --- src/ipdata/test_ipdata.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ipdata/test_ipdata.py b/src/ipdata/test_ipdata.py index a6dd63e..4d6385a 100644 --- a/src/ipdata/test_ipdata.py +++ b/src/ipdata/test_ipdata.py @@ -50,9 +50,8 @@ def test_bulk_v6(ips): # Check we got all results assert len(data.responses) == len(ips) - # Capitalize to handle 2620:11A:A000::' == '2620:11a:a000::' - for pair in zip(ips, [response.ip for response in data.responses]): - assert pair[0].upper() == pair[0].upper() + # Check we got a response for every requested IP. Capitalize to handle '2620:11A:A000::' == '2620:11a:a000::' + assert sorted(response.ip.upper() for response in data.responses) == sorted(ip.upper() for ip in ips) @given(st.lists(st.ip_addresses(network="8.8.8.0/24"), min_size=1, max_size=100)) @@ -65,5 +64,5 @@ def test_bulk_v4(ips): # Check we got all results assert len(data.responses) == len(ips) - for pair in zip(ips, [response.ip for response in data.responses]): - assert pair[0] == pair[0] + # Check we got a response for every requested IP + assert sorted(response.ip for response in data.responses) == sorted(ips) From 07ec54d9245ccd4d75df64762a179e5d1ea24b5f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 12:52:41 +0000 Subject: [PATCH 7/7] Update the CI actions, test on Python 3.14 and move the publish job to Python 3.13 actions/checkout@v4 and actions/setup-python@v5 run on Node 20, which GitHub has deprecated on its runners; both move to their current major versions, which run on Node 24. Python 3.14 joins the test matrix and the classifiers. The publish job no longer pins the 2022 patch release 3.10.4; it uses 3.13, a version the test matrix covers, and installs build without --user. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/python-publish.yml | 22 +++++++++------------- setup.cfg | 1 + 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 0d38d54..51d7486 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -16,11 +16,11 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -34,14 +34,14 @@ jobs: build-n-publish: name: Build and publish to PyPI if: github.event_name == 'push' - # the pinned Python 3.10.4 below has no build for Ubuntu 26.04 either + # kept on the same image as the test job runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.10.4 - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - name: Set up Python 3.13 + uses: actions/setup-python@v7 with: - python-version: 3.10.4 + python-version: "3.13" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -53,11 +53,7 @@ jobs: env: IPDATA_API_KEY: ${{ secrets.IPDATA_API_KEY }} - name: Install build - run: >- - python -m - pip install - build - --user + run: python -m pip install build - name: Build a binary wheel and a source tarball run: >- python -m diff --git a/setup.cfg b/setup.cfg index 2b3cca7..4d14fb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,6 +15,7 @@ classifiers = Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 Programming Language :: Python :: 3.13 + Programming Language :: Python :: 3.14 License :: OSI Approved :: MIT License Operating System :: OS Independent