Fix issues found during the connection-reuse review

.github/workflows/python-publish.yml

@@ -16,11 +16,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -34,14 +34,14 @@ jobs:
   build-n-publish:
     name: Build and publish to PyPI
     if: github.event_name == 'push'
-    # the pinned Python 3.10.4 below has no build for Ubuntu 26.04 either
+    # kept on the same image as the test job
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python 3.10.4
-      uses: actions/setup-python@v5
+    - uses: actions/checkout@v7
+    - name: Set up Python 3.13
+      uses: actions/setup-python@v7
       with:
-        python-version: 3.10.4
+        python-version: "3.13"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -53,11 +53,7 @@ jobs:
       env:
         IPDATA_API_KEY: ${{ secrets.IPDATA_API_KEY }}
     - name: Install build
-      run: >-
-        python -m
-        pip install
-        build
-        --user
+      run: python -m pip install build
     - name: Build a binary wheel and a source tarball
       run: >-
         python -m

README.md

@@ -544,10 +544,10 @@ ipdata batch my_ip_backlog.csv --output geolocation_results.json --format JSON
 ### Batch lookup with output to CSV file
 
 ```
-ipdata batch my_ip_backlog.csv --output results.csv --output-format CSV --fields ip --fields country_code
+ipdata batch my_ip_backlog.csv --output results.csv --format CSV --fields ip --fields country_code
 ```
 
-The `--fields` option is required to use CSV output.
+Either `--fields` or `--exclude` is required to use CSV output.
 
 #### Example Results
 

setup.cfg

@@ -15,6 +15,7 @@ classifiers =
     Programming Language :: Python :: 3.11
     Programming Language :: Python :: 3.12
     Programming Language :: Python :: 3.13
+    Programming Language :: Python :: 3.14
     License :: OSI Approved :: MIT License
     Operating System :: OS Independent
 

src/ipdata/cli.py

@@ -380,26 +380,28 @@ def batch(ctx, input, fields, output, format, exclude):
 
     :param input: A list of resources to lookup either IP addresses or ASNs. You could mix them however this could break output processing when writing to CSV
     :param fields: A list of valid fields to extract from the individual responses
+    :param exclude: A list of fields to leave out, every other field is extracted. Mutually exclusive with fields
     :param output: The path to write the results to
-    :param format: The format to write the results to. When using the CSV format it is required to provide fields.
+    :param format: The format to write the results to. When using the CSV format it is required to provide fields or exclude.
     """
 
-    if format == "CSV" and not fields:
-        print(
-            f'You need to specify a "--fields" argument with a list of fields to extract to get results in CSV. To get entire responses use JSON.',
-            file=stderr,
-        )
-        return
-
     # enforce mutual exclusivity of fields and exclude
     if exclude and fields:
         raise click.ClickException(
             "'--fields / -f' and '--exclude / -e' are mutually exclusive."
         )
 
-    # if the user wants to exclude some fields, get all the fields in fields that are not in exclude
+    # if the user wants to exclude some fields, request all the other ones. They are sorted so that the
+    # CSV columns always come out in the same order.
     if exclude:
-        fields = set(ipdata.valid_fields).difference(set(exclude))
+        fields = sorted(IPData.valid_fields.difference(exclude))
+
+    if format == "CSV" and not fields:
+        print(
+            'You need to specify a "--fields" argument with a list of fields to extract (or "--exclude" with the fields to leave out) to get results in CSV. To get entire responses use JSON.',
+            file=stderr,
+        )
+        return
 
     # Prepare requests
     ipdata = IPData(ctx.obj["api-key"])
@@ -427,6 +429,7 @@ def batch(ctx, input, fields, output, format, exclude):
                 f"threat_{sub_field}"
                 for sub_field in [
                     "is_tor",
+                    "is_vpn",
                     "is_icloud_relay",
                     "is_proxy",
                     "is_datacenter",
@@ -436,6 +439,7 @@ def batch(ctx, input, fields, output, format, exclude):
                     "is_threat",
                     "is_bogon",
                     "blocklists",
+                    "scores",
                 ]
             ]
             continue

src/ipdata/ipdata.py

@@ -60,7 +60,8 @@ def __getattr__(*args):
         if type(val) is dict:
             return DotDict(val)
         elif type(val) is list:
-            return [DotDict(item) for item in val]
+            # wrap the dicts in a list of objects; a list of scalars is returned as it is
+            return [DotDict(item) if type(item) is dict else item for item in val]
         return val
 
     __setattr__ = dict.__setitem__
@@ -349,7 +350,7 @@ def bulk(self, resources, fields=[]):
         """
         Lookup up to 100 resources in one request. Makes a POST request wth the resources as a JSON array and the specified fields.
 
-        :param resources: A list of resources. This can be a list of IP addresses or ASNs or a mix of both which is not recommended unless you handle it during parsing. Mixing resources might lead to weird behavior when writing results to CSV.
+        :param resources: A list or set of resources. This can be a list of IP addresses or ASNs or a mix of both which is not recommended unless you handle it during parsing. Mixing resources might lead to weird behavior when writing results to CSV.
         :param fields: A collection of API fields to be returned.
 
         :returns: A DotDict object with the data contained under the 'responses' key.
@@ -368,9 +369,9 @@ def bulk(self, resources, fields=[]):
         self._validate_fields(fields)
         query_params = {"fields": ",".join(fields)}
 
-        # Make the requests
+        # Make the requests (as a list: a set passes the validation above but is not JSON serialisable)
         try:
-            response = self._request("POST", "/bulk", params=query_params, json=resources)
+            response = self._request("POST", "/bulk", params=query_params, json=list(resources))
         except Exception as e:
             raise IPDataException(f"Error when looking up {resources} - {e}")
 

src/ipdata/test_cli.py

@@ -0,0 +1,143 @@
+"""
+Offline tests for the command line interface. The IPData client that the commands create is replaced by one
+wired to an httpx.MockTransport, so nothing here talks to the live API.
+"""
+import csv
+import json
+
+import httpx
+import pytest
+from click.testing import CliRunner
+
+from ipdata import cli as cli_module
+from ipdata.ipdata import IPData
+
+ASN = {"asn": "AS15169", "name": "Google LLC", "domain": "about.google", "route": "8.8.8.0/24", "type": "hosting"}
+
+# the members of the threat object, in the order the API returns them
+THREAT = {
+    "is_tor": False,
+    "is_vpn": True,
+    "is_icloud_relay": False,
+    "is_proxy": False,
+    "is_datacenter": False,
+    "is_anonymous": True,
+    "is_known_attacker": False,
+    "is_known_abuser": False,
+    "is_threat": False,
+    "is_bogon": False,
+    "blocklists": [
+        {
+            "name": "ipdata vpn",
+            "site": "https://ipdata.co",
+            "type": "vpn",
+            "first_seen": "2026-10-31",
+            "last_seen": "2026-11-02",
+            "days_seen": 2,
+        }
+    ],
+    "scores": {"vpn_score": 100, "proxy_score": 0, "threat_score": 0, "trust_score": 0},
+}
+
+
+def bulk_response(request):
+    """Answers POST /bulk with one response per requested resource, in request order."""
+    responses = [
+        {"ip": ip, "country_code": "US", "asn": ASN, "threat": THREAT} for ip in json.loads(request.content)
+    ]
+    return httpx.Response(200, json=responses)
+
+
+@pytest.fixture
+def cli_requests(monkeypatch, tmp_path):
+    """
+    Points the CLI at a stored API key and makes it create clients that record their requests and answer them
+    with bulk_response instead of sending them. Returns the list of recorded requests.
+    """
+    key_file = tmp_path / "api-key"
+    key_file.write_text("test")
+    monkeypatch.setattr(cli_module, "API_KEY_FILE", str(key_file))
+
+    requests = []
+
+    def handler(request):
+        requests.append(request)
+        return bulk_response(request)
+
+    class RecordingIPData(IPData):
+        def __init__(self, api_key=None, **kwargs):
+            super().__init__(api_key, transport=httpx.MockTransport(handler), **kwargs)
+
+    monkeypatch.setattr(cli_module, "IPData", RecordingIPData)
+    return requests
+
+
+def run_batch(tmp_path, *options, resources=("8.8.8.8", "1.1.1.1")):
+    """Runs 'ipdata batch' on a file listing 'resources' and returns the click result and the output path."""
+    source = tmp_path / "ips.txt"
+    source.write_text("".join(f"{resource}\n" for resource in resources))
+    output = tmp_path / "results"
+    result = CliRunner().invoke(cli_module.cli, ["batch", str(source), "--output", str(output), *options])
+    return result, output
+
+
+def test_batch_exclude_requests_every_other_field(cli_requests, tmp_path):
+    result, output = run_batch(tmp_path, "--exclude", "threat", "-e", "asn")
+
+    assert result.exit_code == 0, result.output
+    assert result.exception is None
+    assert len(cli_requests) == 1
+    requested = cli_requests[0].url.params["fields"].split(",")
+    assert set(requested) == IPData.valid_fields - {"threat", "asn"}
+    assert requested == sorted(requested)
+    assert [json.loads(line)["ip"] for line in output.read_text().splitlines()] == ["8.8.8.8", "1.1.1.1"]
+
+
+def test_batch_exclude_with_csv_output(cli_requests, tmp_path):
+    result, output = run_batch(tmp_path, "--format", "CSV", "--exclude", "asn", "-e", "company")
+
+    assert result.exit_code == 0, result.output
+    reader = csv.DictReader(output.read_text().splitlines())
+    rows = list(reader)
+    assert len(rows) == 2
+    assert not any(column.startswith(("asn_", "company_")) for column in reader.fieldnames)
+    # the remaining fields are requested, and therefore written, in sorted order
+    plain_columns = [column for column in reader.fieldnames if not column.startswith("threat_")]
+    assert plain_columns == sorted(plain_columns)
+    assert "ip" in plain_columns and "country_code" in plain_columns
+    assert rows[0]["ip"] == "8.8.8.8"
+    assert rows[0]["country_code"] == "US"
+    assert rows[0]["threat_is_tor"] == "False"
+    assert rows[1]["ip"] == "1.1.1.1"
+
+
+def test_batch_rejects_fields_and_exclude_together(cli_requests, tmp_path):
+    result, output = run_batch(tmp_path, "-f", "ip", "-e", "asn")
+
+    assert result.exit_code != 0
+    assert "mutually exclusive" in result.output
+    assert cli_requests == []
+
+
+def test_batch_csv_needs_fields_or_exclude(cli_requests, tmp_path):
+    result, output = run_batch(tmp_path, "--format", "CSV")
+
+    assert result.exit_code == 0
+    assert cli_requests == []
+    assert not output.exists()
+
+
+def test_batch_csv_threat_columns_include_is_vpn_and_scores(cli_requests, tmp_path):
+    result, output = run_batch(tmp_path, "--format", "CSV", "-f", "ip", "-f", "threat", resources=("8.8.8.8",))
+
+    assert result.exit_code == 0, result.output
+    reader = csv.DictReader(output.read_text().splitlines())
+    rows = list(reader)
+    assert reader.fieldnames == ["ip"] + [f"threat_{member}" for member in THREAT]
+    assert rows[0]["ip"] == "8.8.8.8"
+    assert rows[0]["threat_is_vpn"] == "True"
+    assert rows[0]["threat_is_tor"] == "False"
+    # nested values are written as their Python representation, the blocklist dates included
+    assert rows[0]["threat_blocklists"] == str(THREAT["blocklists"])
+    assert "'first_seen': '2026-10-31'" in rows[0]["threat_blocklists"]
+    assert rows[0]["threat_scores"] == str(THREAT["scores"])

src/ipdata/test_client.py

@@ -12,7 +12,7 @@
 
 import ipdata
 from ipdata import ipdata as ipdata_module
-from ipdata.ipdata import IPData, IPDataException
+from ipdata.ipdata import DotDict, IPData, IPDataException
 
 LOOKUP_RESPONSE = {
     "ip": "8.8.8.8",
@@ -139,6 +139,23 @@ def test_lookup_returns_error_payload_with_status():
     assert data.message.startswith("You have not provided")
 
 
+def test_dotdict_wraps_dict_list_items_and_leaves_the_others_alone():
+    data = DotDict(
+        {
+            "numbers": [1, 2, 3],
+            "mixed": [{"name": "a"}, "b", None],
+            "nested": {"entries": [{"name": "c"}]},
+        }
+    )
+
+    # dot access to a list of scalars used to raise TypeError
+    assert data.numbers == [1, 2, 3]
+    assert data.mixed[0].name == "a"
+    assert data.mixed[1:] == ["b", None]
+    assert data.nested.entries[0].name == "c"
+    assert data.missing is None
+
+
 def test_bulk_posts_json_with_the_client_timeout():
     responses = [dict(LOOKUP_RESPONSE), dict(LOOKUP_RESPONSE, ip="1.1.1.1")]
     client, recorder = make_client(httpx.Response(200, json=responses), timeout=7)
@@ -166,6 +183,18 @@ def test_lookup_uses_the_default_timeout():
     assert recorder.requests[0].extensions["timeout"] == {"connect": 60, "read": 60, "write": 60, "pool": 60}
 
 
+def test_bulk_accepts_a_set_of_resources():
+    responses = [dict(LOOKUP_RESPONSE), dict(LOOKUP_RESPONSE, ip="1.1.1.1")]
+    client, recorder = make_client(httpx.Response(200, json=responses))
+
+    data = client.bulk({"8.8.8.8", "1.1.1.1"}, fields=["ip"])
+
+    # a set passes the validation but is not JSON serialisable, so it is sent as a list
+    assert sorted(json.loads(recorder.requests[0].content)) == ["1.1.1.1", "8.8.8.8"]
+    assert data.status == 200
+    assert [r.ip for r in data.responses] == ["8.8.8.8", "1.1.1.1"]
+
+
 def test_bulk_non_200_returns_payload_with_status():
     client, _ = make_client(httpx.Response(403, json={"message": "Forbidden"}))
 

src/ipdata/test_ipdata.py

@@ -50,9 +50,8 @@ def test_bulk_v6(ips):
     # Check we got all results
     assert len(data.responses) == len(ips)
 
-    # Capitalize to handle 2620:11A:A000::' == '2620:11a:a000::'
-    for pair in zip(ips, [response.ip for response in data.responses]):
-        assert pair[0].upper() == pair[0].upper()
+    # Check we got a response for every requested IP. Capitalize to handle '2620:11A:A000::' == '2620:11a:a000::'
+    assert sorted(response.ip.upper() for response in data.responses) == sorted(ip.upper() for ip in ips)
 
 
 @given(st.lists(st.ip_addresses(network="8.8.8.0/24"), min_size=1, max_size=100))
@@ -65,5 +64,5 @@ def test_bulk_v4(ips):
     # Check we got all results
     assert len(data.responses) == len(ips)
 
-    for pair in zip(ips, [response.ip for response in data.responses]):
-        assert pair[0] == pair[0]
+    # Check we got a response for every requested IP
+    assert sorted(response.ip for response in data.responses) == sorted(ips)