diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index c6df0ed..0d38d54 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -6,11 +6,36 @@ on: - master tags: - '*' + pull_request: jobs: + test: + name: Test (Python ${{ matrix.python-version }}) + # ubuntu-latest now resolves to Ubuntu 26.04, for which actions/setup-python has no Python 3.9 builds + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + - name: Test with pytest (offline tests only, no API key needed) + run: | + python -m pytest -m "not api" + build-n-publish: name: Build and publish to PyPI - runs-on: ubuntu-latest + if: github.event_name == 'push' + # the pinned Python 3.10.4 below has no build for Ubuntu 26.04 either + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Set up Python 3.10.4 diff --git a/README.md b/README.md index 3918dc8..7ab6dab 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Visit our [Documentation](https://docs.ipdata.co/) for more examples and tutoria - [Getting only one field](#getting-only-one-field) - [Getting a number of specific fields](#getting-a-number-of-specific-fields) - [Bulk Lookups](#bulk-lookups) + - [Connection reuse and HTTP/2](#connection-reuse-and-http2) - [Using the ipdata CLI](#using-the-ipdata-cli) - [Windows Installation Notes](#windows-installation-notes) - [Available commands](#available-commands) @@ -367,6 +368,24 @@ NOTE: Alternatively it is much simpler to process bulk lookups using the `ipdata ``` +### Connection reuse and HTTP/2 + +An `IPData` client keeps a persistent HTTPS connection pool for as long as it lives. Create one client per process (or one per API key or endpoint), keep it, and reuse it for every lookup: each new client pays a fresh TLS handshake. Requests to api.ipdata.co are multiplexed over HTTP/2 when the platform negotiates it, and fall back to HTTP/1.1 keep-alive otherwise. + +The client is built on [httpx](https://www.python-httpx.org/) and is thread-safe: one instance can serve every thread of a process at once (the CLI's `batch` command shares a single client across its thread pool). Idle connections are kept open for 60 seconds and up to 20 connections are pooled. Call `close()` at shutdown, or use the client as a context manager, to release the pool. Pass `http2=False` to stay on HTTP/1.1 keep-alive, or `transport=` to inject your own `httpx.BaseTransport` (for example an `httpx.MockTransport` in tests). + +```python +from ipdata import IPData + +with IPData("") as client: # one client, many lookups + for ip in ["8.8.8.8", "1.1.1.1"]: + print(client.lookup(ip).country_name) +``` + +The module-level helpers `ipdata.lookup()` and `ipdata.bulk()` share one process-wide client, `ipdata.default_client`, which is created on first use and lives until the process exits. Connections must not be shared across `os.fork()` or `multiprocessing` boundaries: create your `IPData` instance (or make the first module-level call) in the child process, after forking. + +Failed requests are retried up to `retry_limit` times (default 7). GET and POST requests that fail with a transport error (connection reset, timeout) or with HTTP 500, 502, 503 or 504 are retried with exponential backoff: 0.5, 1, 2, 4, 8, 16 and 32 seconds for the default `retry_backoff_factor=1`. A 429 response is retried only when it carries a `Retry-After` header (waiting at most 60 seconds); without one it is returned immediately, so an exhausted quota fails fast. The `HTTPS_PROXY` and `NO_PROXY` environment variables are honoured; a custom CA bundle is read from `SSL_CERT_FILE` (httpx does not read `REQUESTS_CA_BUNDLE`). + ## Using the ipdata CLI @@ -546,6 +565,18 @@ The `--fields` option is required to use CSV output. A list of all the fields returned by the API is maintained at [Response Fields](https://docs.ipdata.co/api-reference/response-fields) +The following fields were added to the API in November 2026. The library passes them through unchanged, and both top-level flags are accepted by the `fields` and `select_field` parameters of `lookup()` and `bulk()` and by the CLI's `-f` and `-e` options. + +| Field | Description | +|-------|-------------| +| `is_satellite` | `true` when the address belongs to a satellite internet provider (for example Starlink). Present only when true, omitted otherwise. | +| `is_anycast` | `true` when the address is an anycast address announced from several locations at once (for example `8.8.8.8`). Present only when true, omitted otherwise. | +| `threat.blocklists[].first_seen` | The UTC calendar date (a `YYYY-MM-DD` string) on which that blocklist first listed the address. Optional: some feeds, for example ipdata's own vpn feed, do not provide it. | +| `threat.blocklists[].last_seen` | The UTC calendar date (a `YYYY-MM-DD` string) on which that blocklist last listed the address. Optional, as above. | +| `threat.blocklists[].days_seen` | The number of days the address has been listed by that blocklist (an integer). Optional, as above. | + +`is_satellite` and `is_anycast` are included in a response only when they are `true`. When absent they are missing from the dictionary and read as `None` through dot notation (`response.is_satellite`), so treat anything falsy as `false`. The dates are kept as strings and are not parsed. Fields the library does not know about yet are passed through as well. + ## Geofeed tools Geofeed publishers can use the `ipdata validate` command to validate their geofeeds before submission to ipdata. This will catch most but not all issues that might cause processing your geofeed to fail. @@ -754,3 +785,9 @@ To run all tests ```shell pytest ``` + +The tests marked `api` call the live API and need an `IPDATA_API_KEY` environment variable (or a `src/ipdata/.env` file). To run only the offline tests, as CI does for pull requests: + +```shell +pytest -m "not api" +``` diff --git a/requirements.txt b/requirements.txt index 1cbcb21..7da331f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests +httpx[http2]>=0.28,<1 rich click click_default_group diff --git a/setup.cfg b/setup.cfg index 8bcf0d2..2b3cca7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = ipdata -version = 4.0.9 +version = 4.1.0 author = Jonathan Kosgei author_email = jonathan@ipdata.co description = This is the official IPData client library for Python @@ -24,7 +24,7 @@ package_dir = packages = find: python_requires = >=3.9 install_requires = - requests + httpx[http2]>=0.28,<1 rich click click_default_group diff --git a/src/ipdata/geofeeds.py b/src/ipdata/geofeeds.py index 7d8bcba..ddb1dd4 100644 --- a/src/ipdata/geofeeds.py +++ b/src/ipdata/geofeeds.py @@ -8,7 +8,7 @@ import random from pathlib import Path -import requests +import httpx from .codes import COUNTRIES, REGION_CODES @@ -49,7 +49,7 @@ def _download(self): cache_path = f"{self.dir}/{random_name}.csv" try: - response = requests.get(self.source, timeout=60) + response = httpx.get(self.source, timeout=60, follow_redirects=True) text = response.text with open(cache_path, "w") as f: f.write( diff --git a/src/ipdata/ipdata.py b/src/ipdata/ipdata.py index b840d8c..f21ddfe 100644 --- a/src/ipdata/ipdata.py +++ b/src/ipdata/ipdata.py @@ -18,18 +18,32 @@ :class:`~.IPData` is the primary class for making API requests. """ -import os +import email.utils +import importlib.metadata import ipaddress -import requests import logging -import urllib3 -import functools +import os +import time +from datetime import datetime, timezone -from requests.adapters import HTTPAdapter, Retry +import httpx logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +try: + __version__ = importlib.metadata.version("ipdata") +except importlib.metadata.PackageNotFoundError: # running from a source checkout + __version__ = "unknown" + +# HTTP statuses that are retried, for GET and POST requests alike +RETRY_STATUSES = frozenset({500, 502, 503, 504}) +# Longest pause honoured from a 429 Retry-After header, in seconds +MAX_RETRY_AFTER = 60.0 + +# Seam for the tests, which replace it to check the retry schedule without sleeping +_sleep = time.sleep + class IPDataException(Exception): pass @@ -57,12 +71,26 @@ class IPData(object): """ Instances of IPData are used to make API requests. To make requests to the EU endpoint, set "endpoint" to "https://eu-api.ipdata.co". + Every instance owns one HTTPS connection pool that is kept alive and reused for all of its requests + (multiplexed over HTTP/2 when the server negotiates it, HTTP/1.1 keep-alive otherwise), so create one + client per process (or per API key or endpoint), reuse it for every lookup and call :meth:`close` (or use + the instance as a context manager) when you are done with it. Instances are thread-safe. A client must + be created after forking: never share one across os.fork() or multiprocessing boundaries. + + Failed requests are retried up to 'retry_limit' times. GET and POST requests that fail with a transport + error (connection reset, timeout, ...) or with HTTP 500, 502, 503 or 504 are retried after sleeping + retry_backoff_factor * 2 ** (attempt - 2) seconds, i.e. 0.5, 1, 2, 4, 8, 16 and 32 seconds with the + defaults. HTTP 429 is retried only when the response carries a Retry-After header (sleeping at most 60 + seconds); without one the response is returned immediately so that an exhausted quota fails fast. + :param api_key: A valid IPData API key :param endpoint: The API endpoint to send requests to - :param timeout: The default requests timeout + :param timeout: The default requests timeout in seconds, applied to every request including bulk lookups :param retry_limit: The maximum number of retries in case of failure :param retry_backoff_factor: The number of seconds to sleep in between retries. With a 1 second backoff calls with be retried up to 7 times at the following intervals: 0.5, 1, 2, 4, 8, 16, 32 :param debug: A boolean used to set the log level. Set to True when debugging. + :param http2: Negotiate HTTP/2 with automatic fallback to HTTP/1.1. Set to False to use HTTP/1.1 keep-alive only. + :param transport: An optional httpx.BaseTransport used instead of the default HTTPS transport, for example an httpx.MockTransport in tests """ log = logging.getLogger(__name__) @@ -70,6 +98,8 @@ class IPData(object): valid_fields = { "ip", "is_eu", + "is_satellite", + "is_anycast", "city", "region", "region_code", @@ -103,6 +133,8 @@ def __init__( retry_limit=7, retry_backoff_factor=1, debug=False, + http2=True, + transport=None, ): if api_key is None: api_key = os.environ.get("IPDATA_API_KEY") @@ -112,7 +144,12 @@ def __init__( self.api_key = api_key self.endpoint = endpoint.rstrip("/") # remove trailing / self._timeout = timeout - self._headers = {"user-agent": "ipdata-python"} # set default UserAgent + self._retry_limit = retry_limit + self._retry_backoff_factor = retry_backoff_factor + self._headers = { + "user-agent": f"ipdata-python/{__version__}", # set default UserAgent + "accept": "application/json", + } self._query_params = {"api-key": self.api_key} # Enable debugging @@ -126,24 +163,37 @@ def __init__( handler.setFormatter(logging.Formatter("%(message)s", datefmt="[%X]")) self.log.addHandler(handler) - # Work around renamed argument in urllib3. - if hasattr(urllib3.util.Retry.DEFAULT, "allowed_methods"): - methods_arg = "allowed_methods" - else: - methods_arg = "method_whitelist" - - # Retry settings - retry_args = { - "total": retry_limit, - "backoff_factor": retry_backoff_factor, - "status_forcelist": [429, 500, 502, 503, 504], - methods_arg: {"POST"}, - } + # One connection pool per client instance. Connections are kept alive for 60 seconds and reused + # across requests (multiplexed over HTTP/2 when the server negotiates it, HTTP/1.1 keep-alive + # otherwise) instead of paying a TCP + TLS handshake per call. When no transport is injected + # httpx builds its default one, which honours HTTPS_PROXY / NO_PROXY and the SSL_CERT_* variables + # like requests did; with an injected transport httpx uses it as-is and ignores http2 and limits. + self._client = httpx.Client( + base_url=self.endpoint, + http2=http2, + timeout=httpx.Timeout(timeout), + headers=self._headers, + params=self._query_params, + limits=httpx.Limits( + max_connections=20, max_keepalive_connections=20, keepalive_expiry=60.0 + ), + transport=transport, + ) + + def close(self): + """ + Closes the connection pool. The instance can not be used afterwards. Call this at shutdown, or use the instance as a context manager: + + >>> with IPData("YOUR API KEY") as ipdata: + ... ipdata.lookup("8.8.8.8") + """ + self._client.close() - adapter = HTTPAdapter(max_retries=urllib3.Retry(**retry_args)) + def __enter__(self): + return self - self._session = requests.Session() - self._session.mount("http", adapter) + def __exit__(self, exc_type, exc_value, traceback): + self.close() def _validate_fields(self, fields): """ @@ -173,6 +223,74 @@ def _validate_ip_address(self, ip): if request_ip.is_private or request_ip.is_reserved or request_ip.is_multicast: raise ValueError(f"{ip} is a reserved IP Address") + def _backoff(self, attempt): + """ + Seconds to sleep before retry number 'attempt' (1-based): 0.5, 1, 2, 4, 8, 16, 32 with the default backoff factor. + """ + return self._retry_backoff_factor * 2 ** (attempt - 2) + + @staticmethod + def _retry_after(value): + """ + Parses a Retry-After header, given either as delay-seconds or as an HTTP-date. + + :returns: The number of seconds to wait, or None if the value can not be parsed + """ + value = value.strip() + if value.isdigit(): + return float(value) + try: + when = email.utils.parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + if when.tzinfo is None: + when = when.replace(tzinfo=timezone.utc) + return max((when - datetime.now(timezone.utc)).total_seconds(), 0.0) + + def _request(self, method, path, **kwargs): + """ + Sends a request over the shared connection pool, retrying it as described in the class docstring. + + :returns: The last httpx.Response received + :raises httpx.TransportError: if the request still fails to get a response once the retries are exhausted + """ + attempt = 0 + while True: + attempt += 1 + try: + response = self._client.request(method, path, **kwargs) + except httpx.UnsupportedProtocol: + raise # a configuration error that no retry can fix + except httpx.TransportError as e: + if attempt > self._retry_limit: + raise + delay = self._backoff(attempt) + reason = f"{type(e).__name__}: {e}" + else: + status_code = response.status_code + if attempt > self._retry_limit: + return response + if status_code in RETRY_STATUSES: + delay = self._backoff(attempt) + elif status_code == 429 and "retry-after" in response.headers: + delay = self._retry_after(response.headers["retry-after"]) + if delay is None: + delay = self._backoff(attempt) + delay = min(delay, MAX_RETRY_AFTER) + else: + return response + reason = f"HTTP {status_code}" + self.log.debug( + "Retrying %s %s in %.1fs (retry %d of %d) after %s", + method, + path, + delay, + attempt, + self._retry_limit, + reason, + ) + _sleep(delay) + def lookup(self, resource="", fields=[], select_field=None): """ Makes a GET request to the IPData API for the specified 'resource' and the given 'fields'. @@ -204,16 +322,11 @@ def lookup(self, resource="", fields=[], select_field=None): fields = [select_field] self._validate_fields(fields) - query_params = self._query_params | {"fields": ",".join(fields)} + query_params = {"fields": ",".join(fields)} # Make the request try: - response = self._session.get( - f"{self.endpoint}/{resource}", - headers=self._headers, - params=query_params, - timeout=self._timeout, - ) + response = self._request("GET", f"/{resource}", params=query_params) except Exception as e: raise IPDataException(e) @@ -253,16 +366,11 @@ def bulk(self, resources, fields=[]): raise ValueError("Bulk lookups must contain at least 1 resource.") self._validate_fields(fields) - query_params = self._query_params | {"fields": ",".join(fields)} + query_params = {"fields": ",".join(fields)} # Make the requests try: - response = self._session.post( - f"{self.endpoint}/bulk", - headers=self._headers, - params=query_params, - json=resources, - ) + response = self._request("POST", "/bulk", params=query_params, json=resources) except Exception as e: raise IPDataException(f"Error when looking up {resources} - {e}") diff --git a/src/ipdata/test_client.py b/src/ipdata/test_client.py new file mode 100644 index 0000000..fb8d93e --- /dev/null +++ b/src/ipdata/test_client.py @@ -0,0 +1,437 @@ +""" +Offline tests for the IPData client. Every request goes through an httpx.MockTransport injected with +'transport=', so nothing here talks to the live API. +""" +import json +import re +from datetime import datetime, timedelta, timezone +from email.utils import format_datetime + +import httpx +import pytest + +import ipdata +from ipdata import ipdata as ipdata_module +from ipdata.ipdata import IPData, IPDataException + +LOOKUP_RESPONSE = { + "ip": "8.8.8.8", + "is_eu": False, + "country_code": "US", + "asn": { + "asn": "AS15169", + "name": "Google LLC", + "domain": "about.google", + "route": "8.8.8.0/24", + "type": "hosting", + }, + "count": "0", +} + + +class Recorder: + """ + A MockTransport handler that records every request it receives and replays scripted responses in + order. The last response repeats; an exception in the script is raised instead of being returned. + """ + + def __init__(self, *responses): + self.requests = [] + self.responses = list(responses) or [httpx.Response(200, json=LOOKUP_RESPONSE)] + + def __call__(self, request): + self.requests.append(request) + response = self.responses.pop(0) if len(self.responses) > 1 else self.responses[0] + if isinstance(response, Exception): + raise response + return response + + +def make_client(*responses, **kwargs): + """Returns an IPData client wired to a recording mock transport, plus the recorder.""" + recorder = Recorder(*responses) + kwargs.setdefault("api_key", "test") + return IPData(transport=httpx.MockTransport(recorder), **kwargs), recorder + + +@pytest.fixture +def sleeps(monkeypatch): + """Replaces the retry sleep with a recorder so that the retry schedule can be asserted instantly.""" + recorded = [] + monkeypatch.setattr(ipdata_module, "_sleep", recorded.append) + return recorded + + +def test_lookup_sends_api_key_user_agent_and_fields(): + client, recorder = make_client() + + data = client.lookup("8.8.8.8") + + request = recorder.requests[0] + assert request.method == "GET" + assert request.url.scheme == "https" + assert request.url.host == "api.ipdata.co" + assert request.url.path == "/8.8.8.8" + assert request.url.params["api-key"] == "test" + assert request.url.params["fields"] == "" + assert request.headers["user-agent"] == f"ipdata-python/{ipdata_module.__version__}" + assert re.fullmatch(r"ipdata-python/\S+", request.headers["user-agent"]) + assert request.headers["accept"] == "application/json" + assert data.ip == "8.8.8.8" + assert data.asn.name == "Google LLC" + assert data.status == 200 + assert "endpoint" not in data + + +def test_lookup_fields_and_eu_endpoint(): + client, recorder = make_client( + endpoint="https://eu-api.ipdata.co/", api_key="eu-key" + ) + + data = client.lookup("8.8.8.8", fields=["ip", "asn"]) + + request = recorder.requests[0] + assert request.url.host == "eu-api.ipdata.co" + assert request.url.params["api-key"] == "eu-key" + assert request.url.params["fields"] == "ip,asn" + assert data.endpoint == "EU" + + +def test_lookup_select_field_ipv6_and_asn_paths(): + client, recorder = make_client() + + client.lookup("2620:11a:a000::", select_field="asn") + client.lookup("as15169") + client.lookup() + + # lookup() upper-cases the resource so that typoed ASNs such as "as15169" still work + assert [r.url.path for r in recorder.requests] == ["/2620:11A:A000::", "/AS15169", "/"] + assert recorder.requests[0].url.params["fields"] == "asn" + + +def test_lookup_validation_happens_before_any_request(): + client, recorder = make_client() + + with pytest.raises(ValueError): + client.lookup("10.0.0.1") # reserved address + with pytest.raises(ValueError): + client.lookup("8.8.8.8", fields=["nope"]) + with pytest.raises(ValueError): + client.lookup("8.8.8.8", fields="ip") + with pytest.raises(ValueError): + client.lookup("8.8.8.8", fields=["ip"], select_field="asn") + assert recorder.requests == [] + + +def test_lookup_invalid_json_raises_ipdata_exception(): + client, _ = make_client(httpx.Response(200, text="not json")) + + with pytest.raises(IPDataException, match="decoding the API response"): + client.lookup("8.8.8.8") + + +def test_lookup_returns_error_payload_with_status(): + client, _ = make_client(httpx.Response(401, json={"message": "You have not provided a valid API Key."})) + + data = client.lookup("8.8.8.8") + + assert data.status == 401 + assert data.message.startswith("You have not provided") + + +def test_bulk_posts_json_with_the_client_timeout(): + responses = [dict(LOOKUP_RESPONSE), dict(LOOKUP_RESPONSE, ip="1.1.1.1")] + client, recorder = make_client(httpx.Response(200, json=responses), timeout=7) + + data = client.bulk(["8.8.8.8", "1.1.1.1"], fields=["ip", "asn"]) + + request = recorder.requests[0] + assert request.method == "POST" + assert request.url.path == "/bulk" + assert request.url.params["api-key"] == "test" + assert request.url.params["fields"] == "ip,asn" + assert request.headers["content-type"] == "application/json" + assert json.loads(request.content) == ["8.8.8.8", "1.1.1.1"] + # bulk() previously sent its request without any timeout; it now uses the client timeout + assert request.extensions["timeout"] == {"connect": 7, "read": 7, "write": 7, "pool": 7} + assert [r.ip for r in data.responses] == ["8.8.8.8", "1.1.1.1"] + assert data.status == 200 + + +def test_lookup_uses_the_default_timeout(): + client, recorder = make_client() + + client.lookup("8.8.8.8") + + assert recorder.requests[0].extensions["timeout"] == {"connect": 60, "read": 60, "write": 60, "pool": 60} + + +def test_bulk_non_200_returns_payload_with_status(): + client, _ = make_client(httpx.Response(403, json={"message": "Forbidden"})) + + data = client.bulk(["8.8.8.8"]) + + assert data == {"message": "Forbidden", "status": 403} + + +def test_bulk_validation_happens_before_any_request(): + client, recorder = make_client() + + with pytest.raises(ValueError): + client.bulk("8.8.8.8") + with pytest.raises(ValueError): + client.bulk([]) + with pytest.raises(ValueError): + client.bulk(["8.8.8.8"], fields=["nope"]) + assert recorder.requests == [] + + +def test_retry_schedule_on_server_errors(sleeps): + client, recorder = make_client( + httpx.Response(503, json={"message": "Service Unavailable"}), + httpx.Response(502, text="Bad Gateway"), + httpx.Response(500, json={"message": "Internal server error"}), + httpx.Response(504, text="Gateway Timeout"), + httpx.Response(200, json=LOOKUP_RESPONSE), + ) + + data = client.lookup("8.8.8.8") + + assert len(recorder.requests) == 5 + assert sleeps == [0.5, 1, 2, 4] + assert data.status == 200 + assert data.ip == "8.8.8.8" + + +def test_retry_schedule_uses_the_backoff_factor(sleeps): + client, recorder = make_client( + *[httpx.Response(503, text="Service Unavailable")] * 7, + httpx.Response(200, json=LOOKUP_RESPONSE), + retry_backoff_factor=2, + ) + + data = client.lookup("8.8.8.8") + + assert len(recorder.requests) == 8 + assert sleeps == [1, 2, 4, 8, 16, 32, 64] + assert data.status == 200 + + +def test_full_default_schedule_then_last_response_is_returned(sleeps): + client, recorder = make_client(httpx.Response(503, json={"message": "Service Unavailable"})) + + data = client.lookup("8.8.8.8") + + # 1 request + 7 retries, with the documented sleeps in between + assert len(recorder.requests) == 8 + assert sleeps == [0.5, 1, 2, 4, 8, 16, 32] + assert data.status == 503 + + +def test_retry_limit_is_honoured_for_bulk(sleeps): + client, recorder = make_client(httpx.Response(500, json={"message": "Internal server error"}), retry_limit=3) + + data = client.bulk(["8.8.8.8"]) + + assert len(recorder.requests) == 4 + assert all(r.method == "POST" for r in recorder.requests) + assert sleeps == [0.5, 1, 2] + assert data == {"message": "Internal server error", "status": 500} + + +def test_retry_on_transport_errors(sleeps): + client, recorder = make_client( + httpx.ConnectError("connection refused"), + httpx.ReadTimeout("timed out"), + httpx.RemoteProtocolError("Server disconnected without sending a response."), + httpx.Response(200, json=LOOKUP_RESPONSE), + ) + + data = client.lookup("8.8.8.8") + + assert len(recorder.requests) == 4 + assert sleeps == [0.5, 1, 2] + assert data.status == 200 + + +def test_bulk_retries_a_request_that_died_on_a_stale_connection(sleeps): + client, recorder = make_client( + httpx.RemoteProtocolError("Server disconnected without sending a response."), + httpx.Response(200, json=[LOOKUP_RESPONSE]), + ) + + data = client.bulk(["8.8.8.8"]) + + assert len(recorder.requests) == 2 + assert sleeps == [0.5] + assert data.responses[0].ip == "8.8.8.8" + + +def test_exhausted_transport_errors_raise_ipdata_exception(sleeps): + client, recorder = make_client(httpx.ConnectError("connection refused"), retry_limit=2) + + with pytest.raises(IPDataException, match="connection refused"): + client.lookup("8.8.8.8") + assert len(recorder.requests) == 3 + assert sleeps == [0.5, 1] + + with pytest.raises(IPDataException, match="Error when looking up"): + client.bulk(["8.8.8.8"]) + assert len(recorder.requests) == 6 + + +def test_retry_limit_zero_disables_retries(sleeps): + client, recorder = make_client(httpx.Response(503, json={"message": "Service Unavailable"}), retry_limit=0) + data = client.lookup("8.8.8.8") + assert data.status == 503 + assert len(recorder.requests) == 1 + + client, recorder = make_client(httpx.ConnectError("connection refused"), retry_limit=0) + with pytest.raises(IPDataException): + client.lookup("8.8.8.8") + assert len(recorder.requests) == 1 + assert sleeps == [] + + +def test_429_without_retry_after_fails_fast(sleeps): + client, recorder = make_client(httpx.Response(429, json={"message": "You have exceeded your quota."})) + + data = client.lookup("8.8.8.8") + + assert len(recorder.requests) == 1 + assert sleeps == [] + assert data.status == 429 + assert data.message == "You have exceeded your quota." + + +def test_429_with_retry_after_seconds_is_retried(sleeps): + client, recorder = make_client( + httpx.Response(429, headers={"Retry-After": "2"}, json={"message": "slow down"}), + httpx.Response(200, json=LOOKUP_RESPONSE), + ) + + data = client.lookup("8.8.8.8") + + assert len(recorder.requests) == 2 + assert sleeps == [2.0] + assert data.status == 200 + + +def test_429_retry_after_is_capped_at_60_seconds(sleeps): + client, recorder = make_client( + httpx.Response(429, headers={"Retry-After": "600"}, json={"message": "slow down"}), + httpx.Response(200, json=LOOKUP_RESPONSE), + ) + + client.lookup("8.8.8.8") + + assert sleeps == [60.0] + + +def test_429_retry_after_http_date(sleeps): + when = datetime.now(timezone.utc) + timedelta(seconds=30) + client, recorder = make_client( + httpx.Response(429, headers={"Retry-After": format_datetime(when, usegmt=True)}, json={}), + httpx.Response(200, json=LOOKUP_RESPONSE), + ) + + client.lookup("8.8.8.8") + + assert len(sleeps) == 1 + assert 25 < sleeps[0] <= 30 + + +def test_429_with_unparseable_retry_after_uses_the_backoff(sleeps): + client, recorder = make_client( + httpx.Response(429, headers={"Retry-After": "soon"}, json={}), + httpx.Response(200, json=LOOKUP_RESPONSE), + ) + + client.lookup("8.8.8.8") + + assert sleeps == [0.5] + + +def test_429_retries_count_against_the_retry_limit(sleeps): + client, recorder = make_client( + httpx.Response(429, headers={"Retry-After": "1"}, json={"message": "slow down"}), retry_limit=2 + ) + + data = client.lookup("8.8.8.8") + + assert len(recorder.requests) == 3 + assert sleeps == [1.0, 1.0] + assert data.status == 429 + + +def test_requests_share_one_transport_and_client(): + recorder = Recorder(httpx.Response(200, json=LOOKUP_RESPONSE), httpx.Response(200, json=LOOKUP_RESPONSE), httpx.Response(200, json=[LOOKUP_RESPONSE])) + transport = httpx.MockTransport(recorder) + client = IPData("test", transport=transport) + http_client = client._client + assert isinstance(http_client, httpx.Client) + assert http_client._transport is transport + + client.lookup("8.8.8.8") + client.lookup("1.1.1.1") + client.bulk(["8.8.8.8"]) + + # all three calls went through the same transport object and the same httpx.Client + assert len(recorder.requests) == 3 + assert client._client is http_client + assert http_client._transport is transport + assert not http_client.is_closed + + +def test_close_and_context_manager(): + client, recorder = make_client() + with client as entered: + assert entered is client + client.lookup("8.8.8.8") + assert not client._client.is_closed + assert client._client.is_closed + # a closed client can not be used again, and the failure surfaces as the usual IPDataException + with pytest.raises(IPDataException): + client.lookup("8.8.8.8") + assert len(recorder.requests) == 1 + + client, _ = make_client() + client.close() + client.close() # closing twice is harmless + assert client._client.is_closed + + +def test_default_transport_pool_configuration(): + client = IPData("test") + try: + assert isinstance(client._client._transport, httpx.HTTPTransport) + pool = client._client._transport._pool + assert pool._keepalive_expiry == 60.0 + assert pool._max_connections == 20 + assert pool._max_keepalive_connections == 20 + assert pool._http2 is True + assert client._client.headers["user-agent"] == f"ipdata-python/{ipdata_module.__version__}" + assert client._client.params["api-key"] == "test" + assert client._client.timeout == httpx.Timeout(60) + finally: + client.close() + + client = IPData("test", http2=False) + try: + assert client._client._transport._pool._http2 is False + finally: + client.close() + + +def test_module_level_helpers_reuse_one_default_client(monkeypatch): + recorder = Recorder(httpx.Response(200, json=LOOKUP_RESPONSE), httpx.Response(200, json=LOOKUP_RESPONSE), httpx.Response(200, json=[LOOKUP_RESPONSE])) + client = IPData("test", transport=httpx.MockTransport(recorder)) + monkeypatch.setattr(ipdata, "default_client", client) + + ipdata.lookup("8.8.8.8") + ipdata.lookup("1.1.1.1", fields=["ip"]) + ipdata.bulk(["8.8.8.8"]) + + assert ipdata.default_client is client + assert len(recorder.requests) == 3 + assert recorder.requests[1].url.params["fields"] == "ip" diff --git a/src/ipdata/test_geofeeds.py b/src/ipdata/test_geofeeds.py index d35a89a..184b1dc 100644 --- a/src/ipdata/test_geofeeds.py +++ b/src/ipdata/test_geofeeds.py @@ -1,6 +1,7 @@ +import httpx import pytest -from .geofeeds import Entry, GeofeedValidationError +from .geofeeds import Entry, Geofeed, GeofeedValidationError @pytest.mark.parametrize( @@ -18,3 +19,28 @@ def test_validation_rules(entry): entry = Entry(*entry.split(",")) with pytest.raises(GeofeedValidationError): entry.validate() + + +@pytest.mark.geofeed +def test_download_follows_redirects(monkeypatch, tmp_path): + from . import geofeeds + + calls = [] + + def fake_get(url, **kwargs): + calls.append((url, kwargs)) + return httpx.Response( + 200, text="# comment line\n8.8.8.0/24,US,US-CA,Mountain View,94043\n" + ) + + monkeypatch.setattr(geofeeds.httpx, "get", fake_get) + feed = Geofeed("https://example.com/geofeed.csv", dir=str(tmp_path)) + + entries = list(feed) + + assert calls == [("https://example.com/geofeed.csv", {"timeout": 60, "follow_redirects": True})] + assert feed.cache_path.startswith(str(tmp_path)) + assert len(entries) == 1 + assert entries[0].ip_range == "8.8.8.0/24" + assert entries[0].country == "US" + entries[0].validate() diff --git a/src/ipdata/test_response_fields.py b/src/ipdata/test_response_fields.py new file mode 100644 index 0000000..f94b25d --- /dev/null +++ b/src/ipdata/test_response_fields.py @@ -0,0 +1,324 @@ +""" +Offline tests for the response fields the API added in November 2026: the top-level booleans +'is_satellite' and 'is_anycast', which the API includes only when they are true, and the optional +'first_seen', 'last_seen' and 'days_seen' attributes of 'threat.blocklists' entries. The library passes +the JSON through unchanged. The fixtures are built from the specification samples; nothing here calls +the live API. +""" +import json + +import httpx +import pytest + +from ipdata.ipdata import IPData + +# Sample A: satellite ISP. All other fields keep today's shape. +SATELLITE_RESPONSE = { + "ip": "98.97.34.0", + "is_satellite": True, + "is_eu": False, + "city": "Seattle", + "country_code": "US", + "asn": { + "asn": "AS14593", + "name": "Space Exploration Technologies Corporation", + "domain": "spacex.com", + "route": "98.97.32.0/21", + "type": "business", + }, + "threat": { + "is_tor": False, + "is_vpn": False, + "is_icloud_relay": False, + "is_proxy": False, + "is_datacenter": False, + "is_anonymous": False, + "is_known_attacker": False, + "is_known_abuser": False, + "is_threat": False, + "is_bogon": False, + "blocklists": [], + "scores": {"vpn_score": 0, "proxy_score": 1, "threat_score": 8, "trust_score": 97}, + }, + "count": "0", +} + +# Sample B: anycast address with dated blocklist entries (complete response). +ANYCAST_RESPONSE = { + "ip": "8.8.8.8", + "is_anycast": True, + "is_eu": False, + "city": None, + "region": None, + "region_code": None, + "region_type": None, + "country_name": "United States", + "country_code": "US", + "continent_name": "North America", + "continent_code": "NA", + "latitude": 39.7599983215332, + "longitude": -98.5, + "postal": None, + "calling_code": "1", + "flag": "https://ipdata.co/flags/us.png", + "emoji_flag": "\U0001F1FA\U0001F1F8", + "emoji_unicode": "U+1F1FA U+1F1F8", + "asn": { + "asn": "AS15169", + "name": "Google LLC", + "domain": "about.google", + "route": "8.8.8.0/24", + "type": "hosting", + }, + "company": { + "name": "Google LLC", + "domain": "about.google", + "network": "8.8.8.0/24", + "type": "hosting", + }, + "languages": [{"name": "English", "native": "English", "code": "en"}], + "currency": { + "name": "US Dollar", + "code": "USD", + "symbol": "$", + "native": "$", + "plural": "US dollars", + }, + "time_zone": { + "name": None, + "abbr": None, + "offset": None, + "is_dst": None, + "current_time": None, + }, + "threat": { + "is_tor": False, + "is_vpn": False, + "is_icloud_relay": False, + "is_proxy": False, + "is_datacenter": True, + "is_anonymous": False, + "is_known_attacker": False, + "is_known_abuser": False, + "is_threat": False, + "is_bogon": False, + "blocklists": [ + { + "days_seen": 1, + "first_seen": "2026-11-02", + "last_seen": "2026-11-02", + "name": "ipdata", + "site": "https://ipdata.co", + "type": "cloud", + }, + { + "days_seen": 1, + "first_seen": "2026-11-02", + "last_seen": "2026-11-02", + "name": "ipdata", + "site": "https://ipdata.co", + "type": "open_resolvers", + }, + ], + "scores": {"vpn_score": 100, "proxy_score": 93, "threat_score": 100, "trust_score": 2}, + }, + "count": "0", +} + +# Sample C: VPN exit. Neither top-level flag is present and the third blocklist entry has no dates. +VPN_RESPONSE = { + "ip": "187.13.250.1", + "is_eu": False, + "threat": { + "is_tor": False, + "is_vpn": True, + "is_icloud_relay": False, + "is_proxy": False, + "is_datacenter": True, + "is_anonymous": True, + "is_known_attacker": True, + "is_known_abuser": True, + "is_threat": True, + "is_bogon": False, + "blocklists": [ + { + "days_seen": 2, + "first_seen": "2026-10-31", + "last_seen": "2026-11-02", + "name": "Gluetun", + "site": "https://github.com/qdm12/gluetun", + "type": "vpn", + }, + { + "days_seen": 1, + "first_seen": "2026-11-02", + "last_seen": "2026-11-02", + "name": "ipdata", + "site": "https://ipdata.co", + "type": "cloud", + }, + {"name": "ipdata", "site": "https://ipdata.co", "type": "vpn"}, + { + "days_seen": 2, + "first_seen": "2026-10-31", + "last_seen": "2026-11-02", + "name": "NordVPN", + "site": "https://nordvpn.com", + "type": "vpn", + }, + { + "days_seen": 2, + "first_seen": "2026-10-31", + "last_seen": "2026-11-02", + "name": "Wikimedia Global Blocklist", + "site": "https://wikimedia.org", + "type": "general", + }, + { + "days_seen": 2, + "first_seen": "2026-10-31", + "last_seen": "2026-11-02", + "name": "X4BNet", + "site": "https://github.com/X4BNet/lists_vpn", + "type": "vpn", + }, + ], + "scores": {"vpn_score": 100, "proxy_score": 99, "threat_score": 100, "trust_score": 0}, + }, + "count": "0", +} + + +def make_client(payload): + """Returns an IPData client whose mock transport answers every request with 'payload', plus the requests.""" + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, json=payload) + + return IPData("test", transport=httpx.MockTransport(handler)), requests + + +def test_is_satellite_is_passed_through_and_absent_flags_read_false(): + client, _ = make_client(SATELLITE_RESPONSE) + + data = client.lookup("98.97.34.0") + + assert data.is_satellite is True + assert data["is_satellite"] is True + assert data.is_eu is False + # the API omits is_anycast when it is false; a missing key reads as None through dot notation + assert "is_anycast" not in data + assert data.is_anycast is None + assert not data.is_anycast + assert data.threat.blocklists == [] + assert data.threat.scores.trust_score == 97 + assert data.asn.name == "Space Exploration Technologies Corporation" + assert data.status == 200 + + +def test_is_anycast_with_dated_blocklist_entries(): + client, _ = make_client(ANYCAST_RESPONSE) + + data = client.lookup("8.8.8.8") + + assert data.is_anycast is True + assert "is_satellite" not in data + assert data.is_satellite is None + assert data.threat.is_datacenter is True + entries = data.threat.blocklists + assert [entry.type for entry in entries] == ["cloud", "open_resolvers"] + for entry in entries: + assert entry.name == "ipdata" + assert entry.site == "https://ipdata.co" + # the dates stay plain YYYY-MM-DD strings; they are not parsed into date objects + assert entry.first_seen == "2026-11-02" + assert entry.last_seen == "2026-11-02" + assert entry.days_seen == 1 + assert data.threat.scores.trust_score == 2 + assert data.time_zone.current_time is None + assert data.city is None + + +def test_blocklist_entry_without_dates(): + client, _ = make_client(VPN_RESPONSE) + + data = client.lookup("187.13.250.1") + + assert "is_satellite" not in data + assert "is_anycast" not in data + assert data.threat.is_vpn is True + entries = data.threat.blocklists + assert len(entries) == 6 + undated = entries[2] + assert undated.name == "ipdata" + assert undated.type == "vpn" + assert "first_seen" not in undated + assert undated.first_seen is None + assert undated.last_seen is None + assert undated.days_seen is None + assert entries[0].name == "Gluetun" + assert entries[0].first_seen == "2026-10-31" + assert entries[0].last_seen == "2026-11-02" + assert [entry.get("days_seen") for entry in entries] == [2, 1, None, 2, 2, 2] + + +def test_bulk_items_carry_the_same_shape(): + client, requests = make_client([SATELLITE_RESPONSE, ANYCAST_RESPONSE, VPN_RESPONSE]) + + data = client.bulk(["98.97.34.0", "8.8.8.8", "187.13.250.1"]) + + assert requests[0].method == "POST" + assert data.status == 200 + satellite, anycast, vpn = data.responses + assert satellite.is_satellite is True + assert satellite.is_anycast is None + assert anycast.is_anycast is True + assert anycast.is_satellite is None + assert anycast.threat.blocklists[1].type == "open_resolvers" + assert anycast.threat.blocklists[1].first_seen == "2026-11-02" + assert vpn.is_satellite is None + assert vpn.is_anycast is None + assert vpn.threat.blocklists[2].first_seen is None + assert vpn.threat.blocklists[3].days_seen == 2 + + +def test_payload_passes_through_unchanged_and_tolerates_unknown_keys(): + payload = dict(ANYCAST_RESPONSE, brand_new_field={"nested": "value", "numbers": [1, 2, 3]}) + client, _ = make_client(payload) + + data = client.lookup("8.8.8.8") + + # DotDict is a plain dict; 'status' is the only key the library adds + expected = dict(payload, status=200) + assert dict(data) == expected + assert json.loads(json.dumps(data)) == expected + assert data.brand_new_field.nested == "value" + assert data["brand_new_field"]["numbers"] == [1, 2, 3] + + +@pytest.mark.parametrize("field", ["is_satellite", "is_anycast"]) +def test_new_field_names_are_accepted_for_field_selection(field): + client, requests = make_client({"ip": "8.8.8.8", field: True}) + assert field in IPData.valid_fields + client._validate_fields([field]) + client._validate_fields({field, "ip"}) + client._validate_fields((field,)) + + data = client.lookup("8.8.8.8", fields=[field]) + assert requests[-1].url.params["fields"] == field + assert data[field] is True + + client.lookup("8.8.8.8", select_field=field) + assert requests[-1].url.params["fields"] == field + + +def test_both_new_fields_in_one_fields_list_for_lookup_and_bulk(): + client, requests = make_client([SATELLITE_RESPONSE]) + + client.bulk(["98.97.34.0"], fields=["ip", "is_satellite", "is_anycast", "threat"]) + assert requests[-1].url.params["fields"] == "ip,is_satellite,is_anycast,threat" + + with pytest.raises(ValueError): + client.lookup("98.97.34.0", fields=["is_satelite"]) # typo is still rejected